diff --git a/cli/src/commands/dev/dev.ts b/cli/src/commands/dev/dev.ts index b2d86cc6ed..41b6b8fd9c 100644 --- a/cli/src/commands/dev/dev.ts +++ b/cli/src/commands/dev/dev.ts @@ -16,33 +16,34 @@ import { resolveWorkspace } from "../../core/context.ts"; import { SyncOptions, mergeConfigWithConfigFile, - readConfigFile, } from "../../core/conf.ts"; import { exts, removeExtensionToPath } from "../script/script.ts"; import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { OpenFlow } from "../../../gen/types.gen.ts"; import { FlowFile } from "../flow/flow.ts"; -import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; +import { replaceInlineScripts, replaceAllPathScriptsWithLocal } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; import { parseMetadataFile } from "../../utils/metadata.ts"; import { getFolderSuffixWithSep, getMetadataFileName, extractFolderPath, } from "../../utils/resource_folders.ts"; +import { listSyncCodebases } from "../../utils/codebase.ts"; +import { createPreviewLocalScriptReader } from "../../utils/local_path_scripts.ts"; const PORT = 3001; async function dev(opts: GlobalOptions & SyncOptions) { + opts = await mergeConfigWithConfigFile(opts); const workspace = await resolveWorkspace(opts); await requireLogin(opts); log.info("Started dev mode"); - const conf = await readConfigFile(); let currentLastEdit: LastEditScript | LastEditFlow | undefined = undefined; const fsWatcher = watch(".", { recursive: true }); const base = await realpath("."); - opts = await mergeConfigWithConfigFile(opts); const ignore = await ignoreF(opts); + const codebases = await listSyncCodebases(opts); const changesTimeouts: Record> = {}; function watchChanges() { @@ -56,7 +57,11 @@ async function dev(opts: GlobalOptions & SyncOptions) { } changesTimeouts[key] = setTimeout(async () => { delete changesTimeouts[key]; - await loadPaths([filePath]); + await loadPaths([filePath]).catch((error) => { + log.error( + `Failed to reload ${filePath}: ${error instanceof Error ? error.message : error}` + ); + }); }, 100); }); fsWatcher.on("error", (err) => { @@ -94,6 +99,13 @@ async function dev(opts: GlobalOptions & SyncOptions) { SEP, undefined, ); + // Replace PathScript modules with local file content so dev mode uses local versions + const localScriptReader = createPreviewLocalScriptReader({ + exts, + defaultTs: opts.defaultTs, + codebases, + }); + await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log); currentLastEdit = { type: "flow", flow: localFlow, @@ -105,7 +117,7 @@ async function dev(opts: GlobalOptions & SyncOptions) { const content = await readFile(cpath, "utf-8"); const splitted = cpath.split("."); const wmPath = splitted[0]; - const lang = inferContentTypeFromFilePath(cpath, conf.defaultTs); + const lang = inferContentTypeFromFilePath(cpath, opts.defaultTs); const typed = (await parseMetadataFile( removeExtensionToPath(cpath), diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 3b01385fb9..4d92405d01 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -18,8 +18,19 @@ import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts"; import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts"; import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts"; import { Flow } from "../../../gen/types.gen.ts"; -import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; +import { + collectPathScriptPaths, + replaceInlineScripts, + replaceAllPathScriptsWithLocal, +} from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; import { generateFlowLockInternal } from "./flow_metadata.ts"; +import { exts } from "../script/script.ts"; +import type { SyncCodebase } from "../../utils/codebase.ts"; +import { listSyncCodebases } from "../../utils/codebase.ts"; +import { + createPreviewLocalScriptReader, + resolvePreviewLocalScriptState, +} from "../../utils/local_path_scripts.ts"; export interface FlowFile { summary: string; @@ -28,6 +39,90 @@ export interface FlowFile { schema?: any; } +function normalizeOptionalString(value: string | null | undefined): string | undefined { + return typeof value === "string" && value.trim() === "" ? undefined : value ?? undefined; +} + +function normalizeComparableContent(value: string | undefined): string | undefined { + return value?.replaceAll("\r\n", "\n").replace(/\n$/, ""); +} + +async function findDivergedLocalPathScripts( + workspaceId: string, + scriptPaths: string[], + opts: { + exts: string[]; + defaultTs?: "bun" | "deno"; + codebases: SyncCodebase[]; + } +): Promise<{ changed: string[]; missing: string[] }> { + const changed: string[] = []; + const missing: string[] = []; + + for (const scriptPath of scriptPaths) { + const localScript = await resolvePreviewLocalScriptState(scriptPath, opts); + if (!localScript) { + continue; + } + + let remoteScript; + try { + remoteScript = await wmill.getScriptByPath({ + workspace: workspaceId, + path: scriptPath, + }); + } catch { + missing.push(scriptPath); + continue; + } + + const remoteLock = normalizeOptionalString(remoteScript.lock); + const diverged = + normalizeComparableContent(localScript.content) !== + normalizeComparableContent(remoteScript.content) || + localScript.language !== remoteScript.language || + (localScript.lock !== undefined && + normalizeComparableContent(localScript.lock) !== + normalizeComparableContent(remoteLock)) || + localScript.tag !== normalizeOptionalString(remoteScript.tag) || + localScript.codebaseDigest !== normalizeOptionalString(remoteScript.codebase); + + if (diverged) { + changed.push(scriptPath); + } + } + + return { changed, missing }; +} + +function warnAboutLocalPathScriptDivergence( + divergence: { changed: string[]; missing: string[] } +): void { + if (divergence.changed.length === 0 && divergence.missing.length === 0) { + return; + } + + const details: string[] = []; + if (divergence.changed.length > 0) { + details.push( + `These workspace scripts differ from the deployed version:\n${divergence.changed + .map((path) => `- ${path}`) + .join("\n")}` + ); + } + if (divergence.missing.length > 0) { + details.push( + `These scripts do not exist in the workspace yet:\n${divergence.missing + .map((path) => `- ${path}`) + .join("\n")}` + ); + } + + log.warn( + `Using local PathScript files for flow preview.\n${details.join("\n")}\nUse --remote to preview deployed workspace scripts instead.` + ); +} + const alreadySynced: string[] = []; export async function pushFlow( @@ -233,11 +328,17 @@ async function preview( opts: GlobalOptions & { data?: string; silent: boolean; - }, + remote?: boolean; + } & SyncOptions, flowPath: string ) { + const useLocalPathScripts = !opts.remote; + if (useLocalPathScripts) { + opts = await mergeConfigWithConfigFile(opts); + } const workspace = await resolveWorkspace(opts); await requireLogin(opts); + const codebases = useLocalPathScripts ? listSyncCodebases(opts) : []; // Normalize path - ensure it's a directory path to a .flow folder if (!flowPath.endsWith(".flow") && !flowPath.endsWith(".flow" + SEP)) { @@ -274,6 +375,31 @@ async function preview( await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, flowPath, SEP); } + if (useLocalPathScripts) { + const scriptPaths = collectPathScriptPaths(localFlow.value); + if (scriptPaths.length > 0) { + const divergence = await findDivergedLocalPathScripts( + workspace.workspaceId, + scriptPaths, + { + exts, + defaultTs: opts.defaultTs, + codebases, + } + ); + if (!opts.silent) { + warnAboutLocalPathScriptDivergence(divergence); + } + } + + const localScriptReader = createPreviewLocalScriptReader({ + exts, + defaultTs: opts.defaultTs, + codebases, + }); + await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log); + } + const input = opts.data ? await resolve(opts.data) : {}; if (!opts.silent) { @@ -444,7 +570,7 @@ const command = new Command() .action(run as any) .command( "preview", - "preview a local flow without deploying it. Runs the flow definition from local files." + "preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default." ) .arguments("") .option( @@ -455,6 +581,10 @@ const command = new Command() "-s --silent", "Do not output anything other then the final output. Useful for scripting." ) + .option( + "--remote", + "Use deployed workspace scripts for PathScript steps instead of local files." + ) .action(preview as any) .command( "generate-locks", diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index a742d52f28..a3867055da 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -5039,9 +5039,10 @@ flow related commands - \`flow run \` - run a flow by path. - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. -- \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files. +- \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. + - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`flow generate-locks [flow:file]\` - re-generate the lock files of all inline scripts of all updated flows - \`--yes\` - Skip confirmation prompt - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/cli/src/utils/local_path_scripts.ts b/cli/src/utils/local_path_scripts.ts new file mode 100644 index 0000000000..f4bf8aa4f4 --- /dev/null +++ b/cli/src/utils/local_path_scripts.ts @@ -0,0 +1,155 @@ +import { execFileSync } from "node:child_process"; +import { readFile, stat } from "node:fs/promises"; +import type { SyncCodebase } from "./codebase.ts"; +import { parseMetadataFileIfExists } from "./metadata.ts"; +import { inferContentTypeFromFilePath } from "./script_common.ts"; +import { findCodebase } from "../commands/sync/sync.ts"; +import type { LocalScriptInfo } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts"; +import type { RawScript } from "../../../gen/types.gen.ts"; + +export class UnsupportedLocalPathScriptPreviewError extends Error { + constructor(message: string) { + super(message); + this.name = "UnsupportedLocalPathScriptPreviewError"; + } +} + +async function readOptionalLock(scriptPath: string): Promise { + try { + return await readFile(scriptPath + ".script.lock", "utf-8"); + } catch { + return undefined; + } +} + +function normalizeOptionalLock(lock: string | undefined): string | undefined { + return typeof lock === "string" && lock.trim() === "" ? undefined : lock; +} + +async function bundleSingleFileCodebaseScript( + filePath: string, + codebase: SyncCodebase +): Promise { + if (codebase.customBundler) { + // Pass the script path as a positional shell argument so existing shell-based + // custom bundlers still work without interpolating the path into the command. + return execFileSync( + "sh", + ["-lc", `${codebase.customBundler} "$1"`, "sh", filePath], + { + maxBuffer: 1024 * 1024 * 50, + } + ).toString(); + } + + const esbuild = await import("esbuild"); + const out = await esbuild.build({ + entryPoints: [filePath], + // Inline rawscripts are executed through the standard module wrapper, + // so the bundle must expose `main` as an ESM export. + format: "esm", + bundle: true, + write: false, + external: codebase.external, + inject: codebase.inject, + define: codebase.define, + loader: codebase.loader ?? { ".node": "file" }, + outdir: "/", + platform: "node", + packages: "bundle", + target: "esnext", + banner: codebase.banner, + }); + + if (out.outputFiles.length === 0) { + throw new Error(`No output files found for ${filePath}`); + } + if (out.outputFiles.length > 1) { + throw new UnsupportedLocalPathScriptPreviewError( + `Local PathScript ${filePath} requires a multi-file bundle, which flow preview/dev cannot inline yet` + ); + } + if (Array.isArray(codebase.assets) && codebase.assets.length > 0) { + throw new UnsupportedLocalPathScriptPreviewError( + `Local PathScript ${filePath} requires codebase assets, which flow preview/dev cannot inline yet` + ); + } + + return out.outputFiles[0].text; +} + +export function createPreviewLocalScriptReader(opts: { + exts: string[]; + defaultTs?: "bun" | "deno"; + codebases: SyncCodebase[]; +}): (scriptPath: string) => Promise { + return async (scriptPath) => { + const localScript = await resolvePreviewLocalScriptState(scriptPath, opts); + if (!localScript) { + return undefined; + } + + const content = localScript.codebase + ? await bundleSingleFileCodebaseScript(localScript.filePath, localScript.codebase) + : localScript.content; + + return { + content, + language: localScript.language, + lock: localScript.lock, + tag: localScript.tag, + }; + }; +} + +export type PreviewLocalScriptState = { + filePath: string; + content: string; + language: RawScript["language"]; + lock?: string; + tag?: string; + codebase?: SyncCodebase; + codebaseDigest?: string; +}; + +export async function resolvePreviewLocalScriptState( + scriptPath: string, + opts: { + exts: string[]; + defaultTs?: "bun" | "deno"; + codebases: SyncCodebase[]; + } +): Promise { + for (const ext of opts.exts) { + const filePath = scriptPath + ext; + let fileStat; + try { + fileStat = await stat(filePath); + } catch { + continue; + } + if (!fileStat.isFile()) continue; + + const language = inferContentTypeFromFilePath(filePath, opts.defaultTs); + const metadata = await parseMetadataFileIfExists(scriptPath); + const rawLock = metadata?.payload?.lock ?? (await readOptionalLock(scriptPath)); + const codebase = + language === "bun" ? findCodebase(filePath, opts.codebases) : undefined; + + return { + filePath, + content: await readFile(filePath, "utf-8"), + language, + lock: normalizeOptionalLock(rawLock), + tag: metadata?.payload?.tag, + codebase, + codebaseDigest: codebase + ? await codebase.getDigest( + Array.isArray(codebase.assets) && codebase.assets.length > 0 + ) + : undefined, + }; + } + + return undefined; +} diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index f211e3dc71..0a3340cc6d 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -921,6 +921,38 @@ export function replaceLock(o?: { lock?: string | string[] }) { } } } + +export async function parseMetadataFileIfExists( + scriptPath: string +): Promise<{ isJson: boolean; payload: any; path: string } | undefined> { + let metadataFilePath = scriptPath + ".script.json"; + try { + await stat(metadataFilePath); + const payload = JSON.parse(await readFile(metadataFilePath, "utf-8")); + replaceLock(payload); + return { + path: metadataFilePath, + payload, + isJson: true, + }; + } catch { + try { + metadataFilePath = scriptPath + ".script.yaml"; + await stat(metadataFilePath); + const payload: any = await yamlParseFile(metadataFilePath); + replaceLock(payload); + + return { + path: metadataFilePath, + payload, + isJson: false, + }; + } catch { + return undefined; + } + } +} + export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: diff --git a/cli/test/preview.test.ts b/cli/test/preview.test.ts index 4b54184d1b..4ead0a06c9 100644 --- a/cli/test/preview.test.ts +++ b/cli/test/preview.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "bun:test"; import { mkdir, writeFile } from "node:fs/promises"; import { withTestBackend } from "./test_backend.ts"; +import { shouldSkipOnCI } from "./cargo_backend.ts"; // ============================================================================= // PREVIEW COMMAND INTEGRATION TESTS @@ -27,6 +28,7 @@ async function createWmillConfig( relative_path: string; includes?: string[]; format?: "cjs" | "esm"; + customBundler?: string; assets?: Array<{ from: string; to: string }>; }>; } @@ -44,6 +46,9 @@ async function createWmillConfig( if (cb.format) { yamlContent += ` format: ${cb.format}\n`; } + if (cb.customBundler) { + yamlContent += ` customBundler: ${JSON.stringify(cb.customBundler)}\n`; + } if (cb.assets && cb.assets.length > 0) { yamlContent += " assets:\n"; for (const asset of cb.assets) { @@ -122,6 +127,39 @@ schema: await writeFile(`${dir}/flow.yaml`, flowYaml, "utf-8"); } +async function createPathScriptFlow( + tempDir: string, + flowPath: string, + options: { + summary: string; + scriptPath: string; + inputTransforms?: string; + } +): Promise { + const dir = `${tempDir}/${flowPath}`; + await mkdir(dir, { recursive: true }); + const inputTransforms = options.inputTransforms + ? ` input_transforms:\n${options.inputTransforms}` + : " input_transforms: {}\n"; + + const flowYaml = `summary: "${options.summary}" +description: "Test flow" +value: + modules: + - id: "a" + value: + type: "script" + path: "${options.scriptPath}" +${inputTransforms} +schema: + $schema: "https://json-schema.org/draft/2020-12/schema" + type: object + properties: {} + required: [] +`; + await writeFile(`${dir}/flow.yaml`, flowYaml, "utf-8"); +} + // ============================================================================= // SCRIPT PREVIEW TESTS // ============================================================================= @@ -491,3 +529,280 @@ test("flow preview: simple flow", async () => { }); }); +test("flow preview: uses local PathScript by default and remote PathScript with --remote", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { defaultTs: "bun" }); + + await createScript( + tempDir, + "f/test/helper_script.ts", + `export function main(name: string = "World") { return \`Remote script says: \${name}!\`; }` + ); + + const pushResult = await backend.runCLICommand( + ["script", "push", "f/test/helper_script.ts"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + await writeFile( + `${tempDir}/f/test/helper_script.ts`, + `export function main(name: string = "World") { return \`Local script says: \${name}!\`; }`, + "utf-8" + ); + + await createPathScriptFlow(tempDir, "f/test/path_flow.flow", { + summary: "Flow with PathScript", + scriptPath: "f/test/helper_script", + inputTransforms: ` name: + type: "static" + value: "PathTest" +`, + }); + + const localResult = await backend.runCLICommand( + ["flow", "preview", "f/test/path_flow.flow"], + tempDir + ); + + expect(localResult.code).toEqual(0); + expect(localResult.stdout + localResult.stderr).toContain( + "Local script says: PathTest!" + ); + expect(localResult.stdout + localResult.stderr).toContain( + "Using local PathScript files for flow preview." + ); + expect(localResult.stdout + localResult.stderr).toContain( + "These workspace scripts differ from the deployed version:\n- f/test/helper_script" + ); + + const remoteResult = await backend.runCLICommand( + ["flow", "preview", "--remote", "f/test/path_flow.flow"], + tempDir + ); + + expect(remoteResult.code).toEqual(0); + expect(remoteResult.stdout + remoteResult.stderr).toContain( + "Remote script says: PathTest!" + ); + expect(remoteResult.stdout + remoteResult.stderr).not.toContain( + "Using local PathScript files for flow preview." + ); + }); +}); + +test.skipIf(shouldSkipOnCI())("flow preview: respects defaultTs when resolving local PathScripts", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { defaultTs: "deno" }); + + await createScript( + tempDir, + "f/test/deno_helper.ts", + `export function main() { return Deno.version.deno ? "deno-runtime" : "missing"; }` + ); + + await createPathScriptFlow(tempDir, "f/test/deno_path_flow.flow", { + summary: "Flow with Deno PathScript", + scriptPath: "f/test/deno_helper", + }); + + const result = await backend.runCLICommand( + ["flow", "preview", "f/test/deno_path_flow.flow"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout + result.stderr).toContain("deno-runtime"); + }); +}); + +test("flow preview: bundles local PathScripts with local imports", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { + defaultTs: "bun", + codebases: [{ relative_path: "f/flow_codebase", includes: ["**"] }], + }); + + await mkdir(`${tempDir}/f/flow_codebase`, { recursive: true }); + await writeFile( + `${tempDir}/f/flow_codebase/helper.ts`, + `export function greet(name: string): string { + return \`Hello from local flow codebase, \${name}!\`; +}`, + "utf-8" + ); + + await createScript( + tempDir, + "f/flow_codebase/main_script.ts", + `import { greet } from "./helper"; + +export function main(name: string = "World") { + return greet(name); +}` + ); + + await createPathScriptFlow(tempDir, "f/test/importing_path_flow.flow", { + summary: "Flow with imported PathScript", + scriptPath: "f/flow_codebase/main_script", + inputTransforms: ` name: + type: "static" + value: "FlowTest" +`, + }); + + const result = await backend.runCLICommand( + ["flow", "preview", "f/test/importing_path_flow.flow"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout + result.stderr).toContain( + "Hello from local flow codebase, FlowTest!" + ); + }); +}); + +test("flow preview: customBundler handles script paths with spaces", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { + defaultTs: "bun", + codebases: [{ + relative_path: "f/codebase custom", + includes: ["f/codebase custom/**"], + customBundler: "cat", + }], + }); + + await createScript( + tempDir, + "f/codebase custom/custom bundler.ts", + `export function main() { + return "Custom bundler path with spaces"; +}` + ); + + await createPathScriptFlow(tempDir, "f/test/custom_bundler_path.flow", { + summary: "Flow with customBundler path", + scriptPath: "f/codebase custom/custom bundler", + }); + + const result = await backend.runCLICommand( + ["flow", "preview", "f/test/custom_bundler_path.flow"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout + result.stderr).toContain( + "Custom bundler path with spaces" + ); + }); +}); + +test("flow preview: warns when local PathScript is not deployed remotely", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { defaultTs: "bun" }); + + await createScript( + tempDir, + "f/test/undeployed_helper.ts", + `export function main() { return "Local only script"; }` + ); + + await createPathScriptFlow(tempDir, "f/test/undeployed_path_flow.flow", { + summary: "Flow with undeployed PathScript", + scriptPath: "f/test/undeployed_helper", + }); + + const result = await backend.runCLICommand( + ["flow", "preview", "f/test/undeployed_path_flow.flow"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout + result.stderr).toContain("Local only script"); + expect(result.stdout + result.stderr).toContain( + "These scripts do not exist in the workspace yet:\n- f/test/undeployed_helper" + ); + }); +}); + +test("flow preview: does not warn when local and deployed PathScripts match", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { defaultTs: "bun" }); + + await createScript( + tempDir, + "f/test/matching_helper.ts", + `export function main() { return "Matching script"; }` + ); + + const pushResult = await backend.runCLICommand( + ["script", "push", "f/test/matching_helper.ts"], + tempDir + ); + expect(pushResult.code).toEqual(0); + + await createPathScriptFlow(tempDir, "f/test/matching_path_flow.flow", { + summary: "Flow with matching PathScript", + scriptPath: "f/test/matching_helper", + }); + + const result = await backend.runCLICommand( + ["flow", "preview", "f/test/matching_path_flow.flow"], + tempDir + ); + + expect(result.code).toEqual(0); + expect(result.stdout + result.stderr).toContain("Matching script"); + expect(result.stdout + result.stderr).not.toContain( + "Using local PathScript files for flow preview." + ); + }); +}); + +test("flow preview: fails loudly for asset-backed codebase scripts", async () => { + await withTestBackend(async (backend, tempDir) => { + await createWmillConfig(tempDir, { + defaultTs: "bun", + codebases: [{ + relative_path: "f/codebase_tar", + includes: ["**"], + assets: [{ from: "f/codebase_tar/data.json", to: "data.json" }], + }], + }); + + await mkdir(`${tempDir}/f/codebase_tar`, { recursive: true }); + await writeFile( + `${tempDir}/f/codebase_tar/data.json`, + JSON.stringify({ message: "Hello from asset!" }), + "utf-8" + ); + + await createScript( + tempDir, + "f/codebase_tar/main_script.ts", + `import * as fs from "fs"; + +export function main() { + const data = JSON.parse(fs.readFileSync("data.json", "utf-8")); + return data.message; +}` + ); + + await createPathScriptFlow(tempDir, "f/test/assets_path_flow.flow", { + summary: "Flow with asset-backed PathScript", + scriptPath: "f/codebase_tar/main_script", + }); + + const localResult = await backend.runCLICommand( + ["flow", "preview", "f/test/assets_path_flow.flow"], + tempDir + ); + + expect(localResult.code).not.toEqual(0); + expect(localResult.stdout + localResult.stderr).toContain( + "requires codebase assets" + ); + }); +}); diff --git a/cli/test/replace_path_scripts.test.ts b/cli/test/replace_path_scripts.test.ts new file mode 100644 index 0000000000..0e8ba5649e --- /dev/null +++ b/cli/test/replace_path_scripts.test.ts @@ -0,0 +1,322 @@ +/** + * Unit tests for replacePathScriptsWithLocal. + * + * Tests that PathScript ("script" type) modules are correctly converted + * to RawScript ("rawscript" type) using local file content during + * flow preview / dev mode. + */ + +import { expect, test, describe } from "bun:test"; +import { replacePathScriptsWithLocal, type LocalScriptInfo } from "../windmill-utils-internal/src/inline-scripts/replacer.ts"; +import type { FlowModule } from "../windmill-utils-internal/src/gen/types.gen.ts"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makePathScriptModule( + id: string, + path: string, + inputTransforms: Record = {}, + tagOverride?: string, +): FlowModule { + return { + id, + value: { + type: "script" as const, + path, + input_transforms: inputTransforms, + tag_override: tagOverride, + }, + }; +} + +function makeRawscriptModule( + id: string, + content: string, + language: "bun" | "python3" | "deno" = "bun", +): FlowModule { + return { + id, + value: { + type: "rawscript" as const, + content, + language, + input_transforms: {}, + }, + }; +} + +const noopLogger = { + info: () => {}, + error: () => {}, +}; + +// --------------------------------------------------------------------------- +// Basic conversion tests +// --------------------------------------------------------------------------- + +describe("replacePathScriptsWithLocal", () => { + test("converts PathScript to RawScript when local file exists", async () => { + const module = makePathScriptModule("a", "f/scripts/my_script", { + x: { type: "static", value: 42 }, + }); + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/my_script") { + return { + content: 'export function main() { return "hello"; }', + language: "bun", + lock: "some-lock", + }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + + expect(module.value.type).toBe("rawscript"); + expect((module.value as any).content).toBe('export function main() { return "hello"; }'); + expect((module.value as any).language).toBe("bun"); + expect((module.value as any).lock).toBe("some-lock"); + expect((module.value as any).path).toBe("f/scripts/my_script"); + expect((module.value as any).input_transforms).toEqual({ x: { type: "static", value: 42 } }); + }); + + test("preserves tag_override as tag", async () => { + const module = makePathScriptModule("a", "f/scripts/tagged", {}, "my-worker"); + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/tagged") { + return { content: "code", language: "python3", tag: "script-worker" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + + expect(module.value.type).toBe("rawscript"); + expect((module.value as any).tag).toBe("my-worker"); + }); + + test("uses local script tag when no tag_override is set", async () => { + const module = makePathScriptModule("a", "f/scripts/tagged"); + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/tagged") { + return { content: "code", language: "python3", tag: "script-worker" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + + expect(module.value.type).toBe("rawscript"); + expect((module.value as any).tag).toBe("script-worker"); + }); + + test("leaves PathScript untouched when local file not found", async () => { + const module = makePathScriptModule("a", "f/scripts/remote_only"); + + const scriptReader = async (): Promise => undefined; + + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + + expect(module.value.type).toBe("script"); + expect((module.value as any).path).toBe("f/scripts/remote_only"); + }); + + test("does not affect rawscript modules", async () => { + const module = makeRawscriptModule("a", "existing code", "bun"); + + const scriptReader = async (): Promise => { + throw new Error("should not be called for rawscript"); + }; + + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + + expect(module.value.type).toBe("rawscript"); + expect((module.value as any).content).toBe("existing code"); + }); + + test("handles mixed PathScript and RawScript modules", async () => { + const pathModule = makePathScriptModule("a", "f/scripts/local_script"); + const rawModule = makeRawscriptModule("b", "inline code", "bun"); + const remoteModule = makePathScriptModule("c", "f/scripts/remote_only"); + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/local_script") { + return { content: "local code", language: "python3" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal( + [pathModule, rawModule, remoteModule], + scriptReader, + noopLogger, + ); + + // PathScript with local file → converted + expect(pathModule.value.type).toBe("rawscript"); + expect((pathModule.value as any).content).toBe("local code"); + + // RawScript → untouched + expect(rawModule.value.type).toBe("rawscript"); + expect((rawModule.value as any).content).toBe("inline code"); + + // PathScript without local file → untouched + expect(remoteModule.value.type).toBe("script"); + expect((remoteModule.value as any).path).toBe("f/scripts/remote_only"); + }); + + test("handles module without lock", async () => { + const module = makePathScriptModule("a", "f/scripts/no_lock"); + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/no_lock") { + return { content: "code", language: "go" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + + expect(module.value.type).toBe("rawscript"); + expect((module.value as any).lock).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// Nested structure tests +// --------------------------------------------------------------------------- + +describe("replacePathScriptsWithLocal nested structures", () => { + test("processes modules inside forloopflow", async () => { + const innerModule = makePathScriptModule("inner", "f/scripts/loop_script"); + const loopModule: FlowModule = { + id: "loop", + value: { + type: "forloopflow" as const, + iterator: { type: "static" as const, value: "" }, + modules: [innerModule], + skip_failures: false, + }, + }; + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/loop_script") { + return { content: "loop code", language: "bun" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([loopModule], scriptReader, noopLogger); + + expect(innerModule.value.type).toBe("rawscript"); + expect((innerModule.value as any).content).toBe("loop code"); + }); + + test("processes modules inside whileloopflow", async () => { + const innerModule = makePathScriptModule("inner", "f/scripts/while_script"); + const whileModule: FlowModule = { + id: "while", + value: { + type: "whileloopflow" as const, + modules: [innerModule], + skip_failures: false, + }, + }; + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/while_script") { + return { content: "while code", language: "bun" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([whileModule], scriptReader, noopLogger); + + expect(innerModule.value.type).toBe("rawscript"); + expect((innerModule.value as any).content).toBe("while code"); + }); + + test("processes modules inside branchall", async () => { + const branch1Module = makePathScriptModule("b1", "f/scripts/branch1"); + const branch2Module = makePathScriptModule("b2", "f/scripts/branch2"); + const branchAllModule: FlowModule = { + id: "branches", + value: { + type: "branchall" as const, + branches: [ + { modules: [branch1Module], skip_failure: false }, + { modules: [branch2Module], skip_failure: false }, + ], + }, + }; + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/branch1") { + return { content: "branch1 code", language: "bun" }; + } + if (scriptPath === "f/scripts/branch2") { + return { content: "branch2 code", language: "python3" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([branchAllModule], scriptReader, noopLogger); + + expect(branch1Module.value.type).toBe("rawscript"); + expect((branch1Module.value as any).content).toBe("branch1 code"); + expect(branch2Module.value.type).toBe("rawscript"); + expect((branch2Module.value as any).content).toBe("branch2 code"); + expect((branch2Module.value as any).language).toBe("python3"); + }); + + test("processes modules inside branchone (branches + default)", async () => { + const branchModule = makePathScriptModule("b1", "f/scripts/branch"); + const defaultModule = makePathScriptModule("d1", "f/scripts/default"); + const branchOneModule: FlowModule = { + id: "branchone", + value: { + type: "branchone" as const, + branches: [ + { modules: [branchModule], expr: "true" }, + ], + default: [defaultModule], + }, + }; + + const scriptReader = async (scriptPath: string): Promise => { + if (scriptPath === "f/scripts/branch") { + return { content: "branch code", language: "bun" }; + } + if (scriptPath === "f/scripts/default") { + return { content: "default code", language: "bash" }; + } + return undefined; + }; + + await replacePathScriptsWithLocal([branchOneModule], scriptReader, noopLogger); + + expect(branchModule.value.type).toBe("rawscript"); + expect((branchModule.value as any).content).toBe("branch code"); + expect(defaultModule.value.type).toBe("rawscript"); + expect((defaultModule.value as any).content).toBe("default code"); + expect((defaultModule.value as any).language).toBe("bash"); + }); + + test("handles empty modules array", async () => { + const scriptReader = async (): Promise => undefined; + // Should not throw + await replacePathScriptsWithLocal([], scriptReader, noopLogger); + }); + + test("handles module with undefined value gracefully", async () => { + const module: FlowModule = { id: "x", value: undefined as any }; + const scriptReader = async (): Promise => undefined; + // Should not throw + await replacePathScriptsWithLocal([module], scriptReader, noopLogger); + }); +}); diff --git a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts index 03a159ee83..11b2cfaa1b 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts @@ -1,4 +1,11 @@ -import { FlowModule, RawScript } from "../gen/types.gen.ts"; +import { AiAgent, FlowModule, FlowValue, RawScript } from "../gen/types.gen.ts"; + +export type LocalScriptInfo = { + content: string; + language: RawScript["language"]; + lock?: string; + tag?: string; +}; async function replaceRawscriptInline( id: string, @@ -118,4 +125,154 @@ export async function replaceInlineScripts( })); } })); - } \ No newline at end of file + } + +/** + * Replaces PathScript ("script" type) modules with RawScript ("rawscript" type) using local file content. + * This is used during flow preview so that local script changes are tested instead of remote versions. + * + * @param modules - Array of flow modules to process + * @param scriptReader - Function that takes a script path and returns local content/language/lock, or undefined if not found locally + * @param logger - Logger for info/error messages + */ +export async function replacePathScriptsWithLocal( + modules: FlowModule[], + scriptReader: (scriptPath: string) => Promise, + logger: { + info: (message: string) => void; + error: (message: string) => void; + } = { + info: () => {}, + error: () => {}, + } +): Promise { + await Promise.all(modules.map(async (module) => { + if (!module.value) { + return; + } + + if (module.value.type === "script") { + const scriptPath = module.value.path; + const localScript = await scriptReader(scriptPath); + if (localScript) { + const pathScript = module.value; + module.value = { + type: "rawscript", + content: localScript.content, + language: localScript.language, + lock: localScript.lock, + path: scriptPath, + input_transforms: pathScript.input_transforms, + tag: pathScript.tag_override ?? localScript.tag, + } satisfies RawScript; + } + } else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") { + await replacePathScriptsWithLocal(module.value.modules, scriptReader, logger); + } else if (module.value.type === "branchall") { + await Promise.all(module.value.branches.map(async (branch) => { + await replacePathScriptsWithLocal(branch.modules, scriptReader, logger); + })); + } else if (module.value.type === "branchone") { + await Promise.all(module.value.branches.map(async (branch) => { + await replacePathScriptsWithLocal(branch.modules, scriptReader, logger); + })); + await replacePathScriptsWithLocal(module.value.default, scriptReader, logger); + } else if (module.value.type === "aiagent") { + await Promise.all((module.value.tools ?? []).map(async (tool) => { + const toolValue = tool.value; + if (!toolValue || toolValue.tool_type !== "flowmodule" || toolValue.type !== "script") { + return; + } + const localScript = await scriptReader(toolValue.path); + if (localScript) { + (tool as AiAgent["tools"][number]).value = { + tool_type: "flowmodule", + type: "rawscript", + content: localScript.content, + language: localScript.language, + lock: localScript.lock, + path: toolValue.path, + input_transforms: toolValue.input_transforms, + tag: toolValue.tag_override ?? localScript.tag, + }; + } + })); + } + })); +} + +function collectPathScriptPathsFromModules( + modules: FlowModule[], + paths: Set +): void { + for (const module of modules) { + if (!module.value) { + continue; + } + + if (module.value.type === "script") { + paths.add(module.value.path); + } else if ( + module.value.type === "forloopflow" || + module.value.type === "whileloopflow" + ) { + collectPathScriptPathsFromModules(module.value.modules, paths); + } else if (module.value.type === "branchall") { + for (const branch of module.value.branches) { + collectPathScriptPathsFromModules(branch.modules, paths); + } + } else if (module.value.type === "branchone") { + for (const branch of module.value.branches) { + collectPathScriptPathsFromModules(branch.modules, paths); + } + collectPathScriptPathsFromModules(module.value.default, paths); + } else if (module.value.type === "aiagent") { + for (const tool of module.value.tools ?? []) { + const toolValue = tool.value; + if ( + toolValue && + toolValue.tool_type === "flowmodule" && + toolValue.type === "script" + ) { + paths.add(toolValue.path); + } + } + } + } +} + +/** + * Replaces all PathScript modules in a flow value (modules, failure_module, preprocessor_module) + * with RawScript using local file content. + */ +export async function replaceAllPathScriptsWithLocal( + flowValue: FlowValue, + scriptReader: (scriptPath: string) => Promise, + logger: { + info: (message: string) => void; + error: (message: string) => void; + } = { + info: () => {}, + error: () => {}, + } +): Promise { + await replacePathScriptsWithLocal(flowValue.modules, scriptReader, logger); + if (flowValue.failure_module) { + await replacePathScriptsWithLocal([flowValue.failure_module], scriptReader, logger); + } + if (flowValue.preprocessor_module) { + await replacePathScriptsWithLocal([flowValue.preprocessor_module], scriptReader, logger); + } +} + +export function collectPathScriptPaths(flowValue: FlowValue): string[] { + const paths = new Set(); + collectPathScriptPathsFromModules(flowValue.modules, paths); + if (flowValue.failure_module) { + collectPathScriptPathsFromModules([flowValue.failure_module], paths); + } + if (flowValue.preprocessor_module) { + collectPathScriptPathsFromModules([flowValue.preprocessor_module], paths); + } + return [...paths]; +} diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 4b0f934175..d22d5b2856 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -86,9 +86,10 @@ flow related commands - `flow run ` - run a flow by path. - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting. -- `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files. +- `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. + - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files. - `flow generate-locks [flow:file]` - re-generate the lock files of all inline scripts of all updated flows - `--yes` - Skip confirmation prompt - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index d7f2d14922..069c4b922c 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1460,9 +1460,10 @@ flow related commands - \`flow run \` - run a flow by path. - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. -- \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files. +- \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. + - \`--remote\` - Use deployed workspace scripts for PathScript steps instead of local files. - \`flow generate-locks [flow:file]\` - re-generate the lock files of all inline scripts of all updated flows - \`--yes\` - Skip confirmation prompt - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index b31f479626..1290fb1395 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -91,9 +91,10 @@ flow related commands - `flow run ` - run a flow by path. - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. - `-s --silent` - Do not ouput anything other then the final output. Useful for scripting. -- `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files. +- `flow preview ` - preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default. - `-d --data ` - Inputs specified as a JSON string or a file using @ or stdin using @-. - `-s --silent` - Do not output anything other then the final output. Useful for scripting. + - `--remote` - Use deployed workspace scripts for PathScript steps instead of local files. - `flow generate-locks [flow:file]` - re-generate the lock files of all inline scripts of all updated flows - `--yes` - Skip confirmation prompt - `-i --includes ` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)