From cd4151a84b2c1e0f2e616079091d0429bf469f4e Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Wed, 18 Feb 2026 13:17:54 +0100 Subject: [PATCH] perf(cli): skip relock more accurate (#7993) * perf(cli): skip relock more accurate Signed-off-by: pyranota * Update cli/src/utils/metadata.ts Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * Update cli/src/commands/flow/flow_metadata.ts Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * fix Signed-off-by: pyranota * use structuredClone for safety Signed-off-by: pyranota * chore: update ee-repo-ref to 592848d59ca2304926fb2bd85d000668a7f46a77 This commit updates the EE repository reference after PR #420 was merged in windmill-ee-private. Previous ee-repo-ref: 931813b75b8260faa13ddc07f36a11607b7e3bf6 New ee-repo-ref: 592848d59ca2304926fb2bd85d000668a7f46a77 Automated by sync-ee-ref workflow. * fix ci Signed-off-by: pyranota * add simple tests Signed-off-by: pyranota --------- Signed-off-by: pyranota Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- cli/src/commands/app/app_metadata.ts | 70 +- cli/src/commands/flow/flow_metadata.ts | 45 +- cli/src/commands/sync/sync.ts | 2 +- cli/src/utils/metadata.ts | 116 +++- cli/test.nu | 32 +- cli/test/workspace_deps_filter.test.ts | 600 ++++++++++++++++++ .../src/inline-scripts/extractor.ts | 12 +- 8 files changed, 820 insertions(+), 59 deletions(-) create mode 100644 cli/test/workspace_deps_filter.test.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a288a4215f..2ffcfc7f02 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -931813b75b8260faa13ddc07f36a11607b7e3bf6 \ No newline at end of file +592848d59ca2304926fb2bd85d000668a7f46a77 diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 2a4ded06ae..51c8a97ffd 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -16,6 +16,9 @@ import { inferSchema, getRawWorkspaceDependencies, normalizeLockPath, + filterWorkspaceDependencies, + filterWorkspaceDependenciesForScripts, + InlineScriptInfo, } from "../../utils/metadata.ts"; import { ScriptLanguage, @@ -118,8 +121,23 @@ export async function generateAppLocksInternal( const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); - let hashes = await generateAppHash( + // Read the app file first to filter workspace dependencies + const appFilePath = path.join( + appFolder, + rawApp ? "raw_app.yaml" : "app.yaml" + ); + const appFile = (await yamlParseFile(appFilePath)) as AppFile; + + // Filter workspace dependencies based on inline scripts' languages and annotations + const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value; + const filteredDeps = await filterWorkspaceDependenciesForApp( + appValue, rawWorkspaceDependencies, + appFolder + ); + + let hashes = await generateAppHash( + filteredDeps, appFolder, rawApp, opts.defaultTs @@ -141,7 +159,7 @@ export async function generateAppLocksInternal( return remote_path; } - if (Object.keys(rawWorkspaceDependencies).length > 0) { + if (Object.keys(filteredDeps).length > 0) { log.info( (await blueColor())( `Found workspace dependencies (${workspaceDependenciesLanguages @@ -151,13 +169,6 @@ export async function generateAppLocksInternal( ); } - // Read the app file - const appFilePath = path.join( - appFolder, - rawApp ? "raw_app.yaml" : "app.yaml" - ); - const appFile = (await yamlParseFile(appFilePath)) as AppFile; - if (!justUpdateMetadataLock) { const changedScripts = []; // Find hashes that do not correspond to previous hashes @@ -195,7 +206,7 @@ export async function generateAppLocksInternal( runnables, remote_path, appFolder, - rawWorkspaceDependencies, + filteredDeps, opts.defaultTs ); // Note: updateRawAppRunnables now writes each runnable to its own file @@ -211,7 +222,7 @@ export async function generateAppLocksInternal( normalAppFile.value, remote_path, appFolder, - rawWorkspaceDependencies, + filteredDeps, opts.defaultTs ); @@ -228,7 +239,7 @@ export async function generateAppLocksInternal( // Regenerate hashes after updates hashes = await generateAppHash( - rawWorkspaceDependencies, + filteredDeps, appFolder, rawApp, opts.defaultTs @@ -252,6 +263,31 @@ type InlineScriptProcessor = ( } ) => Promise; +/** + * Filters raw workspace dependencies for an app by traversing all inline scripts, + * filtering deps for each based on language and annotations, then computing the union. + */ +export async function filterWorkspaceDependenciesForApp( + appValue: any, + rawWorkspaceDependencies: Record, + folder: string +): Promise> { + // Collect all inline scripts (use clone to avoid any mutations) + const scripts: InlineScriptInfo[] = []; + + await traverseAndProcessInlineScripts(structuredClone(appValue), async (inlineScript) => { + if (inlineScript.content && inlineScript.language) { + scripts.push({ + content: inlineScript.content, + language: inlineScript.language as ScriptLanguage, + }); + } + return inlineScript; + }); + + return await filterWorkspaceDependenciesForScripts(scripts, rawWorkspaceDependencies, folder, SEP); +} + /** * Traverses an app structure (either app.value for normal apps or app.runnables for raw apps) * and processes all inline scripts found, returning the updated structure @@ -552,6 +588,11 @@ async function generateInlineScriptLock( scriptPath: string, rawWorkspaceDependencies: Record | undefined ): Promise { + // Filter workspace dependencies to only include those matching this script's language and annotations + const filteredDeps = rawWorkspaceDependencies + ? filterWorkspaceDependencies(rawWorkspaceDependencies, content, language as ScriptLanguage) + : undefined; + const extraHeaders = getHeaders(); const rawResponse = await fetch( @@ -572,9 +613,8 @@ async function generateInlineScriptLock( }, ], raw_workspace_dependencies: - rawWorkspaceDependencies && - Object.keys(rawWorkspaceDependencies).length > 0 - ? rawWorkspaceDependencies + filteredDeps && Object.keys(filteredDeps).length > 0 + ? filteredDeps : null, entrypoint: scriptPath, }), diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index b0565830d0..26883f7512 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -16,7 +16,9 @@ import { LockfileGenerationError, getRawWorkspaceDependencies, normalizeLockPath, + filterWorkspaceDependenciesForScripts, } from "../../utils/metadata.ts"; +import { ScriptLanguage } from "../../utils/script_common.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; @@ -70,8 +72,16 @@ export async function generateFlowLockInternal( // Always get out-of-sync workspace dependencies const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); + + const flowValue = (await yamlParseFile( + folder! + SEP + "flow.yaml" + )) as FlowFile; + + // Filter workspace dependencies based on inline scripts' languages and annotations + const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); + let hashes = await generateFlowHash( - rawWorkspaceDependencies, + filteredDeps, folder, opts.defaultTs ); @@ -88,7 +98,7 @@ export async function generateFlowLockInternal( return remote_path; } - if (Object.keys(rawWorkspaceDependencies).length > 0) { + if (Object.keys(filteredDeps).length > 0) { log.info( (await blueColor())( `Found workspace dependencies (${workspaceDependenciesLanguages @@ -98,9 +108,6 @@ export async function generateFlowLockInternal( ); } - const flowValue = (await yamlParseFile( - folder! + SEP + "flow.yaml" - )) as FlowFile; if (!justUpdateMetadataLock) { const changedScripts = []; @@ -131,7 +138,7 @@ export async function generateFlowLockInternal( workspace, flowValue.value, remote_path, - rawWorkspaceDependencies + filteredDeps ); const inlineScripts = extractInlineScriptsForFlows( @@ -152,7 +159,7 @@ export async function generateFlowLockInternal( } hashes = await generateFlowHash( - rawWorkspaceDependencies, + filteredDeps, folder, opts.defaultTs ); @@ -163,6 +170,25 @@ export async function generateFlowLockInternal( log.info(colors.green(`Flow ${remote_path} lockfiles updated`)); } +/** + * Filters raw workspace dependencies for a flow by extracting all inline scripts, + * filtering deps for each based on language and annotations, then computing the union. + */ +async function filterWorkspaceDependenciesForFlow( + flowValue: FlowValue, + rawWorkspaceDependencies: Record, + folder: string +): Promise> { + const inlineScripts = extractInlineScriptsForFlows(structuredClone(flowValue.modules), {}, SEP, undefined); + + // Filter out lock files and map to common interface + const scripts = inlineScripts + .filter(s => !s.is_lock) + .map(s => ({ content: s.content, language: s.language as ScriptLanguage })); + + return await filterWorkspaceDependenciesForScripts(scripts, rawWorkspaceDependencies, folder, SEP); +} + export async function updateFlow( workspace: Workspace, flow_value: FlowValue, @@ -191,10 +217,7 @@ export async function updateFlow( flow_value, path: remotePath, use_local_lockfiles: true, - raw_workspace_dependencies: - Object.keys(rawWorkspaceDependencies).length > 0 - ? rawWorkspaceDependencies - : null, + raw_workspace_dependencies: rawWorkspaceDependencies, }), } ); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 10b4b856d8..b7ceb86f75 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2018,7 +2018,7 @@ export async function pull( log.info(`Updating lock metadata for raw app ${change}`); await generateAppLocksInternal( change, - false, + true, true, workspace, opts, diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 5ed715cc9d..25ce5fdaa5 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -54,19 +54,90 @@ export async function getRawWorkspaceDependencies(): Promise, + scriptContent: string, + language: ScriptLanguage +): Record { + const wda = extractWorkspaceDepsAnnotation(scriptContent, language); + const filtered: Record = {}; + + for (const [depPath, depContent] of Object.entries(rawWorkspaceDependencies)) { + const depInfo = workspaceDependenciesPathToLanguageAndFilename(depPath); + + if (depInfo && depInfo.language === language) { + if ((wda && wda.external.includes(depInfo.name ?? "default")) || (wda == null && depInfo.name == undefined)) { + filtered[depPath] = depContent; } } + } + + return filtered; +} + +export interface InlineScriptInfo { + content: string; + language: ScriptLanguage; +} + +/** + * Filters workspace dependencies for multiple scripts, resolving !inline refs and computing union. + * Common helper used by flows and apps. + */ +export async function filterWorkspaceDependenciesForScripts( + scripts: InlineScriptInfo[], + rawWorkspaceDependencies: Record, + folder: string, + sep: string +): Promise> { + const filtered: Record = {}; + + for (const script of scripts) { + let content = script.content; + + // Resolve !inline reference to actual content + if (content.startsWith("!inline ")) { + const filePath = folder + sep + content.replace("!inline ", ""); + try { + content = await Deno.readTextFile(filePath); + } catch { + continue; + } + } + + const scriptFiltered = filterWorkspaceDependencies( + rawWorkspaceDependencies, + content, + script.language + ); + + for (const [depPath, depContent] of Object.entries(scriptFiltered)) { + filtered[depPath] = depContent; + } + } + + return filtered; } // on windows, when using powershell, blue is not readable @@ -95,14 +166,6 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); - // Filter workspace dependencies to only include those matching the script's language - const filteredRawWorkspaceDependencies: Record = {}; - for (const [depPath, depContent] of Object.entries(rawWorkspaceDependencies)) { - const depInfo = workspaceDependenciesPathToLanguageAndFilename(depPath); - if (depInfo && depInfo.language === language) { - filteredRawWorkspaceDependencies[depPath] = depContent; - } - } const metadataWithType = await parseMetadataFile( remotePath, @@ -112,7 +175,14 @@ export async function generateScriptMetadataInternal( // read script content const scriptContent = await Deno.readTextFile(scriptPath); const metadataContent = await Deno.readTextFile(metadataWithType.path); - + + const filteredRawWorkspaceDependencies = filterWorkspaceDependencies( + rawWorkspaceDependencies, + scriptContent, + language + ); + + // Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent); @@ -671,12 +741,12 @@ export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: | (GlobalOptions & { - path: string; - workspaceRemote: Workspace; - schemaOnly?: boolean; - rawWorkspaceDependencies: Record; - codebases: SyncCodebase[] - }) + path: string; + workspaceRemote: Workspace; + schemaOnly?: boolean; + rawWorkspaceDependencies: Record; + codebases: SyncCodebase[] + }) | undefined ): Promise<{ isJson: boolean; payload: any; path: string }> { let metadataFilePath = scriptPath + ".script.json"; diff --git a/cli/test.nu b/cli/test.nu index af0a66882d..57eabb0dba 100755 --- a/cli/test.nu +++ b/cli/test.nu @@ -30,12 +30,12 @@ const languages = [ ]; def main [] { - main test deno; + main test languages deno; main build; - main test node; + main test languages node; } -def 'main test deno' [] { +def 'main test languages deno' [] { main clean; print $"Testing Deno" $languages | each { |l| @@ -46,7 +46,7 @@ def 'main test deno' [] { print $"\nDeno has passed!\n" } -def 'main test node' [] { +def 'main test languages node' [] { main clean; print $"Testing Node" $languages | each { |l| @@ -69,6 +69,30 @@ def 'main build' [] { cd ../local } +alias wm-cli = deno run -A --no-check ../cli/src/main.ts + + +def 'main setup workspace_deps' [] { + mkdir f/workspace_deps + mkdir dependencies + + wm-cli script bootstrap f/workspace_deps/test python3 + "# requirements: test" | save -f f/workspace_deps/test.py + + wm-cli script bootstrap f/workspace_deps/demo python3 + "# requirements: demo" | save -f f/workspace_deps/demo.py + + wm-cli script bootstrap f/workspace_deps/default python3 + "# requirements: default" | save -f f/workspace_deps/default.py + + # wm-cli script bootstrap f/workspace_deps/default_and_demo python3 + # "# requirements: default" | save -f f/workspace_deps/default_and_demo.py + + "tiny" | save -f dependencies/test.requirements.in + "tiny" | save -f dependencies/demo.requirements.in + "tiny" | save -f dependencies/requirements.in +} + # def main [] { # main clean; # $languages | each { |l| diff --git a/cli/test/workspace_deps_filter.test.ts b/cli/test/workspace_deps_filter.test.ts new file mode 100644 index 0000000000..eab576cd0d --- /dev/null +++ b/cli/test/workspace_deps_filter.test.ts @@ -0,0 +1,600 @@ +/** + * Workspace Dependencies Filtering Tests + * + * Tests that workspace dependencies are correctly filtered based on: + * 1. Script language (only matching language deps) + * 2. Annotations (explicit deps vs default) + * + * These tests use --dry-run mode to avoid needing workers to generate locks. + * We pre-compute hashes to simulate an initial state, then verify that + * changing specific deps only marks the expected scripts as stale. + */ + +import { assertEquals, assertStringIncludes, assert } from "https://deno.land/std@0.224.0/assert/mod.ts"; +import { withTestBackend } from "./test_backend.ts"; +import { addWorkspace } from "../workspace.ts"; +import { ensureDir } from "https://deno.land/std@0.224.0/fs/mod.ts"; +import { stringify as stringifyYaml } from "jsr:@std/yaml"; + +// Import hash generation utilities from CLI +import { generateHash } from "../src/utils/utils.ts"; +import { filterWorkspaceDependencies } from "../src/utils/metadata.ts"; +import { filterWorkspaceDependenciesForApp } from "../src/commands/app/app_metadata.ts"; + +/** + * Generate a script hash the same way the CLI does + */ +async function generateScriptHash( + filteredWorkspaceDependencies: Record, + scriptContent: string, + metadataContent: string +): Promise { + return await generateHash( + JSON.stringify(filteredWorkspaceDependencies) + scriptContent + metadataContent + ); +} + +/** + * Helper to create wmill-lock.yaml content + */ +function createLockfile(locks: Record): string { + return stringifyYaml({ version: "v2", locks }); +} + +// ============================================================================= +// Test 1: Scripts - changing default dep only marks scripts without annotation as stale +// ============================================================================= + +Deno.test({ + name: "Workspace deps: Scripts - dry-run shows correct stale scripts when default dep changes", + ignore: false, + sanitizeResources: false, + sanitizeOps: false, + fn: async () => { + await withTestBackend(async (backend, tempDir) => { + // Set up workspace + const testWorkspace = { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "workspace_deps_test", + token: backend.token + }; + await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir }); + + // Create wmill.yaml + await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - "**" +excludes: []`); + + // Setup dependencies folder (Bun/TypeScript) + await ensureDir(`${tempDir}/dependencies`); + const defaultDep = `{"dependencies": {"lodash": "4.17.21"}}`; + const explicitDep = `{"dependencies": {"axios": "1.6.0"}}`; + await Deno.writeTextFile(`${tempDir}/dependencies/package.json`, defaultDep); + await Deno.writeTextFile(`${tempDir}/dependencies/explicit.package.json`, explicitDep); + + // Setup script folder + await ensureDir(`${tempDir}/f/test`); + + // Script 1: No annotation - uses default dep + const script1Content = `export async function main() { + return "uses default deps"; +} +`; + const script1Metadata = `summary: "uses default" +schema: + type: object + properties: {} +lock: "" +`; + await Deno.writeTextFile(`${tempDir}/f/test/uses_default.ts`, script1Content); + await Deno.writeTextFile(`${tempDir}/f/test/uses_default.script.yaml`, script1Metadata); + + // Script 2: Uses explicit dep (TypeScript/Bun with annotation) + const script2Content = `// package_json: explicit +export async function main() { + return "uses explicit deps"; +} +`; + const script2Metadata = `summary: "uses explicit" +schema: + type: object + properties: {} +lock: "" +`; + await Deno.writeTextFile(`${tempDir}/f/test/uses_explicit.ts`, script2Content); + await Deno.writeTextFile(`${tempDir}/f/test/uses_explicit.script.yaml`, script2Metadata); + + // Build raw workspace dependencies map (as the CLI would) + const rawWorkspaceDeps: Record = { + "dependencies/package.json": defaultDep, + "dependencies/explicit.package.json": explicitDep, + }; + + // Filter deps for each script and compute hashes + const script1FilteredDeps = filterWorkspaceDependencies(rawWorkspaceDeps, script1Content, "bun"); + const script2FilteredDeps = filterWorkspaceDependencies(rawWorkspaceDeps, script2Content, "bun"); + + const script1Hash = await generateScriptHash(script1FilteredDeps, script1Content, script1Metadata); + const script2Hash = await generateScriptHash(script2FilteredDeps, script2Content, script2Metadata); + + // Create initial wmill-lock.yaml with these hashes + await Deno.writeTextFile(`${tempDir}/wmill-lock.yaml`, createLockfile({ + "f/test/uses_default": script1Hash, + "f/test/uses_explicit": script2Hash, + })); + + // Verify initial state - both scripts should be up-to-date + const initialResult = await backend.runCLICommand( + ["script", "generate-metadata", "-i", "f/test/uses_default*,f/test/uses_explicit*", "--yes", "--dry-run"], + tempDir, + "workspace_deps_test" + ); + assertEquals(initialResult.code, 0, `Initial dry-run should succeed: ${initialResult.stderr}`); + assertStringIncludes(initialResult.stdout, "No metadata to update", + `Initial state should show no updates needed. Output: ${initialResult.stdout}`); + + // Now change package.json (default dep) + const newDefaultDep = `{"dependencies": {"lodash": "4.17.22"}}`; + await Deno.writeTextFile(`${tempDir}/dependencies/package.json`, newDefaultDep); + + // Run dry-run again + const afterDefaultChangeResult = await backend.runCLICommand( + ["script", "generate-metadata", "-i", "f/test/uses_default*,f/test/uses_explicit*", "--yes", "--dry-run"], + tempDir, + "workspace_deps_test" + ); + assertEquals(afterDefaultChangeResult.code, 0, `Dry-run should succeed: ${afterDefaultChangeResult.stderr}`); + + // uses_default should be stale (uses default dep which changed) + assertStringIncludes(afterDefaultChangeResult.stdout, "uses_default", + `uses_default should be marked stale after default dep change. Output: ${afterDefaultChangeResult.stdout}`); + + // uses_explicit should NOT be stale (uses explicit dep, not default) + assert(!afterDefaultChangeResult.stdout.includes("uses_explicit"), + `uses_explicit should NOT be marked stale after default dep change. Output: ${afterDefaultChangeResult.stdout}`); + + // Reset and test the reverse: change explicit dep + await Deno.writeTextFile(`${tempDir}/dependencies/package.json`, defaultDep); // restore original + const newExplicitDep = `{"dependencies": {"axios": "1.6.1"}}`; + await Deno.writeTextFile(`${tempDir}/dependencies/explicit.package.json`, newExplicitDep); + + // Run dry-run again + const afterExplicitChangeResult = await backend.runCLICommand( + ["script", "generate-metadata", "-i", "f/test/uses_default*,f/test/uses_explicit*", "--yes", "--dry-run"], + tempDir, + "workspace_deps_test" + ); + assertEquals(afterExplicitChangeResult.code, 0, `Dry-run should succeed: ${afterExplicitChangeResult.stderr}`); + + // uses_explicit should be stale (uses explicit dep which changed) + assertStringIncludes(afterExplicitChangeResult.stdout, "uses_explicit", + `uses_explicit should be marked stale after explicit dep change. Output: ${afterExplicitChangeResult.stdout}`); + + // uses_default should NOT be stale (uses default dep, not explicit) + assert(!afterExplicitChangeResult.stdout.includes("uses_default"), + `uses_default should NOT be marked stale after explicit dep change. Output: ${afterExplicitChangeResult.stdout}`); + }); + }, +}); + +// ============================================================================= +// Test 2: Flows - filterWorkspaceDependenciesForScripts correctly filters by annotation +// ============================================================================= + +Deno.test({ + name: "Workspace deps: Flows - filterWorkspaceDependenciesForScripts correctly filters inline scripts", + ignore: false, + sanitizeResources: false, + sanitizeOps: false, + fn: async () => { + // This test verifies the filtering logic used by flows without needing workers + // We test filterWorkspaceDependenciesForScripts directly since flow generate-locks + // doesn't have a --dry-run option + + const defaultDep = `{"dependencies": {"lodash": "4.17.21"}}`; + const explicitDep = `{"dependencies": {"axios": "1.6.0"}}`; + + const rawWorkspaceDeps: Record = { + "dependencies/package.json": defaultDep, + "dependencies/explicit.package.json": explicitDep, + }; + + // Script using default dep (no annotation) + const defaultScriptContent = `export async function main() { + return "uses default deps"; +} +`; + + // Script using explicit dep (with annotation) + const explicitScriptContent = `// package_json: explicit +export async function main() { + return "uses explicit deps"; +} +`; + + // Filter for default script - should only include default dep + const defaultFiltered = filterWorkspaceDependencies(rawWorkspaceDeps, defaultScriptContent, "bun"); + assertEquals(Object.keys(defaultFiltered).length, 1, + `Default script should have 1 filtered dep, got: ${JSON.stringify(defaultFiltered)}`); + assert("dependencies/package.json" in defaultFiltered, + `Default script should have package.json`); + assert(!("dependencies/explicit.package.json" in defaultFiltered), + `Default script should NOT have explicit.package.json`); + + // Filter for explicit script - should only include explicit dep + const explicitFiltered = filterWorkspaceDependencies(rawWorkspaceDeps, explicitScriptContent, "bun"); + assertEquals(Object.keys(explicitFiltered).length, 1, + `Explicit script should have 1 filtered dep, got: ${JSON.stringify(explicitFiltered)}`); + assert("dependencies/explicit.package.json" in explicitFiltered, + `Explicit script should have explicit.package.json`); + assert(!("dependencies/package.json" in explicitFiltered), + `Explicit script should NOT have package.json`); + + // Verify hashes change correctly when deps change + const defaultHash1 = await generateScriptHash(defaultFiltered, defaultScriptContent, "metadata"); + const explicitHash1 = await generateScriptHash(explicitFiltered, explicitScriptContent, "metadata"); + + // Change default dep + const newDefaultDep = `{"dependencies": {"lodash": "4.17.22"}}`; + const newRawWorkspaceDeps1: Record = { + "dependencies/package.json": newDefaultDep, + "dependencies/explicit.package.json": explicitDep, + }; + + const defaultFiltered2 = filterWorkspaceDependencies(newRawWorkspaceDeps1, defaultScriptContent, "bun"); + const explicitFiltered2 = filterWorkspaceDependencies(newRawWorkspaceDeps1, explicitScriptContent, "bun"); + + const defaultHash2 = await generateScriptHash(defaultFiltered2, defaultScriptContent, "metadata"); + const explicitHash2 = await generateScriptHash(explicitFiltered2, explicitScriptContent, "metadata"); + + // Default script hash should change (its dep changed) + assert(defaultHash1 !== defaultHash2, + `Default script hash should change when default dep changes`); + + // Explicit script hash should NOT change (its dep didn't change) + assertEquals(explicitHash1, explicitHash2, + `Explicit script hash should NOT change when default dep changes`); + + // Now change explicit dep + const newExplicitDep = `{"dependencies": {"axios": "1.6.1"}}`; + const newRawWorkspaceDeps2: Record = { + "dependencies/package.json": defaultDep, // back to original + "dependencies/explicit.package.json": newExplicitDep, + }; + + const defaultFiltered3 = filterWorkspaceDependencies(newRawWorkspaceDeps2, defaultScriptContent, "bun"); + const explicitFiltered3 = filterWorkspaceDependencies(newRawWorkspaceDeps2, explicitScriptContent, "bun"); + + const defaultHash3 = await generateScriptHash(defaultFiltered3, defaultScriptContent, "metadata"); + const explicitHash3 = await generateScriptHash(explicitFiltered3, explicitScriptContent, "metadata"); + + // Default script hash should be back to original (dep is back to original) + assertEquals(defaultHash1, defaultHash3, + `Default script hash should be same as original when dep reverts`); + + // Explicit script hash should change (its dep changed) + assert(explicitHash1 !== explicitHash3, + `Explicit script hash should change when explicit dep changes`); + }, +}); + +// ============================================================================= +// Test 3: Cross-language isolation - Python dep change doesn't affect Bun script +// ============================================================================= + +Deno.test({ + name: "Workspace deps: Cross-language - Python dep change doesn't affect Bun script", + ignore: false, + sanitizeResources: false, + sanitizeOps: false, + fn: async () => { + await withTestBackend(async (backend, tempDir) => { + // Set up workspace + const testWorkspace = { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "workspace_deps_cross_lang_test", + token: backend.token + }; + await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir }); + + // Create wmill.yaml + await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - "**" +excludes: []`); + + // Setup dependencies folder with deps for multiple languages + await ensureDir(`${tempDir}/dependencies`); + const pythonDep = "requests==2.31.0"; + const bunDep = `{"dependencies": {"lodash": "4.17.21"}}`; + await Deno.writeTextFile(`${tempDir}/dependencies/requirements.in`, pythonDep); + await Deno.writeTextFile(`${tempDir}/dependencies/package.json`, bunDep); + + // Setup script folder + await ensureDir(`${tempDir}/f/test`); + + // Python script + const pythonContent = `def main(): + return "python script" +`; + const pythonMetadata = `summary: "python script" +schema: + type: object + properties: {} +lock: "" +`; + await Deno.writeTextFile(`${tempDir}/f/test/python_script.py`, pythonContent); + await Deno.writeTextFile(`${tempDir}/f/test/python_script.script.yaml`, pythonMetadata); + + // Bun script + const bunContent = `export async function main() { + return "bun script"; +} +`; + const bunMetadata = `summary: "bun script" +schema: + type: object + properties: {} +lock: "" +`; + await Deno.writeTextFile(`${tempDir}/f/test/bun_script.ts`, bunContent); + await Deno.writeTextFile(`${tempDir}/f/test/bun_script.script.yaml`, bunMetadata); + + // Build raw workspace dependencies map + const rawWorkspaceDeps: Record = { + "dependencies/requirements.in": pythonDep, + "dependencies/package.json": bunDep, + }; + + // Filter deps for each script - this is where language filtering happens + const pythonFilteredDeps = filterWorkspaceDependencies(rawWorkspaceDeps, pythonContent, "python3"); + const bunFilteredDeps = filterWorkspaceDependencies(rawWorkspaceDeps, bunContent, "bun"); + + // Python script should only get requirements.in + assertEquals(Object.keys(pythonFilteredDeps).length, 1, + `Python script should only have 1 filtered dep, got: ${JSON.stringify(pythonFilteredDeps)}`); + assert("dependencies/requirements.in" in pythonFilteredDeps, + `Python script should have requirements.in in filtered deps`); + + // Bun script should only get package.json + assertEquals(Object.keys(bunFilteredDeps).length, 1, + `Bun script should only have 1 filtered dep, got: ${JSON.stringify(bunFilteredDeps)}`); + assert("dependencies/package.json" in bunFilteredDeps, + `Bun script should have package.json in filtered deps`); + + // Compute initial hashes + const pythonHash = await generateScriptHash(pythonFilteredDeps, pythonContent, pythonMetadata); + const bunHash = await generateScriptHash(bunFilteredDeps, bunContent, bunMetadata); + + // Create initial wmill-lock.yaml + await Deno.writeTextFile(`${tempDir}/wmill-lock.yaml`, createLockfile({ + "f/test/python_script": pythonHash, + "f/test/bun_script": bunHash, + })); + + // Verify initial state - both scripts should be up-to-date + const initialResult = await backend.runCLICommand( + ["script", "generate-metadata", "-i", "f/test/python_script*,f/test/bun_script*", "--yes", "--dry-run"], + tempDir, + "workspace_deps_cross_lang_test" + ); + assertEquals(initialResult.code, 0, `Initial dry-run should succeed: ${initialResult.stderr}`); + assertStringIncludes(initialResult.stdout, "No metadata to update", + `Initial state should show no updates needed. Output: ${initialResult.stdout}`); + + // Change Python dep (requirements.in) + await Deno.writeTextFile(`${tempDir}/dependencies/requirements.in`, "requests==2.32.0"); + + // Run dry-run + const afterPythonChangeResult = await backend.runCLICommand( + ["script", "generate-metadata", "-i", "f/test/python_script*,f/test/bun_script*", "--yes", "--dry-run"], + tempDir, + "workspace_deps_cross_lang_test" + ); + assertEquals(afterPythonChangeResult.code, 0, `Dry-run should succeed: ${afterPythonChangeResult.stderr}`); + + // python_script should be stale + assertStringIncludes(afterPythonChangeResult.stdout, "python_script", + `python_script should be marked stale after Python dep change. Output: ${afterPythonChangeResult.stdout}`); + + // bun_script should NOT be stale (different language) + assert(!afterPythonChangeResult.stdout.includes("bun_script"), + `bun_script should NOT be marked stale after Python dep change. Output: ${afterPythonChangeResult.stdout}`); + + // Reset and test the reverse + await Deno.writeTextFile(`${tempDir}/dependencies/requirements.in`, pythonDep); + await Deno.writeTextFile(`${tempDir}/dependencies/package.json`, `{"dependencies": {"lodash": "4.17.22"}}`); + + const afterBunChangeResult = await backend.runCLICommand( + ["script", "generate-metadata", "-i", "f/test/python_script*,f/test/bun_script*", "--yes", "--dry-run"], + tempDir, + "workspace_deps_cross_lang_test" + ); + assertEquals(afterBunChangeResult.code, 0, `Dry-run should succeed: ${afterBunChangeResult.stderr}`); + + // bun_script should be stale + assertStringIncludes(afterBunChangeResult.stdout, "bun_script", + `bun_script should be marked stale after Bun dep change. Output: ${afterBunChangeResult.stdout}`); + + // python_script should NOT be stale (different language) + assert(!afterBunChangeResult.stdout.includes("python_script"), + `python_script should NOT be marked stale after Bun dep change. Output: ${afterBunChangeResult.stdout}`); + }); + }, +}); + +// ============================================================================= +// Test 4: Apps - Create app via API and test filterWorkspaceDependenciesForApp +// ============================================================================= + +Deno.test({ + name: "Workspace deps: Apps - filterWorkspaceDependenciesForApp with real app via API", + ignore: false, + sanitizeResources: false, + sanitizeOps: false, + fn: async () => { + await withTestBackend(async (backend, tempDir) => { + // Set up workspace + const testWorkspace = { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "workspace_deps_app_test", + token: backend.token + }; + await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir }); + + // Create wmill.yaml + await Deno.writeTextFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - "**" +excludes: []`); + + // Create app with multiple inline scripts via backend API + const appPath = "f/test/multi_script_app"; + const appValue = { + type: "app", + grid: [ + { + id: "default_bun", + data: { + type: "buttoncomponent", + componentInput: { + type: "runnable", + runnable: { + type: "runnableByName", + inlineScript: { + content: `export async function main() { return "uses default bun deps"; }`, + language: "bun", + }, + }, + }, + }, + }, + { + id: "explicit_bun", + data: { + type: "buttoncomponent", + componentInput: { + type: "runnable", + runnable: { + type: "runnableByName", + inlineScript: { + content: `// package_json: explicit\nexport async function main() { return "uses explicit bun deps"; }`, + language: "bun", + }, + }, + }, + }, + }, + { + id: "python_script", + data: { + type: "buttoncomponent", + componentInput: { + type: "runnable", + runnable: { + type: "runnableByName", + inlineScript: { + content: `def main():\n return "python script"`, + language: "python3", + }, + }, + }, + }, + }, + ], + hiddenInlineScripts: [], + css: {}, + norefreshbar: false, + }; + + // Create app via API + const createResponse = await backend.apiRequest!( + `/api/w/${backend.workspace}/apps/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: appPath, + value: appValue, + summary: "Multi-script app for workspace deps testing", + policy: { + on_behalf_of: null, + on_behalf_of_email: null, + triggerables: {}, + execution_mode: "viewer", + }, + }), + } + ); + assertEquals(createResponse.ok, true, `Failed to create app: ${await createResponse.text()}`); + + // Pull the app to disk + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir, + "workspace_deps_app_test" + ); + assertEquals(pullResult.code, 0, `Sync pull should succeed: ${pullResult.stderr}`); + + // Setup workspace dependencies + await ensureDir(`${tempDir}/dependencies`); + const defaultBunDep = `{"dependencies": {"lodash": "4.17.21"}}`; + const explicitBunDep = `{"dependencies": {"axios": "1.6.0"}}`; + const pythonDep = "requests==2.31.0"; + await Deno.writeTextFile(`${tempDir}/dependencies/package.json`, defaultBunDep); + await Deno.writeTextFile(`${tempDir}/dependencies/explicit.package.json`, explicitBunDep); + await Deno.writeTextFile(`${tempDir}/dependencies/requirements.in`, pythonDep); + + // Read the pulled app.yaml + const { yamlParseFile } = await import("../deps.ts"); + const appFilePath = `${tempDir}/${appPath}.app/app.yaml`; + const appFile = await yamlParseFile(appFilePath); + + const rawWorkspaceDeps: Record = { + "dependencies/package.json": defaultBunDep, + "dependencies/explicit.package.json": explicitBunDep, + "dependencies/requirements.in": pythonDep, + }; + + // Call the app-specific filtering API + const filteredDeps = await filterWorkspaceDependenciesForApp( + appFile.value, + rawWorkspaceDeps, + `${tempDir}/${appPath}.app` + ); + + // Verify all 3 dep types are included + assertEquals(Object.keys(filteredDeps).length, 3, + `App with bun (default), bun (explicit), and python should have 3 filtered deps, got: ${JSON.stringify(filteredDeps)}`); + assert("dependencies/package.json" in filteredDeps, + `Should include default package.json for default bun script`); + assert("dependencies/explicit.package.json" in filteredDeps, + `Should include explicit.package.json for annotated bun script`); + assert("dependencies/requirements.in" in filteredDeps, + `Should include requirements.in for python script`); + + // Verify hash changes when deps change + const hash1 = await generateHash(JSON.stringify(filteredDeps)); + + const newDefaultBunDep = `{"dependencies": {"lodash": "4.17.22"}}`; + const newRawDeps: Record = { + "dependencies/package.json": newDefaultBunDep, + "dependencies/explicit.package.json": explicitBunDep, + "dependencies/requirements.in": pythonDep, + }; + + const filteredDeps2 = await filterWorkspaceDependenciesForApp( + appFile.value, + newRawDeps, + `${tempDir}/${appPath}.app` + ); + + const hash2 = await generateHash(JSON.stringify(filteredDeps2)); + assert(hash1 !== hash2, `Hash should change when filtered deps change`); + }); + }, +}); diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index d7be5338da..df9a1db08b 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -1,14 +1,17 @@ import { newPathAssigner, PathAssigner } from "../path-utils/path-assigner.ts"; -import { FlowModule, RawScript } from "../gen/types.gen.ts"; +import { FlowModule, RawScript, ScriptLang } from "../gen/types.gen.ts"; /** * Represents an inline script extracted from a flow module */ -interface InlineScript { +export interface InlineScript { /** File path where the script content should be written */ path: string; /** The actual script content */ content: string; + /** The script language */ + language: ScriptLang; + is_lock: boolean; } function extractRawscriptInline( @@ -21,14 +24,15 @@ function extractRawscriptInline( ): InlineScript[] { const [basePath, ext] = assigner.assignPath(summary ?? id, rawscript.language); const path = mapping[id] ?? basePath + ext; + const language = rawscript.language; const content = rawscript.content; - const r = [{ path: path, content: content }]; + const r = [{ path: path, content: content, language, is_lock: false}]; rawscript.content = "!inline " + path.replaceAll(separator, "/"); const lock = rawscript.lock; if (lock && lock != "") { const lockPath = basePath + "lock"; rawscript.lock = "!inline " + lockPath.replaceAll(separator, "/"); - r.push({ path: lockPath, content: lock }); + r.push({ path: lockPath, content: lock, language, is_lock: true}); } return r; }