all

cli/flow.ts

@@ -12,7 +12,7 @@ import {
   yamlParse,
 } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { resolve, track_job } from "./script.ts";
+import { findGlobalDeps, resolve, track_job } from "./script.ts";
 import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts";
 import { generateFlowLockInternal } from "./metadata.ts";
 import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts";
@@ -237,8 +237,15 @@ async function generateLocks(
   await requireLogin(opts);
   opts = await mergeConfigWithConfigFile(opts);
   if (folder) {
+    const globalDeps = await findGlobalDeps();
     // read script metadata file
-    await generateFlowLockInternal(folder, false, workspace);
+    await generateFlowLockInternal(
+      folder,
+      false,
+      workspace,
+      globalDeps,
+      opts.defaultTs
+    );
   } else {
     const ignore = await ignoreF(opts);
     const elems = Object.keys(
@@ -258,8 +265,16 @@ async function generateLocks(
     ).map((x) => x.substring(0, x.lastIndexOf(SEP)));
     let hasAny = false;
 
+    const globalDeps = await findGlobalDeps();
+
     for (const folder of elems) {
-      const candidate = await generateFlowLockInternal(folder, true, workspace);
+      const candidate = await generateFlowLockInternal(
+        folder,
+        true,
+        workspace,
+        globalDeps,
+        opts.defaultTs
+      );
       if (candidate) {
         hasAny = true;
         log.info(colors.green(`+ ${candidate}`));
@@ -281,7 +296,13 @@ async function generateLocks(
       return;
     }
     for (const folder of elems) {
-      await generateFlowLockInternal(folder, false, workspace);
+      await generateFlowLockInternal(
+        folder,
+        false,
+        workspace,
+        globalDeps,
+        opts.defaultTs
+      );
     }
   }
 }

cli/metadata.ts

@@ -46,7 +46,7 @@ import { FlowFile, replaceInlineScripts } from "./flow.ts";
 export async function generateAllMetadata() {}
 
 function findClosestRawReqs(
-  lang: "bun" | "python3" | "php" | undefined,
+  lang: "bun" | "python3" | "php" | string | undefined,
   remotePath: string,
   globalDeps: GlobalDeps
 ): string | undefined {
@@ -84,20 +84,56 @@ function findClosestRawReqs(
 }
 
 const TOP_HASH = "__flow_hash";
-async function generateFlowHash(folder: string) {
+async function generateFlowHash(
+  folder: string,
+  globalDeps: GlobalDeps,
+  defaultTs: "bun" | "deno" | undefined
+): Promise<{
+  hashes: Record<string, string>;
+  pathsWithRawReqs: string[];
+}> {
   const elems = await FSFSElement(path.join(Deno.cwd(), folder), []);
   const hashes: Record<string, string> = {};
+  const pathsWithRawReqs: string[] = [];
+
   for await (const f of elems.getChildren()) {
-    if (exts.some((e) => f.path.endsWith(e))) {
-      hashes[f.path] = await generateHash(await f.getContentText());
+    const found_ext = exts.find((e) => f.path.endsWith(e));
+
+    if (found_ext) {
+      const lang = inferContentTypeFromFilePath(f.path, defaultTs);
+      // log.info(remote_path + "/script" + ext);
+      const raw = findClosestRawReqs(
+        lang as "bun" | "php" | "python3" | undefined,
+        path.join(folder, f.path),
+        globalDeps
+      );
+      // if (raw) {
+      //   rawReqs[ext] = raw;
+      // }
+
+      let text = await f.getContentText();
+      if (raw) {
+        text = raw + text;
+        pathsWithRawReqs.push(f.path);
+      }
+      hashes[f.path] = await generateHash(text);
     }
   }
-  return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
+  return {
+    hashes: {
+      ...hashes,
+      [TOP_HASH]: await generateHash(JSON.stringify(hashes)),
+    },
+    pathsWithRawReqs,
+  };
 }
+
 export async function generateFlowLockInternal(
   folder: string,
   dryRun: boolean,
   workspace: Workspace,
+  globalDeps: GlobalDeps,
+  defaultTs: "bun" | "deno" | undefined,
   justUpdateMetadataLock?: boolean
 ): Promise<string | undefined> {
   if (folder.endsWith(SEP)) {
@@ -110,7 +146,11 @@ export async function generateFlowLockInternal(
     log.info(`Generating lock for flow ${folder} at ${remote_path}`);
   }
 
-  let hashes = await generateFlowHash(folder);
+  let { hashes, pathsWithRawReqs } = await generateFlowHash(
+    folder,
+    globalDeps,
+    defaultTs
+  );
 
   const conf = await readLockfile();
   if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
@@ -126,6 +166,7 @@ export async function generateFlowLockInternal(
     await Deno.readTextFile(folder! + SEP + "flow.yaml")
   ) as FlowFile;
 
+  const fullReqs: Partial<Record<ScriptLanguage, string>> = {};
   if (!justUpdateMetadataLock) {
     const changedScripts = [];
     //find hashes that do not correspond to previous hashes
@@ -139,6 +180,19 @@ export async function generateFlowLockInternal(
     }
 
     log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
+
+    for (const script of changedScripts) {
+      if (pathsWithRawReqs.includes(script)) {
+        const lang = inferContentTypeFromFilePath(script, defaultTs);
+        log.info(`Inline script ${script} has raw requirements`);
+        let req = fullReqs[lang];
+        if (!req) {
+          const rawReq = findClosestRawReqs(lang, script, globalDeps) ?? "";
+          const fullReq = await getLockfileFromWindmill();
+          fullReqs[lang] = req;
+        }
+      }
+    }
     replaceInlineScripts(
       flowValue.value.modules,
       folder + SEP!,
@@ -161,7 +215,7 @@ export async function generateFlowLockInternal(
       });
   }
 
-  hashes = await generateFlowHash(folder);
+  hashes = (await generateFlowHash(folder, globalDeps, defaultTs)).hashes;
 
   for (const [path, hash] of Object.entries(hashes)) {
     await updateMetadataGlobalLock(folder, hash, path);
@@ -308,27 +362,13 @@ export async function updateScriptSchema(
   metadataContent.schema = newSchema;
 }
 
-async function updateScriptLock(
+async function getLockfileFromWindmill(
   workspace: Workspace,
   scriptContent: string,
   language: ScriptLanguage,
   remotePath: string,
-  metadataContent: Record<string, any>,
   rawDeps: string | undefined
-): Promise<void> {
-  if (
-    !(
-      language == "bun" ||
-      language == "python3" ||
-      language == "go" ||
-      language == "deno" ||
-      language == "php"
-    )
-  ) {
-    return;
-  }
-  // generate the script lock running a dependency job in Windmill and update it inplace
-  // TODO: update this once the client is released
+): Promise<{ lock: string | undefined }> {
   const rawResponse = await fetch(
     `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`,
     {
@@ -354,27 +394,7 @@ async function updateScriptLock(
   let responseText = "reading response failed";
   try {
     responseText = await rawResponse.text();
-    const response = JSON.parse(responseText);
-    const lock = response.lock;
-    if (lock === undefined) {
-      throw new Error(
-        `Failed to generate lockfile. Full response was: ${JSON.stringify(
-          response
-        )}`
-      );
-    }
-    const lockPath = remotePath + ".script.lock";
-    if (lock != "") {
-      await Deno.writeTextFile(lockPath, lock);
-      metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
-    } else {
-      try {
-        if (await Deno.stat(lockPath)) {
-          await Deno.remove(lockPath);
-        }
-      } catch {}
-      metadataContent.lock = "";
-    }
+    return JSON.parse(responseText);
   } catch (e) {
     throw new Error(
       `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
@@ -382,6 +402,57 @@ async function updateScriptLock(
   }
 }
 
+async function updateScriptLock(
+  workspace: Workspace,
+  scriptContent: string,
+  language: ScriptLanguage,
+  remotePath: string,
+  metadataContent: Record<string, any>,
+  rawDeps: string | undefined
+): Promise<void> {
+  if (
+    !(
+      language == "bun" ||
+      language == "python3" ||
+      language == "go" ||
+      language == "deno" ||
+      language == "php"
+    )
+  ) {
+    return;
+  }
+  // generate the script lock running a dependency job in Windmill and update it inplace
+  // TODO: update this once the client is released
+
+  const response = getLockfileFromWindmill(
+    workspace,
+    scriptContent,
+    language,
+    remotePath,
+    rawDeps
+  );
+  const lock = response.lock;
+  if (lock === undefined) {
+    throw new Error(
+      `Failed to generate lockfile. Full response was: ${JSON.stringify(
+        response
+      )}`
+    );
+  }
+  const lockPath = remotePath + ".script.lock";
+  if (lock != "") {
+    await Deno.writeTextFile(lockPath, lock);
+    metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
+  } else {
+    try {
+      if (await Deno.stat(lockPath)) {
+        await Deno.remove(lockPath);
+      }
+    } catch {}
+    metadataContent.lock = "";
+  }
+}
+
 export async function updateFlow(
   workspace: Workspace,
   flow_value: FlowValue,

cli/sync.ts

@@ -1070,7 +1070,14 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
     }
     for (const change of changedFlows) {
       log.info(`Updating lock for flow ${change}`);
-      await generateFlowLockInternal(change, false, workspace, true);
+      await generateFlowLockInternal(
+        change,
+        false,
+        workspace,
+        globalDeps,
+        opts.defaultTs,
+        true
+      );
     }
     log.info(
       colors.bold.green.underline(