cli yaml parsing error more verbose

2024-10-12 03:01:46 +02:00
parent f82f091290
commit e06c845ed4
12 changed files with 132 additions and 75 deletions
--- a/cli/apps.ts
+++ b/cli/apps.ts
@@ -1,6 +1,6 @@
 // deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { colors, Command, log, SEP, Table, yamlParse } from "./deps.ts";
+import { colors, Command, log, SEP, Table, yamlParseFile } from "./deps.ts";
 import * as wmill from "./gen/services.gen.ts";
 import { ListableApp, Policy } from "./gen/types.gen.ts";

@@ -40,8 +40,8 @@ export async function pushApp(
  if (!localPath.endsWith(SEP)) {
    localPath += SEP;
  }
-  const localAppRaw = await Deno.readTextFile(localPath + "app.yaml");
-  const localApp = yamlParse(localAppRaw) as AppFile;
+  const path = localPath + "app.yaml";
+  const localApp = (await yamlParseFile(path)) as AppFile;

  function replaceInlineScripts(rec: any) {
    if (!rec) {
--- a/cli/conf.ts
+++ b/cli/conf.ts
@@ -1,4 +1,4 @@
-import { log, yamlParse } from "./deps.ts";
+import { log, yamlParseFile } from "./deps.ts";

 export interface SyncOptions {
  stateful?: boolean;
@@ -40,9 +40,7 @@ export interface Codebase {

 export async function readConfigFile(): Promise<SyncOptions> {
  try {
-    const conf = yamlParse(
-      await Deno.readTextFile("wmill.yaml")
-    ) as SyncOptions;
+    const conf = (await yamlParseFile("wmill.yaml")) as SyncOptions;
    if (conf?.defaultTs == undefined) {
      log.warn(
        "No defaultTs defined in your wmill.yaml. Using 'bun' as default."
--- a/cli/deps.ts
+++ b/cli/deps.ts
@@ -21,7 +21,29 @@ export { copy } from "jsr:@std/io/copy";
 export { readAll } from "jsr:@std/io/read-all";

 export * as log from "jsr:@std/log";
-export { stringify as yamlStringify, parse as yamlParse } from "jsr:@std/yaml";
+export { stringify as yamlStringify } from "jsr:@std/yaml";
+
+import { parse as yamlParse, ParseOptions } from "jsr:@std/yaml";
+
+export async function yamlParseFile(path: string, options: ParseOptions = {}) {
+  try {
+    return yamlParse(await Deno.readTextFile(path), options);
+  } catch (e) {
+    throw new Error(`Error parsing yaml ${path}`, { cause: e });
+  }
+}
+
+export function yamlParseContent(
+  path: string,
+  content: string,
+  options: ParseOptions = {}
+) {
+  try {
+    return yamlParse(content, options);
+  } catch (e) {
+    throw new Error(`Error parsing yaml ${path}`, { cause: e });
+  }
+}

 // other

--- a/cli/flow.ts
+++ b/cli/flow.ts
@@ -1,7 +1,7 @@
 // deno-lint-ignore-file no-explicit-any
 import { GlobalOptions, isSuperset } from "./types.ts";
 import { Confirm, SEP, log, yamlStringify } from "./deps.ts";
-import { colors, Command, Table, yamlParse } from "./deps.ts";
+import { colors, Command, Table, yamlParseFile } from "./deps.ts";
 import * as wmill from "./gen/services.gen.ts";

 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
@@ -89,8 +89,7 @@ export async function pushFlow(
  if (!localPath.endsWith(SEP)) {
    localPath += SEP;
  }
-  const localFlowRaw = await Deno.readTextFile(localPath + "flow.yaml");
-  const localFlow = yamlParse(localFlowRaw) as FlowFile;
+  const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile;

  replaceInlineScripts(localFlow.value.modules, localPath, undefined);

--- a/cli/gen/core/OpenAPI.ts
+++ b/cli/gen/core/OpenAPI.ts
@@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = {
 	PASSWORD: undefined,
 	TOKEN: getEnv("WM_TOKEN"),
 	USERNAME: undefined,
-	VERSION: '1.401.0',
+	VERSION: '1.407.2',
 	WITH_CREDENTIALS: true,
 	interceptors: {
 		request: new Interceptors(),
--- a/cli/gen/types.gen.ts
+++ b/cli/gen/types.gen.ts
@@ -1015,6 +1015,9 @@ export type FlowModule = {
        skip_if_stopped?: boolean;
        expr: string;
    };
+    skip_if?: {
+        expr: string;
+    };
    sleep?: InputTransform;
    cache_ttl?: number;
    timeout?: number;
@@ -1176,6 +1179,7 @@ export type FlowStatusModule = {
        approver: string;
    }>;
    failed_retries?: Array<(string)>;
+    skipped?: boolean;
 };

 export type type4 = 'WaitingForPriorSteps' | 'WaitingForEvents' | 'WaitingForExecutor' | 'InProgress' | 'Success' | 'Failure';
--- a/cli/instance.ts
+++ b/cli/instance.ts
@@ -3,7 +3,7 @@ import {
  path,
  Confirm,
  yamlStringify,
-  yamlParse,
+  yamlParseFile,
  Command,
  setClient,
  Table,
@@ -459,9 +459,9 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
      }

      try {
-        const workspaceSettings = yamlParse(
-          await Deno.readTextFile("settings.yaml")
-        ) as SimplifiedSettings;
+        const workspaceSettings = (await yamlParseFile(
+          "settings.yaml"
+        )) as SimplifiedSettings;
        await workspaceSetup(
          {
            token: instance.token,
--- a/cli/metadata.ts
+++ b/cli/metadata.ts
@@ -1,6 +1,13 @@
 // deno-lint-ignore-file no-explicit-any
 import { GlobalOptions } from "./types.ts";
-import { SEP, colors, log, path, yamlParse, yamlStringify } from "./deps.ts";
+import {
+  SEP,
+  colors,
+  log,
+  path,
+  yamlParseFile,
+  yamlStringify,
+} from "./deps.ts";
 import {
  ScriptMetadata,
  defaultScriptMetadata,
@@ -119,9 +126,9 @@ export async function generateFlowLockInternal(
    return remote_path;
  }

-  const flowValue = yamlParse(
-    await Deno.readTextFile(folder! + SEP + "flow.yaml")
-  ) as FlowFile;
+  const flowValue = (await yamlParseFile(
+    folder! + SEP + "flow.yaml"
+  )) as FlowFile;

  if (!justUpdateMetadataLock) {
    const changedScripts = [];
@@ -798,7 +805,7 @@ export async function parseMetadataFile(
    try {
      metadataFilePath = scriptPath + ".script.yaml";
      await Deno.stat(metadataFilePath);
-      const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath));
+      const payload: any = await yamlParseFile(metadataFilePath);
      replaceLock(payload);

      return {
@@ -840,9 +847,9 @@ export async function parseMetadataFile(
            codebases,
            false
          );
-          scriptInitialMetadata = yamlParse(
-            await Deno.readTextFile(metadataFilePath)
-          ) as ScriptMetadata;
+          scriptInitialMetadata = (await yamlParseFile(
+            metadataFilePath
+          )) as ScriptMetadata;
          replaceLock(scriptInitialMetadata);
        } catch (e) {
          log.info(
@@ -868,8 +875,7 @@ interface Lock {
 const WMILL_LOCKFILE = "wmill-lock.yaml";
 export async function readLockfile(): Promise<Lock> {
  try {
-    const lockfile = await Deno.readTextFile(WMILL_LOCKFILE);
-    const read = yamlParse(lockfile);
+    const read = await yamlParseFile(WMILL_LOCKFILE);
    if (typeof read == "object") {
      return read as Lock;
    } else {
--- a/cli/settings.ts
+++ b/cli/settings.ts
@@ -1,7 +1,7 @@
 import { yamlStringify } from "./deps.ts";
 import { Confirm } from "./deps.ts";
 import { colors } from "./deps.ts";
-import { yamlParse } from "./deps.ts";
+import { yamlParseFile } from "./deps.ts";
 import { log } from "./deps.ts";
 import { compareInstanceObjects } from "./instance.ts";
 import { isSuperset } from "./types.ts";
@@ -259,17 +259,24 @@ export async function pushWorkspaceKey(
  }
 }

+export async function readInstanceSettings() {
+  let localSettings: GlobalSetting[] = [];
+
+  try {
+    localSettings = (await yamlParseFile(
+      "instance_settings.yaml"
+    )) as GlobalSetting[];
+  } catch {
+    log.warn("No instance_settings.yaml found");
+  }
+  return localSettings;
+}
+
 export async function pullInstanceSettings(preview = false) {
  const remoteSettings = await wmill.listGlobalSettings();

  if (preview) {
-    let localSettings: GlobalSetting[] = [];
-
-    try {
-      localSettings = yamlParse(
-        await Deno.readTextFile("instance_settings.yaml")
-      ) as GlobalSetting[];
-    } catch {}
+    const localSettings: GlobalSetting[] = await readInstanceSettings();

    return compareInstanceObjects(
      remoteSettings,
@@ -294,9 +301,7 @@ export async function pushInstanceSettings(
  baseUrl?: string
 ) {
  const remoteSettings = await wmill.listGlobalSettings();
-  let localSettings = (await Deno.readTextFile("instance_settings.yaml")
-    .then((raw) => yamlParse(raw))
-    .catch(() => [])) as GlobalSetting[];
+  let localSettings: GlobalSetting[] = await readInstanceSettings();

  if (baseUrl) {
    localSettings = localSettings.filter((s) => s.name !== "base_url");
@@ -354,6 +359,17 @@ export async function pushInstanceSettings(
  }
 }

+export async function readLocalConfigs() {
+  let localConfigs: Config[] = [];
+
+  try {
+    localConfigs = (await yamlParseFile("instance_configs.yaml")) as Config[];
+  } catch {
+    log.warn("No instance_configs.yaml found");
+  }
+  return localConfigs;
+}
+
 export async function pullInstanceConfigs(preview = false) {
  const remoteConfigs = (await wmill.listConfigs()).map((x) => {
    return {
@@ -363,12 +379,7 @@ export async function pullInstanceConfigs(preview = false) {
  });

  if (preview) {
-    let localConfigs: Config[] = [];
-    try {
-      localConfigs = yamlParse(
-        await Deno.readTextFile("instance_configs.yaml")
-      ) as Config[];
-    } catch {}
+    const localConfigs: Config[] = await readLocalConfigs();

    return compareInstanceObjects(
      remoteConfigs,
@@ -395,9 +406,7 @@ export async function pushInstanceConfigs(preview: boolean = false) {
      name: removeWorkerPrefix(x.name),
    };
  });
-  const localConfigs = (await Deno.readTextFile("instance_configs.yaml")
-    .then((raw) => yamlParse(raw))
-    .catch(() => [])) as Config[];
+  const localConfigs = await readLocalConfigs();

  if (preview) {
    return compareInstanceObjects(
@@ -415,7 +424,9 @@ export async function pushInstanceConfigs(preview: boolean = false) {
      }
      try {
        await wmill.updateConfig({
-          name: config.name.startsWith('worker__') ? config.name : `worker__${config.name}`,
+          name: config.name.startsWith("worker__")
+            ? config.name
+            : `worker__${config.name}`,
          requestBody: config.config,
        });
      } catch (err) {
--- a/cli/sync.ts
+++ b/cli/sync.ts
@@ -9,7 +9,7 @@ import {
  path,
  log,
  yamlStringify,
-  yamlParse,
+  yamlParseContent,
  SEP,
 } from "./deps.ts";
 import * as wmill from "./gen/services.gen.ts";
@@ -112,7 +112,7 @@ async function addCodebaseDigestIfRelevant(
  if (isTs) {
    const c = findCodebase(path, codebases);
    if (c) {
-      const parsed: any = yamlParse(content);
+      const parsed: any = yamlParseContent(path, content);
      if (parsed && typeof parsed == "object") {
        parsed["codebase"] = c.digest;
        parsed["lock"] = undefined;
@@ -660,7 +660,7 @@ export async function elementsToMap(
        if (json) {
          o = JSON.parse(content);
        } else {
-          o = yamlParse(content);
+          o = yamlParseContent(path, content);
        }
        if (o["is_secret"]) {
          continue;
@@ -704,7 +704,7 @@ async function compareDynFSElement(

  function parseYaml(k: string, v: string) {
    if (k.endsWith(".script.yaml")) {
-      const o: any = yamlParse(v);
+      const o: any = yamlParseContent(k, v);
      if (typeof o == "object") {
        if (Array.isArray(o?.["lock"])) {
          o["lock"] = o["lock"].join("\n");
@@ -715,7 +715,7 @@ async function compareDynFSElement(
      }
      return o;
    } else if (k.endsWith(".app.yaml")) {
-      const o: any = yamlParse(v);
+      const o: any = yamlParseContent(k, v);
      const o2 = o["policy"];

      if (typeof o2 == "object") {
@@ -728,7 +728,7 @@ async function compareDynFSElement(
      }
      return o;
    } else {
-      return yamlParse(v);
+      return yamlParseContent(k, v);
    }
  }
  for (const [k, v] of Object.entries(m1)) {
--- a/cli/types.ts
+++ b/cli/types.ts
@@ -6,7 +6,7 @@ import {
  colors,
  log,
  path,
-  yamlParse,
+  yamlParseContent,
  yamlStringify,
 } from "./deps.ts";
 import { pushApp } from "./apps.ts";
@@ -115,7 +115,7 @@ export async function pushObj(
  newObj: any,
  plainSecrets: boolean,
  alreadySynced: string[],
-  message?: string,
+  message?: string
 ) {
  const typeEnding = getTypeStrFromPath(p);

@@ -155,7 +155,7 @@ export async function pushObj(

 export function parseFromPath(p: string, content: string): any {
  return p.endsWith(".yaml")
-    ? yamlParse(content)
+    ? yamlParseContent(p, content)
    : p.endsWith(".json")
    ? JSON.parse(content)
    : content;
@@ -164,7 +164,7 @@ export function parseFromFile(p: string): any {
  if (p.endsWith(".json")) {
    return JSON.parse(Deno.readTextFileSync(p));
  } else if (p.endsWith(".yaml") || p.endsWith(".yml")) {
-    return yamlParse(Deno.readTextFileSync(p));
+    return yamlParseContent(p, Deno.readTextFileSync(p));
  } else {
    throw new Error("Could not read file " + p);
  }
@@ -227,7 +227,7 @@ export function getTypeStrFromPath(
    return typeEnding;
  } else {
    if (isFileResource(p)) {
-      return "resource"
+      return "resource";
    }
    throw new Error("Could not infer type of path " + JSON.stringify(parsed));
  }
--- a/cli/user.ts
+++ b/cli/user.ts
@@ -13,7 +13,7 @@ import {
  log,
  Table,
  yamlStringify,
-  yamlParse,
+  yamlParseFile,
 } from "./deps.ts";
 import * as wmill from "./gen/services.gen.ts";
 import {
@@ -170,6 +170,7 @@ export async function pushWorkspaceUser(
        },
      });
    } catch (e) {
+      //@ts-ignore
      console.error(e.body);
      throw e;
    }
@@ -189,6 +190,7 @@ export async function pushWorkspaceUser(
        },
      });
    } catch (e) {
+      //@ts-ignore
      console.error(e.body);
      throw e;
    }
@@ -251,6 +253,7 @@ export async function pushGroup(
        },
      });
    } catch (e) {
+      //@ts-ignore
      console.error(e.body);
      throw e;
    }
@@ -301,6 +304,7 @@ export async function pushGroup(
          });
        }
      } catch (e) {
+        //@ts-ignore
        console.error(e.body);
        throw e;
      }
@@ -329,6 +333,7 @@ export async function pushGroup(
            },
          });
        } catch (e) {
+          //@ts-ignore
          console.error(e.body);
          throw e;
        }
@@ -368,11 +373,13 @@ export async function pushGroup(
            });
          }
        } catch (e) {
+          //@ts-ignore
          console.error(e.body);
          throw e;
        }
      }
    } catch (e) {
+      //@ts-ignore
      console.error(e.body);
      throw e;
    }
@@ -383,11 +390,7 @@ export async function pullInstanceUsers(preview: boolean = false) {
  const remoteUsers = await wmill.globalUsersExport();

  if (preview) {
-    let localUsers: ExportedUser[] = [];
-    try {
-      const raw = await Deno.readTextFile("instance_users.yaml");
-      localUsers = yamlParse(raw) as ExportedUser[];
-    } catch {}
+    const localUsers: ExportedUser[] = await readInstanceUsers();
    return compareInstanceObjects(remoteUsers, localUsers, "email", "user");
  } else {
    log.info("Pulling users from instance...");
@@ -399,11 +402,31 @@ export async function pullInstanceUsers(preview: boolean = false) {
  }
 }

+export async function readInstanceUsers() {
+  let localUsers: ExportedUser[] = [];
+  try {
+    localUsers = (await yamlParseFile("instance_users.yaml")) as ExportedUser[];
+  } catch {
+    log.warn("No instance_users.yaml file found");
+  }
+  return localUsers;
+}
+
+export async function readInstanceGroups() {
+  let localGroups: InstanceGroup[] = [];
+  try {
+    localGroups = (await yamlParseFile(
+      "instance_groups.yaml"
+    )) as ExportedInstanceGroup[];
+  } catch {
+    log.warn("No instance_groups.yaml file found");
+  }
+  return localGroups;
+}
+
 export async function pushInstanceUsers(preview: boolean = false) {
  const remoteUsers = await wmill.globalUsersExport();
-  const localUsers = (await Deno.readTextFile("instance_users.yaml")
-    .then((raw) => yamlParse(raw))
-    .catch(() => [])) as ExportedUser[];
+  const localUsers: ExportedUser[] = await readInstanceUsers();

  if (preview) {
    return compareInstanceObjects(localUsers, remoteUsers, "email", "user");
@@ -421,11 +444,7 @@ export async function pullInstanceGroups(preview = false) {
  const remoteGroups = await wmill.exportInstanceGroups();

  if (preview) {
-    let localGroups: InstanceGroup[] = [];
-    try {
-      const raw = await Deno.readTextFile("instance_groups.yaml");
-      localGroups = yamlParse(raw) as InstanceGroup[];
-    } catch {}
+    const localGroups = await readInstanceGroups();
    return compareInstanceObjects(remoteGroups, localGroups, "name", "group");
  } else {
    log.info("Pulling groups from instance...");
@@ -441,9 +460,7 @@ export async function pullInstanceGroups(preview = false) {

 export async function pushInstanceGroups(preview: boolean = false) {
  const remoteGroups = await wmill.exportInstanceGroups();
-  const localGroups = (await Deno.readTextFile("instance_groups.yaml")
-    .then((raw) => yamlParse(raw))
-    .catch(() => [])) as ExportedInstanceGroup[];
+  const localGroups = await readInstanceGroups();

  if (preview) {
    return compareInstanceObjects(localGroups, remoteGroups, "name", "group");