From e06c845ed494da3c3cb67656fc6b2d1dcad796e4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 12 Oct 2024 03:01:46 +0200 Subject: [PATCH] cli yaml parsing error more verbose --- cli/apps.ts | 6 ++--- cli/conf.ts | 6 ++--- cli/deps.ts | 24 ++++++++++++++++++- cli/flow.ts | 5 ++-- cli/gen/core/OpenAPI.ts | 2 +- cli/gen/types.gen.ts | 4 ++++ cli/instance.ts | 8 +++---- cli/metadata.ts | 26 ++++++++++++-------- cli/settings.ts | 53 +++++++++++++++++++++++++---------------- cli/sync.ts | 12 +++++----- cli/types.ts | 10 ++++---- cli/user.ts | 51 ++++++++++++++++++++++++++------------- 12 files changed, 132 insertions(+), 75 deletions(-) diff --git a/cli/apps.ts b/cli/apps.ts index de513b28c6..ca966e2112 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file no-explicit-any import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { colors, Command, log, SEP, Table, yamlParse } from "./deps.ts"; +import { colors, Command, log, SEP, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { ListableApp, Policy } from "./gen/types.gen.ts"; @@ -40,8 +40,8 @@ export async function pushApp( if (!localPath.endsWith(SEP)) { localPath += SEP; } - const localAppRaw = await Deno.readTextFile(localPath + "app.yaml"); - const localApp = yamlParse(localAppRaw) as AppFile; + const path = localPath + "app.yaml"; + const localApp = (await yamlParseFile(path)) as AppFile; function replaceInlineScripts(rec: any) { if (!rec) { diff --git a/cli/conf.ts b/cli/conf.ts index fa18de880b..eab154fec9 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -1,4 +1,4 @@ -import { log, yamlParse } from "./deps.ts"; +import { log, yamlParseFile } from "./deps.ts"; export interface SyncOptions { stateful?: boolean; @@ -40,9 +40,7 @@ export interface Codebase { export async function readConfigFile(): Promise { try { - const conf = yamlParse( - await Deno.readTextFile("wmill.yaml") - ) as SyncOptions; + const conf = (await yamlParseFile("wmill.yaml")) as SyncOptions; if (conf?.defaultTs == undefined) { log.warn( "No defaultTs defined in your wmill.yaml. Using 'bun' as default." diff --git a/cli/deps.ts b/cli/deps.ts index a8e14f0d3d..53e96a4f9c 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -21,7 +21,29 @@ export { copy } from "jsr:@std/io/copy"; export { readAll } from "jsr:@std/io/read-all"; export * as log from "jsr:@std/log"; -export { stringify as yamlStringify, parse as yamlParse } from "jsr:@std/yaml"; +export { stringify as yamlStringify } from "jsr:@std/yaml"; + +import { parse as yamlParse, ParseOptions } from "jsr:@std/yaml"; + +export async function yamlParseFile(path: string, options: ParseOptions = {}) { + try { + return yamlParse(await Deno.readTextFile(path), options); + } catch (e) { + throw new Error(`Error parsing yaml ${path}`, { cause: e }); + } +} + +export function yamlParseContent( + path: string, + content: string, + options: ParseOptions = {} +) { + try { + return yamlParse(content, options); + } catch (e) { + throw new Error(`Error parsing yaml ${path}`, { cause: e }); + } +} // other diff --git a/cli/flow.ts b/cli/flow.ts index 938ad1659d..edca024f66 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -1,7 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions, isSuperset } from "./types.ts"; import { Confirm, SEP, log, yamlStringify } from "./deps.ts"; -import { colors, Command, Table, yamlParse } from "./deps.ts"; +import { colors, Command, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; @@ -89,8 +89,7 @@ export async function pushFlow( if (!localPath.endsWith(SEP)) { localPath += SEP; } - const localFlowRaw = await Deno.readTextFile(localPath + "flow.yaml"); - const localFlow = yamlParse(localFlowRaw) as FlowFile; + const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile; replaceInlineScripts(localFlow.value.modules, localPath, undefined); diff --git a/cli/gen/core/OpenAPI.ts b/cli/gen/core/OpenAPI.ts index 6af5246be2..cddc3f88de 100644 --- a/cli/gen/core/OpenAPI.ts +++ b/cli/gen/core/OpenAPI.ts @@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: getEnv("WM_TOKEN"), USERNAME: undefined, - VERSION: '1.401.0', + VERSION: '1.407.2', WITH_CREDENTIALS: true, interceptors: { request: new Interceptors(), diff --git a/cli/gen/types.gen.ts b/cli/gen/types.gen.ts index 92bfe933d9..fe297f8bde 100644 --- a/cli/gen/types.gen.ts +++ b/cli/gen/types.gen.ts @@ -1015,6 +1015,9 @@ export type FlowModule = { skip_if_stopped?: boolean; expr: string; }; + skip_if?: { + expr: string; + }; sleep?: InputTransform; cache_ttl?: number; timeout?: number; @@ -1176,6 +1179,7 @@ export type FlowStatusModule = { approver: string; }>; failed_retries?: Array<(string)>; + skipped?: boolean; }; export type type4 = 'WaitingForPriorSteps' | 'WaitingForEvents' | 'WaitingForExecutor' | 'InProgress' | 'Success' | 'Failure'; diff --git a/cli/instance.ts b/cli/instance.ts index bedecdac63..bc3f0968b7 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -3,7 +3,7 @@ import { path, Confirm, yamlStringify, - yamlParse, + yamlParseFile, Command, setClient, Table, @@ -459,9 +459,9 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { } try { - const workspaceSettings = yamlParse( - await Deno.readTextFile("settings.yaml") - ) as SimplifiedSettings; + const workspaceSettings = (await yamlParseFile( + "settings.yaml" + )) as SimplifiedSettings; await workspaceSetup( { token: instance.token, diff --git a/cli/metadata.ts b/cli/metadata.ts index a3b4738cb7..92ac7dfc4b 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -1,6 +1,13 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions } from "./types.ts"; -import { SEP, colors, log, path, yamlParse, yamlStringify } from "./deps.ts"; +import { + SEP, + colors, + log, + path, + yamlParseFile, + yamlStringify, +} from "./deps.ts"; import { ScriptMetadata, defaultScriptMetadata, @@ -119,9 +126,9 @@ export async function generateFlowLockInternal( return remote_path; } - const flowValue = yamlParse( - await Deno.readTextFile(folder! + SEP + "flow.yaml") - ) as FlowFile; + const flowValue = (await yamlParseFile( + folder! + SEP + "flow.yaml" + )) as FlowFile; if (!justUpdateMetadataLock) { const changedScripts = []; @@ -798,7 +805,7 @@ export async function parseMetadataFile( try { metadataFilePath = scriptPath + ".script.yaml"; await Deno.stat(metadataFilePath); - const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath)); + const payload: any = await yamlParseFile(metadataFilePath); replaceLock(payload); return { @@ -840,9 +847,9 @@ export async function parseMetadataFile( codebases, false ); - scriptInitialMetadata = yamlParse( - await Deno.readTextFile(metadataFilePath) - ) as ScriptMetadata; + scriptInitialMetadata = (await yamlParseFile( + metadataFilePath + )) as ScriptMetadata; replaceLock(scriptInitialMetadata); } catch (e) { log.info( @@ -868,8 +875,7 @@ interface Lock { const WMILL_LOCKFILE = "wmill-lock.yaml"; export async function readLockfile(): Promise { try { - const lockfile = await Deno.readTextFile(WMILL_LOCKFILE); - const read = yamlParse(lockfile); + const read = await yamlParseFile(WMILL_LOCKFILE); if (typeof read == "object") { return read as Lock; } else { diff --git a/cli/settings.ts b/cli/settings.ts index b3a1b6a272..328a6d0e25 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -1,7 +1,7 @@ import { yamlStringify } from "./deps.ts"; import { Confirm } from "./deps.ts"; import { colors } from "./deps.ts"; -import { yamlParse } from "./deps.ts"; +import { yamlParseFile } from "./deps.ts"; import { log } from "./deps.ts"; import { compareInstanceObjects } from "./instance.ts"; import { isSuperset } from "./types.ts"; @@ -259,17 +259,24 @@ export async function pushWorkspaceKey( } } +export async function readInstanceSettings() { + let localSettings: GlobalSetting[] = []; + + try { + localSettings = (await yamlParseFile( + "instance_settings.yaml" + )) as GlobalSetting[]; + } catch { + log.warn("No instance_settings.yaml found"); + } + return localSettings; +} + export async function pullInstanceSettings(preview = false) { const remoteSettings = await wmill.listGlobalSettings(); if (preview) { - let localSettings: GlobalSetting[] = []; - - try { - localSettings = yamlParse( - await Deno.readTextFile("instance_settings.yaml") - ) as GlobalSetting[]; - } catch {} + const localSettings: GlobalSetting[] = await readInstanceSettings(); return compareInstanceObjects( remoteSettings, @@ -294,9 +301,7 @@ export async function pushInstanceSettings( baseUrl?: string ) { const remoteSettings = await wmill.listGlobalSettings(); - let localSettings = (await Deno.readTextFile("instance_settings.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as GlobalSetting[]; + let localSettings: GlobalSetting[] = await readInstanceSettings(); if (baseUrl) { localSettings = localSettings.filter((s) => s.name !== "base_url"); @@ -354,6 +359,17 @@ export async function pushInstanceSettings( } } +export async function readLocalConfigs() { + let localConfigs: Config[] = []; + + try { + localConfigs = (await yamlParseFile("instance_configs.yaml")) as Config[]; + } catch { + log.warn("No instance_configs.yaml found"); + } + return localConfigs; +} + export async function pullInstanceConfigs(preview = false) { const remoteConfigs = (await wmill.listConfigs()).map((x) => { return { @@ -363,12 +379,7 @@ export async function pullInstanceConfigs(preview = false) { }); if (preview) { - let localConfigs: Config[] = []; - try { - localConfigs = yamlParse( - await Deno.readTextFile("instance_configs.yaml") - ) as Config[]; - } catch {} + const localConfigs: Config[] = await readLocalConfigs(); return compareInstanceObjects( remoteConfigs, @@ -395,9 +406,7 @@ export async function pushInstanceConfigs(preview: boolean = false) { name: removeWorkerPrefix(x.name), }; }); - const localConfigs = (await Deno.readTextFile("instance_configs.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as Config[]; + const localConfigs = await readLocalConfigs(); if (preview) { return compareInstanceObjects( @@ -415,7 +424,9 @@ export async function pushInstanceConfigs(preview: boolean = false) { } try { await wmill.updateConfig({ - name: config.name.startsWith('worker__') ? config.name : `worker__${config.name}`, + name: config.name.startsWith("worker__") + ? config.name + : `worker__${config.name}`, requestBody: config.config, }); } catch (err) { diff --git a/cli/sync.ts b/cli/sync.ts index 3a12272b6e..ce539b7a1f 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -9,7 +9,7 @@ import { path, log, yamlStringify, - yamlParse, + yamlParseContent, SEP, } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; @@ -112,7 +112,7 @@ async function addCodebaseDigestIfRelevant( if (isTs) { const c = findCodebase(path, codebases); if (c) { - const parsed: any = yamlParse(content); + const parsed: any = yamlParseContent(path, content); if (parsed && typeof parsed == "object") { parsed["codebase"] = c.digest; parsed["lock"] = undefined; @@ -660,7 +660,7 @@ export async function elementsToMap( if (json) { o = JSON.parse(content); } else { - o = yamlParse(content); + o = yamlParseContent(path, content); } if (o["is_secret"]) { continue; @@ -704,7 +704,7 @@ async function compareDynFSElement( function parseYaml(k: string, v: string) { if (k.endsWith(".script.yaml")) { - const o: any = yamlParse(v); + const o: any = yamlParseContent(k, v); if (typeof o == "object") { if (Array.isArray(o?.["lock"])) { o["lock"] = o["lock"].join("\n"); @@ -715,7 +715,7 @@ async function compareDynFSElement( } return o; } else if (k.endsWith(".app.yaml")) { - const o: any = yamlParse(v); + const o: any = yamlParseContent(k, v); const o2 = o["policy"]; if (typeof o2 == "object") { @@ -728,7 +728,7 @@ async function compareDynFSElement( } return o; } else { - return yamlParse(v); + return yamlParseContent(k, v); } } for (const [k, v] of Object.entries(m1)) { diff --git a/cli/types.ts b/cli/types.ts index 84691a37e2..e264521cb7 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -6,7 +6,7 @@ import { colors, log, path, - yamlParse, + yamlParseContent, yamlStringify, } from "./deps.ts"; import { pushApp } from "./apps.ts"; @@ -115,7 +115,7 @@ export async function pushObj( newObj: any, plainSecrets: boolean, alreadySynced: string[], - message?: string, + message?: string ) { const typeEnding = getTypeStrFromPath(p); @@ -155,7 +155,7 @@ export async function pushObj( export function parseFromPath(p: string, content: string): any { return p.endsWith(".yaml") - ? yamlParse(content) + ? yamlParseContent(p, content) : p.endsWith(".json") ? JSON.parse(content) : content; @@ -164,7 +164,7 @@ export function parseFromFile(p: string): any { if (p.endsWith(".json")) { return JSON.parse(Deno.readTextFileSync(p)); } else if (p.endsWith(".yaml") || p.endsWith(".yml")) { - return yamlParse(Deno.readTextFileSync(p)); + return yamlParseContent(p, Deno.readTextFileSync(p)); } else { throw new Error("Could not read file " + p); } @@ -227,7 +227,7 @@ export function getTypeStrFromPath( return typeEnding; } else { if (isFileResource(p)) { - return "resource" + return "resource"; } throw new Error("Could not infer type of path " + JSON.stringify(parsed)); } diff --git a/cli/user.ts b/cli/user.ts index f95747306a..2ddeb997c9 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -13,7 +13,7 @@ import { log, Table, yamlStringify, - yamlParse, + yamlParseFile, } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { @@ -170,6 +170,7 @@ export async function pushWorkspaceUser( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -189,6 +190,7 @@ export async function pushWorkspaceUser( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -251,6 +253,7 @@ export async function pushGroup( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -301,6 +304,7 @@ export async function pushGroup( }); } } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -329,6 +333,7 @@ export async function pushGroup( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -368,11 +373,13 @@ export async function pushGroup( }); } } catch (e) { + //@ts-ignore console.error(e.body); throw e; } } } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -383,11 +390,7 @@ export async function pullInstanceUsers(preview: boolean = false) { const remoteUsers = await wmill.globalUsersExport(); if (preview) { - let localUsers: ExportedUser[] = []; - try { - const raw = await Deno.readTextFile("instance_users.yaml"); - localUsers = yamlParse(raw) as ExportedUser[]; - } catch {} + const localUsers: ExportedUser[] = await readInstanceUsers(); return compareInstanceObjects(remoteUsers, localUsers, "email", "user"); } else { log.info("Pulling users from instance..."); @@ -399,11 +402,31 @@ export async function pullInstanceUsers(preview: boolean = false) { } } +export async function readInstanceUsers() { + let localUsers: ExportedUser[] = []; + try { + localUsers = (await yamlParseFile("instance_users.yaml")) as ExportedUser[]; + } catch { + log.warn("No instance_users.yaml file found"); + } + return localUsers; +} + +export async function readInstanceGroups() { + let localGroups: InstanceGroup[] = []; + try { + localGroups = (await yamlParseFile( + "instance_groups.yaml" + )) as ExportedInstanceGroup[]; + } catch { + log.warn("No instance_groups.yaml file found"); + } + return localGroups; +} + export async function pushInstanceUsers(preview: boolean = false) { const remoteUsers = await wmill.globalUsersExport(); - const localUsers = (await Deno.readTextFile("instance_users.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as ExportedUser[]; + const localUsers: ExportedUser[] = await readInstanceUsers(); if (preview) { return compareInstanceObjects(localUsers, remoteUsers, "email", "user"); @@ -421,11 +444,7 @@ export async function pullInstanceGroups(preview = false) { const remoteGroups = await wmill.exportInstanceGroups(); if (preview) { - let localGroups: InstanceGroup[] = []; - try { - const raw = await Deno.readTextFile("instance_groups.yaml"); - localGroups = yamlParse(raw) as InstanceGroup[]; - } catch {} + const localGroups = await readInstanceGroups(); return compareInstanceObjects(remoteGroups, localGroups, "name", "group"); } else { log.info("Pulling groups from instance..."); @@ -441,9 +460,7 @@ export async function pullInstanceGroups(preview = false) { export async function pushInstanceGroups(preview: boolean = false) { const remoteGroups = await wmill.exportInstanceGroups(); - const localGroups = (await Deno.readTextFile("instance_groups.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as ExportedInstanceGroup[]; + const localGroups = await readInstanceGroups(); if (preview) { return compareInstanceObjects(localGroups, remoteGroups, "name", "group");