import { requireLogin, resolveWorkspace } from "./context.ts"; import { getWorkspaceStream, Workspace } from "./workspace.ts"; import { decoverto, map, MapShape, model, property } from "./decoverto.ts"; import { cbor, colors, Command, Confirm, ensureDir, gitignore_parser, JSZip, microdiff, nanoid, objectHash, path, ScriptService, } from "./deps.ts"; import { Difference, getTypeStrFromPath, GlobalOptions, inferTypeFromPath, setValueByPath, } from "./types.ts"; import { downloadZip } from "./pull.ts"; import { FolderFile } from "./folder.ts"; import { ResourceTypeFile } from "./resource-type.ts"; import { findContentFile, inferContentTypeFromFilePath, pushScript, ScriptFile, } from "./script.ts"; import { ResourceFile } from "./resource.ts"; import { FlowFile } from "./flow.ts"; import { VariableFile } from "./variable.ts"; type TrackedId = string; const TrackedId = String; const CONTENT_ENCODER: cbor.Encoder = new cbor.Encoder({ pack: true }); export class Tracked { #parent: State; path: string; #content_cache?: string; constructor(parent: State, path: string) { this.#parent = parent; this.path = path; } async getContent(): Promise { if (this.#content_cache) { return this.#content_cache; } const f = this.#parent.stateContentFor(this.path); if (!f) { return undefined; } const data = await Deno.readFile( State.getInternalWmillFolder(f), ); const content = CONTENT_ENCODER.decode(data); this.#content_cache = content; return content; } getHash(): string | undefined { return this.#parent.hashes.get(objectHash(this.path)); } } type DynFSElement = { isDirectory: boolean; path: string; getContentBytes(): Promise; getContentText(): Promise; getChildren(): AsyncIterable; }; async function FSFSElement(p: string): Promise { function _internal_element( p: string, isDir: boolean, ): DynFSElement { return { isDirectory: isDir, path: p, async *getChildren(): AsyncIterable { for await (const e of Deno.readDir(p)) { yield _internal_element(path.join(p, e.name), e.isDirectory); } }, async getContentBytes(): Promise { return await Deno.readFile(p); }, async getContentText(): Promise { return await Deno.readTextFile(p); }, }; } return _internal_element(p, (await Deno.stat(p)).isDirectory); } function ZipFSElement(zip: JSZip): DynFSElement { function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement { return { isDirectory: false, path: p, // deno-lint-ignore require-yield async *getChildren(): AsyncIterable { throw new Error("Cannot get children of file"); }, async getContentBytes(): Promise { return await f.async("uint8array"); }, async getContentText(): Promise { return await f.async("text"); }, }; } function _internal_folder(p: string, zip: JSZip): DynFSElement { return { isDirectory: true, path: p, async *getChildren(): AsyncIterable { for (const filename in zip.files) { const file = zip.files[filename]; const totalPath = path.join(p, filename); if (file.dir) { const e = zip.folder(file.name)!; yield _internal_folder(totalPath, e); } else { yield _internal_file(totalPath, file); } } }, async getContentBytes(): Promise { throw new Error("Cannot get content of folder"); }, async getContentText(): Promise { throw new Error("Cannot get content of folder"); }, }; } return _internal_folder("./", zip); } async function* readDirRecursiveWithIgnore( ignore: (path: string) => boolean, root: DynFSElement, ): AsyncGenerator< { path: string; ignored: boolean; isDirectory: boolean; getContentBytes(): Promise; getContentText(): Promise; } > { const stack: { path: string; isDirectory: boolean; ignored: boolean; c(): AsyncIterable; getContentBytes(): Promise; getContentText(): Promise; }[] = [{ path: root.path, ignored: ignore(root.path), isDirectory: root.isDirectory, c: root.getChildren, getContentBytes(): Promise { throw undefined; }, getContentText(): Promise { throw undefined; }, }]; while (stack.length > 0) { const e = stack.pop()!; yield e; if (!e.isDirectory) continue; for await (const e2 of e.c()) { stack.push({ path: e2.path, ignored: e.ignored || ignore(e2.path), isDirectory: e2.isDirectory, getContentBytes: e2.getContentBytes, getContentText: e2.getContentText, c: e2.getChildren, }); } } } @model() export class State { // TODO: Handle script contents, as they are separate files & need to be tracked separately. // I think the way this makes most sense is actually reating them as a separate kind of file everywhere // And then move to using a diff-based push system everywhere. This should greatly simplify code & enable this easily @property(map(() => TrackedId, () => String, { shape: MapShape.Object })) hashes: Map; @property(map(() => TrackedId, () => String, { shape: MapShape.Object })) contentFiles: Map; @property(() => String) workspaceId: string; @property(() => String) remoteUrl: string; constructor( hashes: Map, contentFiles: Map, workspaceId: string, remoteUrl: string, ) { this.hashes = hashes; this.contentFiles = contentFiles; this.workspaceId = workspaceId; this.remoteUrl = remoteUrl; } public stateContentFor(path: string) { const hashOfPath = objectHash(path); const file = this.contentFiles.get(hashOfPath); return file; } public get(path: string): Tracked { // TODO: Normalize path return new Tracked(this, path); } public static getInternalWmillFolder(...subpath: string[]): string { return path.join(Deno.cwd(), ".wmill", ...subpath); } public async *getFiles(): AsyncGenerator<{ localFile: string | undefined; stateFile: Tracked; isIgnored: boolean; path: string; }> { // not sure why the auto-typing doesn't work here, see (a @types package is available...) const ignore: { accepts(file: string): boolean; denies(file: string): boolean; } = gitignore_parser.compile( await Deno.readTextFile(".wmillignore"), ); const base = Deno.cwd(); for await ( const { ignored, path, getContentText, isDirectory } of readDirRecursiveWithIgnore( ignore.denies, await FSFSElement(base), ) ) { if (isDirectory || path.includes(".wmill")) { continue; } const path2 = path.substring(base.length + 1); const stateFile = this.get(path2); let localFile: string | undefined; try { localFile = await getContentText(); } catch { localFile = undefined; } yield { path: path2, isIgnored: ignored, stateFile, localFile }; } } public async save(): Promise { const encoder = new cbor.Encoder({}); const plain = decoverto.type(State).instanceToPlain(this); const result: Uint8Array = encoder.encode(plain, {}); await Deno.writeFile(State.getInternalWmillFolder("main"), result, { create: true, }); } public static async loadState(): Promise { const source = await Deno.readFile(this.getInternalWmillFolder("main")); const encoder = new cbor.Encoder({}); const raw = encoder.decode(source); const state = decoverto.type(State).plainToInstance( raw, ); return Object.freeze(state); } } async function getState(opts: GlobalOptions) { const existingState = await State.loadState(); const workspaceStream = await getWorkspaceStream(); const reader = workspaceStream.getReader(); while (true) { const res = await reader.read(); if (res.value) { if ( new URL(res.value.remote) == new URL(existingState.remoteUrl) && res.value.workspaceId == existingState.workspaceId ) { opts.workspace = res.value.name; (opts as any).__secret_workspace = res.value; break; } } if (res.done) { break; } } return existingState; } async function updateStateFromRemote( workspace: Workspace, state: State, callback: (filename: string) => PromiseLike | boolean, ) { const zipDir = await downloadZip(workspace); if (!zipDir) throw new Error("Failed to pull Zip"); for await ( const entry of readDirRecursiveWithIgnore( (_) => false, ZipFSElement(zipDir), ) ) { if (entry.isDirectory || entry.ignored) continue; const e = state.get(entry.path); if (e) { const val = await entry.getContentText(); if (entry.path.endsWith(".json")) { const parsed = JSON.parse(val); const typed = inferTypeFromPath(entry.path, parsed); const oldHash = e.getHash(); const newHash = objectHash(typed); if (!oldHash || oldHash !== newHash) { if (!await callback(entry.path)) { return; // notice that we are not saving } try { await Deno.stat(entry.path); } catch { await ensureDir(path.dirname(e.path)); await Deno.writeTextFile(e.path, "{}"); } state.hashes.set(objectHash(e.path), newHash); const encoded = CONTENT_ENCODER.encode(typed); const fileName = nanoid(); await Deno.writeFile( State.getInternalWmillFolder(fileName), encoded, { create: true }, ); state.contentFiles.set(objectHash(e.path), fileName); } } else { const fileName = nanoid(); await Deno.writeTextFile( State.getInternalWmillFolder(fileName), val, { create: true }, ); state.contentFiles.set(objectHash(e.path), fileName); try { await Deno.stat(entry.path); } catch { await ensureDir(path.dirname(e.path)); await Deno.writeTextFile(e.path, "tmp_file"); } } } } await state.save(); } async function pull( opts: GlobalOptions & { raw: boolean; rawOverride: boolean }, ) { if (opts.raw) { const opts2 = opts as any; opts2.override = opts2.rawOverride; opts2.raw = undefined; opts2.rawOverride = undefined; await pullRaw(opts2); return; } try { await Deno.stat(State.getInternalWmillFolder("main")) } catch { console.log( colors.yellow( "No state file found, creating empty state folder at .wmill", ), ); init() } const state = await getState(opts); console.log(state); const workspace = await resolveWorkspace(opts); await requireLogin(opts); console.log("Pulling remote changes"); await updateStateFromRemote(workspace, state, (_) => true); const diffs = diffState(state); await copyNonJsonFiles(state); console.log(`Applying changes to files`); for await (const diff of diffs) { await applyDiff( diff.diff, path.join(Deno.cwd(), diff.localPath), ); } console.log(colors.green.underline("Done! All changes applied.")); console.log(state); async function applyDiff(diffs: Difference[], file: string) { ensureDir(path.dirname(file)); let json; try { json = JSON.parse(await Deno.readTextFile(file)); } catch { json = {}; } // TODO: Delegate the below to the object itself // This would work by infering the type of `JSON` (which includes then statically typing it using decoverto) and then // delegating the applying of the diffs to the object via an interface for (const diff of diffs) { if (diff.type === "CREATE") { setValueByPath(json, diff.path, diff.value); } else if (diff.type === "REMOVE") { setValueByPath(json, diff.path, undefined); } else if (diff.type === "CHANGE") { setValueByPath(json, diff.path, diff.value); } } // TODO: Use decoverto instanceToPlain below. await Deno.writeTextFile(file, JSON.stringify(json, undefined, " "), { create: true, }); } async function copyNonJsonFiles(state: State): Promise { for await ( const { path, localFile, stateFile, isIgnored } of state.getFiles() ) { if (path.endsWith(".json") || isIgnored || !localFile) { continue; } const target = await Deno.open(stateFile.path, { create: true, write: true, }); const source = await Deno.open( State.getInternalWmillFolder( state.stateContentFor(stateFile.path)!, ), { read: true, write: false, }, ); await source.readable.pipeTo(target.writable); } } } async function* diffState(state: State): AsyncGenerator { for await ( const { path, localFile, stateFile, isIgnored } of state.getFiles() ) { if (!path.endsWith(".json") || isIgnored) { continue; } const fileText = localFile ?? "{}"; const old = JSON.parse(fileText); const fileHash = objectHash(old); if (fileHash !== stateFile.getHash()) { const stateContent = await stateFile.getContent() as any; const diff = microdiff( old, stateContent ?? {}, { cyclesFix: false }, ); yield new StateDiff(stateFile.path, diff); } } } function prettyDiff(diffs: Difference[]) { for (const diff of diffs) { let pathString = ""; for (const pathSegment of diff.path) { if (typeof pathSegment === "string") { pathString += "."; pathString += pathSegment; } else { pathString += "["; pathString += pathSegment; pathString += "]"; } } if (diff.type === "REMOVE" || diff.type === "CHANGE") { console.log(colors.red("- " + pathString + " = " + diff.oldValue)); } if (diff.type === "CREATE" || diff.type === "CHANGE") { console.log(colors.green("+ " + pathString + " = " + diff.value)); } } } async function push(opts: GlobalOptions & { raw: boolean }) { if (opts.raw) { const opts2 = opts as any; opts2.raw = undefined; await pushRaw(opts2, undefined); return; } const state = await getState(opts); const workspace = await resolveWorkspace(opts); await requireLogin(opts); let error = false; await updateStateFromRemote(workspace, state, async (filename) => { const e = state.get(filename); if (!e) { throw new Error("!? State change on untracked file ?!"); } let fileJSON; try { fileJSON = JSON.parse( await Deno.readTextFile(path.join(Deno.cwd(), e.path)), ); } catch { fileJSON = {}; } const file = inferTypeFromPath(e.path, fileJSON); const eContent = (inferTypeFromPath(e.path, await e.getContent())) ?? {}; const fileHash = objectHash(file); const eHash = objectHash(eContent); if (fileHash !== eHash) { console.log( colors.red("!! Local and Remote change present. Local diff:"), ); prettyDiff(microdiff(eContent as any, file, { cyclesFix: false })); console.log( colors.red( "Consider comitting or otherwise saving your work and pulling to load any remote changes", ), ); error = true; return false; } return true; }); if (error) { return; } for await ( const { path, isIgnored, stateFile, localFile } of state.getFiles() ) { if (!path.endsWith(".json") || isIgnored) continue; const fileJSON = JSON.parse(localFile ?? "{}"); const file = inferTypeFromPath(path, fileJSON); const eContent = (inferTypeFromPath(path, await stateFile.getContent())) ?? {}; const fileHash = objectHash(file); const eHash = objectHash(eContent); if (fileHash !== eHash) { const remotePath = stateFile.path.split(".")[0]; const type = getTypeStrFromPath(stateFile.path); if (type === "script") { // Diffing makes no sense for scripts - instead fetch parent hash & check hash again. // If hash is still missmatched - create new script as child. const typed = decoverto.type(ScriptFile).plainToInstance(file); const contentPath = await findContentFile(stateFile.path); const language = inferContentTypeFromFilePath(contentPath); const content = await Deno.readTextFile(contentPath); try { const remote = await ScriptService.getScriptByPath({ workspace: workspace.workspaceId, path: remotePath, }); if (objectHash(remote) !== fileHash) { await ScriptService.createScript({ workspace: workspace.workspaceId, requestBody: { content, description: typed.description, language, path: remotePath, summary: typed.summary, is_template: typed.is_template, kind: typed.kind, lock: undefined, parent_hash: remote.hash, schema: typed.schema, }, }); } } catch { // no parent hash await ScriptService.createScript({ workspace: workspace.workspaceId, requestBody: { content, description: typed.description, language, path: remotePath, summary: typed.summary, is_template: typed.is_template, kind: typed.kind, lock: undefined, parent_hash: undefined, schema: typed.schema, }, }); } } else { const diff = microdiff(eContent as any, file, { cyclesFix: false }); await applyDiff( workspace.workspaceId, remotePath, file, diff, ); } } } let anyRemoteChanges = false; await updateStateFromRemote(workspace, state, (_) => { anyRemoteChanges = true; return true; }); if (anyRemoteChanges) { console.log("New remote changes - consider pulling"); } function applyDiff( workspace: string, remotePath: string, file: | ScriptFile | VariableFile | FlowFile | ResourceFile | ResourceTypeFile | FolderFile, diffs: Difference[], ) { if (file instanceof ScriptFile) { throw new Error( "This code path should be unreachable - we should never generate diffs for scripts", ); } else if (file instanceof FolderFile) { const parts = remotePath.split("/"); if (parts[0] === "f") { remotePath = parts[1]; } else { remotePath = parts[0]; } } return file.pushDiffs(workspace, remotePath, diffs); } } class StateDiff { localPath: string; diff: Difference[]; constructor( localPath: string, diff: Difference[], ) { this.localPath = localPath; this.diff = diff; } } async function init(opts: GlobalOptions) { try { await Deno.mkdir(State.getInternalWmillFolder()); } catch { console.log( colors.red( "! Looks like this folder is already initialized or we are missing permissions to do so. Exiting.", ), ); return; } await Deno.writeTextFile( ".wmillignore", "# Write any ignores here as you see fit. Same syntax as .gitignore.", ); const workspace = await resolveWorkspace(opts); const newState = new State( new Map(), new Map(), workspace.workspaceId, workspace.remote, ); await newState.save(); } async function pullRaw( opts: GlobalOptions & { override: boolean }, ) { const workspace = await resolveWorkspace(opts); const zipDir = await downloadZip(workspace); if (!zipDir) return; // TODO use ZipFSElement & readDirRecursiveWithIgnore here // TODO also remember to read content via entry methods now instead of direct I/O for await ( const entry of readDirRecursiveWithIgnore( (_) => false, ZipFSElement(zipDir), ) ) { const filePath = entry.path; if (entry.isDirectory) { await ensureDir(filePath); continue; } await ensureDir(path.dirname(filePath)); if (!opts.override) { let exists = false; try { const _stat = await Deno.stat(filePath); exists = true; } catch { exists = false; } if (exists) { if (await Deno.readTextFileSync(filePath) === await entry.getContentText()) { continue; } if ( !(await Confirm.prompt( "Conflict at " + filePath + " do you want to override the local version?", )) ) { continue; } } } console.log("Writing " + filePath) await Deno.writeFile(filePath, await entry.getContentBytes()); } console.log(colors.green("Done. Wrote all files to disk.")); } type PushRawCandidate = { path: string; namespaceKind: "user" | "group" | "folder"; namespaceName: string; }; type PushRawResourceTypeCandidate = { path: string; }; type PushRawFolderCandidate = { path: string; namespaceName: string; }; async function pushRawFindCandidateFiles( dir: string, ): Promise< { normal: PushRawCandidate[]; resourceTypes: PushRawResourceTypeCandidate[]; folders: PushRawFolderCandidate[]; } > { dir = path.resolve(dir); if (path.dirname(dir).startsWith(".")) { return { normal: [], resourceTypes: [], folders: [] }; } const normalCandidates: PushRawCandidate[] = []; const resourceTypeCandidates: PushRawResourceTypeCandidate[] = []; const folderCandidates: PushRawFolderCandidate[] = []; for await (const e of Deno.readDir(dir)) { if (e.isDirectory) { if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name; for await (const e2 of Deno.readDir(newDir)) { if (e2.isDirectory) { if (e2.name.startsWith(".")) continue; const namespaceName = e2.name; const stack: string[] = []; { const path = newDir + "/" + namespaceName + "/"; stack.push(path); try { await Deno.stat(path + "folder.meta.json"); folderCandidates.push({ namespaceName, path: path + "folder.meta.json", }); } catch { } } while (stack.length > 0) { const dir2 = stack.pop()!; for await (const e3 of Deno.readDir(dir2)) { if (e3.isFile) { if (e3.name === "folder.meta.json") continue; normalCandidates.push({ path: dir2 + e3.name, namespaceKind: e.name == "g" ? "group" : e.name == "u" ? "user" : "folder", namespaceName: namespaceName, }); } else { stack.push(dir2 + e3.name + "/"); } } } } } } else { console.log( colors.yellow( "Including organizational folder " + e.name + " in push!", ), ); const { normal, resourceTypes, folders } = await pushRawFindCandidateFiles( path.join(dir, e.name), ); normalCandidates.push(...normal); resourceTypeCandidates.push(...resourceTypes); folderCandidates.push(...folders); } } else { // handle root files if (e.name.endsWith(".resource-type.json")) { resourceTypeCandidates.push({ path: dir + (dir.endsWith("/") ? "" : "/") + e.name, }); } } } return { normal: normalCandidates, folders: folderCandidates, resourceTypes: resourceTypeCandidates, }; } async function pushRaw(opts: GlobalOptions, dir?: string) { dir = dir ?? Deno.cwd(); const workspace = await resolveWorkspace(opts); await requireLogin(opts); console.log(colors.blue("Searching Directory...")); const { normal, resourceTypes, folders } = await pushRawFindCandidateFiles( dir, ); console.log( colors.blue( "Found " + (normal.length + resourceTypes.length + folders.length) + " candidates", ), ); for (const resourceType of resourceTypes) { const fileName = resourceType.path.substring( resourceType.path.lastIndexOf("/") + 1, ); const fileNameParts = fileName.split("."); // invalid file names, like my.cool.script.script.json. Not valid. if (fileNameParts.length != 3) { console.log( colors.yellow("invalid file name found at " + resourceType.path), ); continue; } // filter out non-json files. Note that we filter out script contents above, so this is really an error. if (fileNameParts.at(-1) != "json") { console.log(colors.yellow("non-JSON file found at " + resourceType.path)); continue; } console.log("pushing resource type " + fileNameParts.at(-3)!); await decoverto.type(ResourceTypeFile).rawToInstance( await Deno.readTextFile(resourceType.path), ).push(workspace.workspaceId, fileNameParts.at(-3)!); } for (const folder of folders) { await decoverto.type(FolderFile).plainToInstance( JSON.parse(await Deno.readTextFile(folder.path)), ).push( workspace.workspaceId, "f/" + folder.namespaceName, ); } for (const candidate of normal) { // full file name. No leading /. includes .type.json const fileName = candidate.path.substring( candidate.path.lastIndexOf("/") + 1, ); // figure out just the path after ...../u|g/username|group/ (in extra dir) const dirParts = candidate.path.split("/").filter((x) => x.length > 0); // TODO: check version for folder const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f"); const extraDir = dirParts.slice(gIndex + 2, -1).join("/"); // file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed. const fileNameParts = fileName.split("."); // filter out script content files if ( fileNameParts.at(-1) == "ts" || fileNameParts.at(-1) == "py" || fileNameParts.at(-1) == "go" ) { // probably part of a script. Silent ignore. continue; } // invalid file names, like my.cool.script.script.json. Not valid. if (fileNameParts.length != 3) { console.log( colors.yellow("invalid file name found at " + candidate.path), ); continue; } // filter out non-json files. Note that we filter out script contents above, so this is really an error. if (fileNameParts.at(-1) != "json") { console.log(colors.yellow("non-JSON file found at " + candidate.path)); continue; } // get the type & filter it for valid ones. const type = fileNameParts.at(-2); if (type == "resource-type") { console.log( colors.yellow( "Found resource type file at " + candidate.path + " this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!", ), ); continue; } if ( type != "flow" && type != "resource" && type != "script" && type != "variable" ) { console.log( colors.yellow( "file with invalid type " + type + " found at " + candidate.path, ), ); continue; } // create the remotePath for the API const remotePath = (candidate.namespaceKind === "group" ? "g/" : (candidate.namespaceKind === "user" ? "u/" : "f/")) + candidate.namespaceName + "/" + (extraDir.length > 0 ? extraDir + "/" : "") + fileNameParts.at(-3); console.log("pushing " + type + " to " + remotePath); const typed = inferTypeFromPath( candidate.path, JSON.parse(await Deno.readTextFile(candidate.path)), ); if (typed instanceof ResourceTypeFile || typed instanceof FolderFile) { throw new Error( "Resource Types and Folders should be filtered out at this point!", ); } else if (typed instanceof ScriptFile) { let contentPath: string; try { contentPath = await findContentFile(candidate.path); } catch (e) { console.log(colors.red(e.toString())); continue; } await pushScript( candidate.path, contentPath, workspace.workspaceId, remotePath, ); } else { await typed.push(workspace.workspaceId, remotePath); } } console.log(colors.underline.bold.green("Successfully Pushed all files.")); } const command = new Command() .command("init") .description( "Initialize this folder as sync root for the currently selected workspace & remote." + "\nBegin by initializing state tracking using `init` & add files you want to track using `add`. `push` & `pull` will then use local state to accurately track changes required on the remote.", ) .action(init as any) .command("pull") .description( "Pull any remote changes and apply them locally. Use --raw for usage without local state tracking.", ) .option("--raw", "Pull without using state.") .option("--raw-override", "Always override local files with remote.", { depends: ["raw"], }) .action(pull as any) .command("push") .description( "Push any local changes and apply them remotely. Use --raw for usage without local state tracking.", ) .option("--raw", "Push without using state.") .action(push as any); export default command;