kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cli): 2-Way sync (#1071)

Browse Source 
* Export file type from each file

* Fix example scripts

* Strongly type CLI files

* Allow bash files

* Update API version

* Remove useless files

* WIP: Diff based push

* Fixup other code

* Implement Flow diffing

* Implement resource type

* Remaining impls

* WIP

* Fix missing file error

* Fix misstyping

* Improve error message

* Fix type inferrence

* Allow REMOVE everywhere

* Fix empty changeset

* Fix error message

* Fix type inferrence 2

* Fix variable diffs

* Fix include checks

* Move push & pull

* Handle script in sync

* Handle scripts

* Allow multi-path creation

* Fix merge conflicts

* Fix #1173

* Update Dependencies

* Add missing await

* Apply review comments

* Fix diff

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
This commit is contained in:
Kai Jellinghaus
2023-02-03 19:49:46 +01:00
committed by GitHub
parent 690399e107
commit fc494fbf7f
21 changed files with 1916 additions and 671 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
cli/context.ts
View File

@@ -1,10 +1,12 @@
// deno-lint-ignore-file no-explicit-any
import { colors, setClient } from "./deps.ts";
import { tryGetLoginInfo } from "./login.ts";
import { colors, setClient, UserService } from "./deps.ts";
import { loginInteractive, tryGetLoginInfo } from "./login.ts";
import { GlobalOptions } from "./types.ts";
import {
addWorkspace,
getActiveWorkspace,
getWorkspaceByName,
removeWorkspace,
Workspace,
} from "./workspace.ts";
@@ -67,6 +69,27 @@ export async function requireLogin(opts: GlobalOptions) {
}
setClient(token, workspace.remote.substring(0, workspace.remote.length - 1));
try {
await UserService.globalWhoami();
} catch {
console.log(
"! Could not reach API given existing credentials. Attempting to reauth...",
);
const newToken = await loginInteractive(workspace.remote);
if (!newToken) {
throw new Error("Could not reauth");
}
removeWorkspace(workspace.name);
workspace.token = newToken;
addWorkspace(workspace);
setClient(
token,
workspace.remote.substring(0, workspace.remote.length - 1),
);
await UserService.globalWhoami();
}
}
export async function tryResolveVersion(

8
cli/decoverto.ts Normal file
View File

@@ -0,0 +1,8 @@
// globally shared decoverto instance
import { Decoverto } from "npm:decoverto";
const decoverto = new Decoverto();
// TODO: Properly type FlowModule
export { Any, array, map, MapShape, model, property } from "npm:decoverto";
export { decoverto };

38
cli/deps.ts
View File

@@ -1,32 +1,40 @@
// windmill
export { setClient } from "https://deno.land/x/windmill@v1.56.0/mod.ts";
export * from "https://deno.land/x/windmill@v1.56.0/windmill-api/index.ts";
export { setClient } from "https://deno.land/x/windmill@v1.61.1/mod.ts";
export * from "https://deno.land/x/windmill@v1.61.1/windmill-api/index.ts";
// cliffy
export { Command } from "https://deno.land/x/cliffy@v0.25.6/command/command.ts";
export { Table } from "https://deno.land/x/cliffy@v0.25.6/table/table.ts";
export { colors } from "https://deno.land/x/cliffy@v0.25.6/ansi/colors.ts";
export { Secret } from "https://deno.land/x/cliffy@v0.25.6/prompt/secret.ts";
export { Select } from "https://deno.land/x/cliffy@v0.25.6/prompt/select.ts";
export { Confirm } from "https://deno.land/x/cliffy@v0.25.6/prompt/confirm.ts";
export { Input } from "https://deno.land/x/cliffy@v0.25.6/prompt/input.ts";
export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
export { Table } from "https://deno.land/x/cliffy@v0.25.7/table/table.ts";
export { colors } from "https://deno.land/x/cliffy@v0.25.7/ansi/colors.ts";
export { Secret } from "https://deno.land/x/cliffy@v0.25.7/prompt/secret.ts";
export { Select } from "https://deno.land/x/cliffy@v0.25.7/prompt/select.ts";
export { Confirm } from "https://deno.land/x/cliffy@v0.25.7/prompt/confirm.ts";
export { Input } from "https://deno.land/x/cliffy@v0.25.7/prompt/input.ts";
export {
DenoLandProvider,
UpgradeCommand,
} from "https://deno.land/x/cliffy@v0.25.6/command/upgrade/mod.ts";
} from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
// std
export { Untar } from "https://deno.land/std@0.170.0/archive/untar.ts";
export * as path from "https://deno.land/std@0.170.0/path/mod.ts";
export { ensureDir } from "https://deno.land/std@0.170.0/fs/ensure_dir.ts";
export { Untar } from "https://deno.land/std@0.176.0/archive/untar.ts";
export * as path from "https://deno.land/std@0.176.0/path/mod.ts";
export { ensureDir } from "https://deno.land/std@0.176.0/fs/ensure_dir.ts";
export {
copy,
readAll,
readerFromStreamReader,
} from "https://deno.land/std@0.170.0/streams/mod.ts";
export { DelimiterStream } from "https://deno.land/std@0.170.0/streams/mod.ts";
} from "https://deno.land/std@0.176.0/streams/mod.ts";
export { DelimiterStream } from "https://deno.land/std@0.176.0/streams/mod.ts";
export { iterateReader } from "https://deno.land/std@0.176.0/streams/iterate_reader.ts";
// other
export { getAvailablePort } from "https://deno.land/x/port@1.0.0/mod.ts";
export { default as dir } from "https://deno.land/x/dir@1.5.1/mod.ts";
export { passwordGenerator } from "https://deno.land/x/password_generator@latest/mod.ts"; // TODO: I think the version is called latest, but it's still pinned.
export { nanoid } from "https://deno.land/x/nanoid@v3.0.0/mod.ts";
export * as cbor from "https://deno.land/x/cbor@v1.4.1/index.js";
export { default as Murmurhash3 } from "https://deno.land/x/murmurhash@v1.0.0/mod.ts";
export {
default as microdiff,
} from "https://deno.land/x/microdiff@v1.3.1/index.ts";
export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";

0
cli/examples/u/admin/fib/fib.script.ts → cli/examples/u/admin/fib/fib.ts
View File

0
cli/examples/u/admin/go_test/go_test.script.go → cli/examples/u/admin/go_test/go_test.go
View File

169
cli/flow.ts
View File

@@ -1,16 +1,145 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions } from "./types.ts";
import {
Difference,
GlobalOptions,
PushDiffs,
Resource,
setValueByPath,
} from "./types.ts";
import {
colors,
Command,
Flow,
FlowModule,
FlowService,
JobService,
OpenFlow,
microdiff,
OpenFlowWPath,
Table,
} from "./deps.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { resolve, track_job } from "./script.ts";
import { Any, array, decoverto, model, property } from "./decoverto.ts";
@model()
export class FlowValueFilePart {
@property(array(Any))
modules: Array<FlowModule>;
@property(Any)
failure_module?: FlowModule;
@property(() => Boolean)
same_worker?: boolean;
constructor(modules: Array<FlowModule>) {
this.modules = modules;
}
}
// this is effectively "OpenFlow" but a copy as it is accepted by the CLI
@model()
export class FlowFile implements Resource, PushDiffs {
@property(() => String)
summary: string;
@property(() => String)
description?: string;
@property(() => FlowValueFilePart)
value: FlowValueFilePart;
@property(Any)
schema?: any;
constructor(summary: string, value: FlowValueFilePart) {
this.summary = summary;
this.value = value;
}
async pushDiffs(
workspace: string,
remotePath: string,
diffs: Difference[],
): Promise<void> {
if (
await FlowService.existsFlowByPath({
workspace: workspace,
path: remotePath,
})
) {
console.log(
colors.bold.yellow(
`Applying ${diffs.length} diffs to existing flow...`,
),
);
// TODO: Make these optional in backend (not path ofc)
const changeset: OpenFlowWPath = {
path: remotePath,
summary: this.summary,
value: this.value,
description: this.description, // This is OpenAPIed as optional, but isn't
schema: this.schema, // Same
};
for (const diff of diffs) {
if (
diff.type !== "REMOVE" &&
(
diff.path[0] !== "value" && (
diff.path.length !== 1 ||
!["summary", "description", "schema"].includes(
diff.path[0] as string,
)
)
)
) {
throw new Error("Invalid flow diff with path " + diff.path);
}
if (diff.type === "CREATE" || diff.type === "CHANGE") {
setValueByPath(changeset, diff.path, diff.value);
} else if (diff.type === "REMOVE") {
setValueByPath(changeset, diff.path, null);
}
}
const hasChanges = Object.values(changeset).some((v) =>
v !== null && typeof v !== "undefined"
);
if (!hasChanges) {
console.log(colors.yellow("! Skipping empty changeset"));
return;
}
await FlowService.updateFlow({
workspace: workspace,
path: remotePath,
requestBody: changeset,
});
} else {
console.log(colors.bold.yellow("Creating new flow..."));
await FlowService.createFlow({
workspace: workspace,
requestBody: {
path: remotePath,
summary: this.summary,
value: this.value,
schema: this.schema,
description: this.description,
},
});
}
}
async push(workspace: string, remotePath: string): Promise<void> {
let remote: Flow | undefined;
try {
remote = await FlowService.getFlowByPath({
workspace,
path: remotePath,
});
} catch {
remote = undefined;
}
await this.pushDiffs(
workspace,
remotePath,
microdiff(remote ?? {}, this, { cyclesFix: false }),
);
}
}
type Options = GlobalOptions;
@@ -30,38 +159,10 @@ export async function pushFlow(
workspace: string,
remotePath: string,
) {
const data: OpenFlow = JSON.parse(await Deno.readTextFile(filePath));
if (
await FlowService.existsFlowByPath({
workspace: workspace,
path: remotePath,
})
) {
console.log(colors.bold.yellow("Updating existing flow..."));
await FlowService.updateFlow({
workspace: workspace,
path: remotePath,
requestBody: {
path: remotePath,
summary: data.summary,
value: data.value,
schema: data.schema,
description: data.description,
},
});
} else {
console.log(colors.bold.yellow("Creating new flow..."));
await FlowService.createFlow({
workspace: workspace,
requestBody: {
path: remotePath,
summary: data.summary,
value: data.value,
schema: data.schema,
description: data.description,
},
});
}
const data = decoverto.type(FlowFile).rawToInstance(
await Deno.readTextFile(filePath),
);
await data.push(workspace, remotePath);
}
async function list(opts: GlobalOptions & { showArchived?: boolean }) {

174
cli/folder.ts
View File

@@ -1,6 +1,122 @@
import { colors, Command, Folder, FolderService } from "./deps.ts";
import { colors, Command, Folder, FolderService, microdiff } from "./deps.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { GlobalOptions } from "./types.ts";
import {
Difference,
GlobalOptions,
PushDiffs,
Resource,
setValueByPath,
} from "./types.ts";
import {
array,
decoverto,
map,
MapShape,
model,
property,
} from "./decoverto.ts";
@model()
export class FolderFile implements Resource, PushDiffs {
@property(array(() => String))
owners: Array<string> | undefined;
@property(map(() => String, () => Boolean, { shape: MapShape.Object }))
extra_perms: Map<string, boolean> | undefined;
async push(workspace: string, remotePath: string): Promise<void> {
if (remotePath.startsWith("/")) {
remotePath = remotePath.substring(1);
}
if (remotePath.startsWith("f/")) {
remotePath = remotePath.substring(2);
}
let existing: Folder | undefined;
try {
existing = await FolderService.getFolder({ workspace, name: remotePath });
} catch {
existing = undefined;
}
await this.pushDiffs(
workspace,
remotePath,
microdiff(existing ?? {}, this, { cyclesFix: false }),
);
}
async pushDiffs(
workspace: string,
remotePath: string,
diffs: Difference[],
): Promise<void> {
if (remotePath.startsWith("/")) {
remotePath = remotePath.substring(1);
}
if (remotePath.startsWith("f/")) {
remotePath = remotePath.substring(2);
}
// TODO: Support this in backend
let exists: boolean;
try {
exists = !!await FolderService.getFolder({ workspace, name: remotePath });
} catch {
exists = false;
}
if (exists) {
console.log(
colors.bold.yellow(
`Applying ${diffs.length} diffs to existing folder...`,
),
);
const changeset: {
owners?: string[] | undefined;
extra_perms?: any;
} = {};
for (const diff of diffs) {
if (
diff.type !== "REMOVE" &&
(
diff.path.length !== 1 ||
!["owners", "extra_perms"].includes(diff.path[0] as string)
)
) {
throw new Error("Invalid folder diff with path " + diff.path);
}
if (diff.type === "CREATE" || diff.type === "CHANGE") {
setValueByPath(changeset, diff.path, diff.value);
} else if (diff.type === "REMOVE") {
setValueByPath(changeset, diff.path, null);
}
}
const hasChanges = Object.values(changeset).some((v) =>
v !== null && typeof v !== "undefined"
);
if (!hasChanges) {
console.log(colors.yellow("! Skipping empty changeset"));
return;
}
await FolderService.updateFolder({
workspace: workspace,
name: remotePath,
requestBody: changeset,
});
} else {
console.log(colors.bold.yellow("Creating new folder..."));
await FolderService.createFolder({
workspace: workspace,
requestBody: {
name: remotePath,
extra_perms: this.extra_perms,
owners: this.owners,
},
});
}
}
}
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
const workspace = await resolveWorkspace(opts);
@@ -21,61 +137,15 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
console.log(colors.bold.underline.green("Resource successfully pushed"));
}
type FolderFile = {
owners: Array<string> | undefined;
extra_perms: Record<string, boolean> | undefined;
};
export async function pushFolder(
workspace: string,
filePath: string,
remotePath: string,
) {
if (remotePath.startsWith("/")) {
remotePath = remotePath.substring(1);
}
if (remotePath.startsWith("f/")) {
remotePath = remotePath.substring(2);
}
const data: FolderFile = JSON.parse(await Deno.readTextFile(filePath));
let optFolder: Folder | undefined;
try {
optFolder = await FolderService.getFolder({ workspace, name: remotePath });
} catch {
optFolder = undefined;
}
if (optFolder) {
// for (const [k, v] of Object.entries(optFolder.extra_perms)) {
// if (!data.extra_perms || data.extra_perms[k] !== v) {
// console.log(colors.red.underline.bold(`Extra Perms missmatch on ${k}`));
// return;
// }
// }
console.log(colors.yellow("Updating existing folder..."));
await FolderService.updateFolder({
workspace,
name: remotePath,
requestBody: {
extra_perms: data.extra_perms,
owners: data.owners,
},
});
} else {
console.log(colors.yellow("Creating new folder..."));
await FolderService.createFolder({
workspace,
requestBody: {
name: remotePath,
extra_perms: data.extra_perms,
owners: data.owners,
},
});
// HACK: Workaround backend automatically adding current user to folder.
await pushFolder(workspace, filePath, remotePath);
}
const data = decoverto.type(FolderFile).rawToInstance(
await Deno.readTextFile(filePath),
);
data.push(workspace, remotePath);
}
const command = new Command()

14
cli/hub.ts
View File

@@ -1,6 +1,6 @@
import { Command } from "./deps.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { pushResourceTypeDef } from "./resource-type.ts";
import { ResourceTypeFile } from "./resource-type.ts";
import { GlobalOptions } from "./types.ts";
async function pull(opts: GlobalOptions) {
@@ -56,14 +56,10 @@ async function pull(opts: GlobalOptions) {
const x of list
) {
console.log("syncing " + x.name);
await pushResourceTypeDef(
workspace.workspaceId,
x.name,
{
description: x.description,
schema: JSON.parse(x.schema),
},
);
const f = new ResourceTypeFile();
f.description = x.description;
f.schema = JSON.parse(x.schema);
await f.push(workspace.workspaceId, x.name);
}
}

16
cli/main.ts
View File

@@ -8,13 +8,14 @@ import variable from "./variable.ts";
import push from "./push.ts";
import pull from "./pull.ts";
import hub from "./hub.ts";
// import folder from "./folder.ts";
import folder from "./folder.ts";
import sync from "./sync.ts";
import { tryResolveVersion } from "./context.ts";
import { GlobalOptions } from "./types.ts";
const VERSION = "v1.61.1";
const command = new Command()
let command: any = new Command()
.name("wmill")
.description("A simple CLI tool for windmill.")
.globalOption(
@@ -32,10 +33,9 @@ const command = new Command()
.command("resource", resource)
.command("user", user)
.command("variable", variable)
.command("push", push)
.command("pull", pull)
.command("hub", hub)
// .command("folder", folder)
.command("folder", folder)
.command("sync", sync)
.command("version", "Show version information")
.action(async (opts) => {
console.log("CLI build against " + VERSION);
@@ -61,6 +61,12 @@ const command = new Command()
}),
);
if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
command = command
.command("push", push)
.command("pull", pull);
}
try {
await command.parse(Deno.args);
} catch (e) {

71
cli/pull.ts
View File

@@ -1,20 +1,11 @@
// deno-lint-ignore-file no-explicit-any
import { resolveWorkspace } from "./context.ts";
import { GlobalOptions } from "./types.ts";
import {
colors,
Command,
Confirm,
copy,
ensureDir,
path,
readerFromStreamReader,
Untar,
} from "./deps.ts";
async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
const workspace = await resolveWorkspace(opts);
import { colors, Command, readerFromStreamReader, Untar } from "./deps.ts";
import { Workspace } from "./workspace.ts";
export async function downloadTar(
workspace: Workspace,
): Promise<Untar | undefined> {
const requestHeaders: HeadersInit = new Headers();
requestHeaders.set("Authorization", "Bearer " + workspace.token);
requestHeaders.set("Content-Type", "application/octet-stream");
@@ -35,51 +26,29 @@ async function pull(opts: GlobalOptions & { override: boolean }, dir: string) {
),
);
console.log(await tarResponse.text());
return;
return undefined;
}
const streamReader = tarResponse.body?.getReader();
if (!streamReader) {
console.log(colors.red("Failed to read tar request body"));
return;
return undefined;
}
console.log(colors.yellow("Streaming tarball to disk..."));
const denoReader = readerFromStreamReader(streamReader);
const untar = new Untar(denoReader);
for await (const entry of untar) {
console.log(entry.fileName);
const filePath = path.resolve(dir, entry.fileName);
if (entry.type === "directory") {
await ensureDir(filePath);
continue;
}
await ensureDir(path.dirname(filePath));
if (!opts.override) {
let exists = false;
try {
const _stat = await Deno.stat(filePath);
exists = true;
} catch {
exists = false;
}
if (exists) {
if (
!(await Confirm.prompt(
"Conflict at " +
filePath +
" do you want to override the local version?",
))
) {
continue;
}
}
}
const file = await Deno.open(filePath, { write: true, create: true });
const len = await copy(entry, file);
await file.truncate(len);
file.close();
}
console.log(colors.green("Done. Wrote all files to disk."));
return untar;
}
async function stub(
_opts: GlobalOptions & { override: boolean },
_dir: string,
) {
console.log(
colors.red.underline(
'Pull is deprecated. Use "sync pull --raw" instead. See <TODO_LINK_HERE> for more information.',
),
);
}
const command = new Command()
@@ -87,6 +56,6 @@ const command = new Command()
"Pull all definitions in the current workspace from the API and write them to disk.",
)
.arguments("<dir:string>")
.action(pull as any);
.action(stub as any);
export default command;

261
cli/push.ts
View File

@@ -1,266 +1,21 @@
// deno-lint-ignore-file no-explicit-any
import { colors, Command, path } from "./deps.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { pushFlow } from "./flow.ts";
import { pushResource } from "./resource.ts";
import { findContentFile, pushScript } from "./script.ts";
import { colors, Command } from "./deps.ts";
import { GlobalOptions } from "./types.ts";
import { pushVariable } from "./variable.ts";
import { pushResourceType } from "./resource-type.ts";
import { pushFolder } from "./folder.ts";
type Candidate = {
path: string;
namespaceKind: "user" | "group" | "folder";
namespaceName: string;
};
type ResourceTypeCandidate = {
path: string;
};
type FolderCandidate = {
path: string;
namespaceName: string;
};
async function findCandidateFiles(
dir: string,
): Promise<
{
normal: Candidate[];
resourceTypes: ResourceTypeCandidate[];
folders: FolderCandidate[];
}
> {
dir = path.resolve(dir);
if (path.dirname(dir).startsWith(".")) {
return { normal: [], resourceTypes: [], folders: [] };
}
const normalCandidates: Candidate[] = [];
const resourceTypeCandidates: ResourceTypeCandidate[] = [];
const folderCandidates: FolderCandidate[] = [];
for await (const e of Deno.readDir(dir)) {
if (e.isDirectory) {
if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f
const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name;
for await (const e2 of Deno.readDir(newDir)) {
if (e2.isDirectory) {
if (e2.name.startsWith(".")) continue;
const namespaceName = e2.name;
const stack: string[] = [];
{
const path = newDir + "/" + namespaceName + "/";
stack.push(path);
try {
await Deno.stat(path + "folder.meta.json");
folderCandidates.push({
namespaceName,
path: path + "folder.meta.json",
});
} catch {}
}
while (stack.length > 0) {
const dir2 = stack.pop()!;
for await (const e3 of Deno.readDir(dir2)) {
if (e3.isFile) {
if (e3.name === "folder.meta.json") continue;
normalCandidates.push({
path: dir2 + e3.name,
namespaceKind: e.name == "g"
? "group"
: e.name == "u"
? "user"
: "folder",
namespaceName: namespaceName,
});
} else {
stack.push(dir2 + e3.name + "/");
}
}
}
}
}
} else {
console.log(
colors.yellow(
"Including organizational folder " + e.name + " in push!",
),
);
const { normal, resourceTypes, folders } = await findCandidateFiles(
path.join(dir, e.name),
);
normalCandidates.push(...normal);
resourceTypeCandidates.push(...resourceTypes);
folderCandidates.push(...folders);
}
} else {
// handle root files
if (e.name.endsWith(".resource-type.json")) {
resourceTypeCandidates.push({
path: dir + (dir.endsWith("/") ? "" : "/") + e.name,
});
}
}
}
return {
normal: normalCandidates,
folders: folderCandidates,
resourceTypes: resourceTypeCandidates,
};
}
async function push(opts: GlobalOptions, dir?: string) {
dir = dir ?? Deno.cwd();
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
console.log(colors.blue("Searching Directory..."));
const { normal, resourceTypes, folders } = await findCandidateFiles(dir);
async function stub(
_opts: GlobalOptions,
_dir?: string,
) {
console.log(
colors.blue(
"Found " + (normal.length + resourceTypes.length + folders.length) +
" candidates",
colors.red.underline(
'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.',
),
);
for (const resourceType of resourceTypes) {
const fileName = resourceType.path.substring(
resourceType.path.lastIndexOf("/") + 1,
);
const fileNameParts = fileName.split(".");
// invalid file names, like my.cool.script.script.json. Not valid.
if (fileNameParts.length != 3) {
console.log(
colors.yellow("invalid file name found at " + resourceType.path),
);
continue;
}
// filter out non-json files. Note that we filter out script contents above, so this is really an error.
if (fileNameParts.at(-1) != "json") {
console.log(colors.yellow("non-JSON file found at " + resourceType.path));
continue;
}
console.log("pushing resource type " + fileNameParts.at(-3)!);
await pushResourceType(
workspace.workspaceId,
resourceType.path,
fileNameParts.at(-3)!,
);
}
for (const folder of folders) {
await pushFolder(
workspace.workspaceId,
folder.path,
"f/" + folder.namespaceName,
);
}
for (const candidate of normal) {
// full file name. No leading /. includes .type.json
const fileName = candidate.path.substring(
candidate.path.lastIndexOf("/") + 1,
);
// figure out just the path after ...../u|g/username|group/ (in extra dir)
const dirParts = candidate.path.split("/").filter((x) => x.length > 0);
// TODO: check version for folder
const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f");
const extraDir = dirParts.slice(gIndex + 2, -1).join("/");
// file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed.
const fileNameParts = fileName.split(".");
// filter out script content files
if (
fileNameParts.at(-1) == "ts" ||
fileNameParts.at(-1) == "py" ||
fileNameParts.at(-1) == "go"
) {
// probably part of a script. Silent ignore.
continue;
}
// invalid file names, like my.cool.script.script.json. Not valid.
if (fileNameParts.length != 3) {
console.log(
colors.yellow("invalid file name found at " + candidate.path),
);
continue;
}
// filter out non-json files. Note that we filter out script contents above, so this is really an error.
if (fileNameParts.at(-1) != "json") {
console.log(colors.yellow("non-JSON file found at " + candidate.path));
continue;
}
// get the type & filter it for valid ones.
const type = fileNameParts.at(-2);
if (type == "resource-type") {
console.log(
colors.yellow(
"Found resource type file at " +
candidate.path +
" this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!",
),
);
continue;
}
if (
type != "flow" &&
type != "resource" &&
type != "script" &&
type != "variable"
) {
console.log(
colors.yellow(
"file with invalid type " + type + " found at " + candidate.path,
),
);
continue;
}
// create the remotePath for the API
const remotePath = (candidate.namespaceKind === "group"
? "g/"
: (candidate.namespaceKind === "user" ? "u/" : "f/")) +
candidate.namespaceName +
"/" +
(extraDir.length > 0 ? extraDir + "/" : "") +
fileNameParts.at(-3);
console.log("pushing " + type + " to " + remotePath);
if (type == "flow") {
await pushFlow(candidate.path, workspace.workspaceId, remotePath);
} else if (type == "resource") {
await pushResource(workspace.workspaceId, candidate.path, remotePath);
} else if (type == "script") {
let contentPath: string;
try {
contentPath = await findContentFile(candidate.path);
} catch (e) {
console.log(colors.red(e.toString()));
continue;
}
await pushScript(
candidate.path,
contentPath,
workspace.workspaceId,
remotePath,
);
} else if (type == "variable") {
await pushVariable(workspace.workspaceId, candidate.path, remotePath);
}
}
console.log(colors.underline.bold.green("Successfully Pushed all files."));
}
const command = new Command()
.description("Push all files from a folder")
.arguments("[dir:string]")
.action(push as any);
.action(stub as any);
export default command;

166
cli/resource-type.ts
View File

@@ -1,65 +1,129 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions } from "./types.ts";
import {
Difference,
GlobalOptions,
PushDiffs,
Resource as ResourceI,
setValueByPath,
} from "./types.ts";
import { requireLogin, resolveWorkspace } from "./context.ts";
import { colors, Command, ResourceService, Table } from "./deps.ts";
import {
colors,
Command,
EditResourceType,
microdiff,
ResourceService,
ResourceType,
Table,
} from "./deps.ts";
import { Any, decoverto, model, property } from "./decoverto.ts";
type ResourceTypeFile = {
@model()
export class ResourceTypeFile implements ResourceI, PushDiffs {
@property(Any)
schema?: any;
@property(() => String)
description?: string;
};
async push(workspace: string, remotePath: string): Promise<void> {
let existing: ResourceType | undefined;
try {
existing = await ResourceService.getResourceType({
workspace,
path: remotePath,
});
} catch {
existing = undefined;
}
this.pushDiffs(
workspace,
remotePath,
microdiff(existing ?? {}, this, { cyclesFix: false }),
);
}
async pushDiffs(
workspace: string,
remotePath: string,
diffs: Difference[],
): Promise<void> {
if (
await ResourceService.existsResourceType({
workspace: workspace,
path: remotePath,
})
) {
if (
(await ResourceService.listResourceType({ workspace })).findIndex((x) =>
x.name === remotePath
) === -1
) {
console.log(
"Resource type " + remotePath +
" is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
);
return;
}
console.log(
colors.yellow(
`Applying ${diffs.length} diffs to existing resource type...`,
),
);
const changeset: EditResourceType = {};
for (const diff of diffs) {
if (
diff.type !== "REMOVE" &&
(
diff.path.length !== 1 ||
!["schema", "description"].includes(diff.path[0] as string)
)
) {
throw new Error("Invalid resource type diff with path " + diff.path);
}
if (diff.type === "CREATE" || diff.type === "CHANGE") {
setValueByPath(changeset, diff.path, diff.value);
} else if (diff.type === "REMOVE") {
setValueByPath(changeset, diff.path, null);
}
}
const hasChanges = Object.values(changeset).some((v) =>
v !== null && typeof v !== "undefined"
);
if (!hasChanges) {
console.log(colors.yellow("! Skipping empty changeset"));
return;
}
await ResourceService.updateResourceType({
workspace: workspace,
path: remotePath,
requestBody: changeset,
});
} else {
console.log(colors.yellow("Creating new resource type..."));
await ResourceService.createResourceType({
workspace: workspace,
requestBody: {
name: remotePath,
description: this.description,
schema: this.schema,
workspace_id: workspace,
},
});
}
}
}
export async function pushResourceType(
workspace: string,
filePath: string,
name: string,
) {
const data: ResourceTypeFile = JSON.parse(await Deno.readTextFile(filePath));
await pushResourceTypeDef(workspace, name, data);
}
export async function pushResourceTypeDef(
workspace: string,
name: string,
data: ResourceTypeFile,
) {
if (
await ResourceService.existsResourceType({
workspace: workspace,
path: name,
})
) {
console.log(colors.yellow("Updating existing resource type..."));
if (
(await ResourceService.listResourceType({ workspace })).findIndex((x) =>
x.name === name
) === -1
) {
console.log(
"Resource type " + name +
" is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
);
return;
}
await ResourceService.updateResourceType({
workspace: workspace,
path: name,
requestBody: {
description: data.description,
schema: data.schema,
},
});
} else {
console.log(colors.yellow("Creating new resource type..."));
await ResourceService.createResourceType({
workspace: workspace,
requestBody: {
name: name,
description: data.description,
schema: data.schema,
workspace_id: workspace,
},
});
}
const data: ResourceTypeFile = decoverto.type(ResourceTypeFile).rawToInstance(
await Deno.readTextFile(filePath),
);
await data.push(workspace, name);
}
type PushOptions = GlobalOptions;

194
cli/resource.ts
View File

@@ -1,80 +1,142 @@
import { GlobalOptions } from "./types.ts";
import {
Difference,
GlobalOptions,
PushDiffs,
Resource as Resource2,
setValueByPath,
} from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { colors, Command, Resource, ResourceService, Table } from "./deps.ts";
import {
colors,
Command,
EditResource,
microdiff,
Resource,
ResourceService,
Table,
} from "./deps.ts";
import { Any, decoverto, model, property } from "./decoverto.ts";
type ResourceFile = {
value: any;
@model()
export class ResourceFile implements Resource2, PushDiffs {
@property(Any)
value?: any;
@property(() => String)
description?: string;
@property(() => String)
resource_type: string;
@property(() => Boolean)
is_oauth?: boolean; // deprecated
};
constructor(resource_type: string) {
this.resource_type = resource_type;
}
async pushDiffs(
workspace: string,
remotePath: string,
diffs: Difference[],
): Promise<void> {
if (
await ResourceService.existsResource({
workspace: workspace,
path: remotePath,
})
) {
console.log(
colors.yellow(`Applying ${diffs.length} diffs to existing resource...`),
);
const changeset: EditResource = {
path: remotePath, // TODO: Remove this in backend
};
for (const diff of diffs) {
if (diff.path[0] === "is_oauth") {
console.log(
colors.yellow(
"! is_oauth has been removed in newer versions. Ignoring.",
),
);
continue;
}
if (
diff.type !== "REMOVE" &&
(
diff.path[0] !== "value" && (
diff.path.length !== 1 ||
diff.path[0] !== "description"
)
)
) {
throw new Error("Invalid folder diff with path " + diff.path);
}
if (diff.type === "CREATE" || diff.type === "CHANGE") {
setValueByPath(changeset, diff.path, diff.value);
} else if (diff.type === "REMOVE") {
setValueByPath(changeset, diff.path, null);
}
}
const hasChanges = Object.values(changeset).some((v) =>
v !== null && typeof v !== "undefined"
);
if (!hasChanges) {
console.log(colors.yellow("! Skipping empty changeset"));
return;
}
await ResourceService.updateResource({
workspace: workspace,
path: remotePath,
requestBody: changeset,
});
} else {
if (typeof this.is_oauth !== "undefined") {
console.log(
colors.yellow(
"! is_oauth has been removed in newer versions. Ignoring.",
),
);
}
console.log(colors.yellow("Creating new resource..."));
await ResourceService.createResource({
workspace: workspace,
requestBody: {
path: remotePath,
resource_type: this.resource_type,
value: this.value,
description: this.description,
},
});
}
}
async push(workspace: string, remotePath: string): Promise<void> {
let existing: Resource | undefined;
try {
existing = await ResourceService.getResource({
workspace,
path: remotePath,
});
} catch {
existing = undefined;
}
await this.pushDiffs(
workspace,
remotePath,
microdiff(existing ?? {}, this, { cyclesFix: false }),
);
}
}
export async function pushResource(
workspace: string,
filePath: string,
remotePath: string,
) {
const data: ResourceFile = JSON.parse(await Deno.readTextFile(filePath));
if (
await ResourceService.existsResource({
workspace: workspace,
path: remotePath,
})
) {
console.log(colors.yellow("Updating existing resource..."));
const existing = await ResourceService.getResource({
workspace: workspace,
path: remotePath,
});
if (existing.resource_type != data.resource_type) {
console.log(
colors.red.underline.bold(
"Remote resource at " +
remotePath +
" exists & has a different resource type. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
),
);
return;
}
if (typeof data.is_oauth !== "undefined") {
console.log(
colors.yellow(
"! is_oauth has been removed in newer versions. Ignoring.",
),
);
}
await ResourceService.updateResource({
workspace: workspace,
path: remotePath,
requestBody: {
path: remotePath,
value: data.value,
description: data.description,
},
});
} else {
if (typeof data.is_oauth !== "undefined") {
console.log(
colors.yellow(
"! is_oauth has been removed in newer versions. Ignoring.",
),
);
}
console.log(colors.yellow("Creating new resource..."));
await ResourceService.createResource({
workspace: workspace,
requestBody: {
path: remotePath,
resource_type: data.resource_type,
value: data.value,
description: data.description,
},
});
}
const data = decoverto.type(ResourceFile).rawToInstance(
await Deno.readTextFile(filePath),
);
await data.push(workspace, remotePath);
}
type PushOptions = GlobalOptions;

72
cli/script.ts
View File

@@ -2,20 +2,52 @@
import { GlobalOptions } from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import {
colors,
Command,
JobService,
readAll,
Script,
} from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
import { colors, Command, readAll, ScriptService, Table } from "./deps.ts";
ScriptService,
Table,
} from "./deps.ts";
import { Any, array, decoverto, model, property } from "./decoverto.ts";
type ScriptFile = {
@model()
export class ScriptFile {
@property(() => String)
parent_hash?: string;
@property(() => String)
summary: string;
@property(() => String)
description: string;
@property(Any)
schema?: any;
@property(() => Boolean)
is_template?: boolean;
@property(array(() => String))
lock?: Array<string>;
@property({
toInstance: (data) => {
if (data == null) return data;
if (
data === "script" || data === "failure" || data === "trigger" ||
data === "command" || data === "approvial"
) {
return data;
}
throw new Error("Invalid kind " + data);
},
toPlain: (data) => data,
})
kind?: "script" | "failure" | "trigger" | "command" | "approval";
};
constructor(summary: string, description: string) {
this.summary = summary;
this.description = description;
}
}
type PushOptions = GlobalOptions;
async function push(
@@ -48,10 +80,12 @@ async function push(
}
export async function findContentFile(filePath: string) {
console.log("Searching " + filePath);
const candidates = [
filePath.replace(".script.json", ".ts"),
filePath.replace(".script.json", ".py"),
filePath.replace(".script.json", ".go"),
filePath.replace(".script.json", ".sh"),
];
const validCandidates = (
await Promise.all(
@@ -79,23 +113,35 @@ export async function findContentFile(filePath: string) {
return validCandidates[0];
}
export function inferContentTypeFromFilePath(
contentPath: string,
): "python3" | "deno" | "go" | "bash" {
let language = contentPath.substring(contentPath.lastIndexOf("."));
if (language == ".ts") language = "deno";
if (language == ".py") language = "python3";
if (language == ".sh") language = "bash";
if (language == ".go") language = "go";
if (
language != "python3" && language != "deno" && language != "go" &&
language != "bash"
) {
throw new Error("Invalid language: " + language);
}
return language;
}
export async function pushScript(
filePath: string,
contentPath: string,
workspace: string,
remotePath: string,
) {
const data: ScriptFile = JSON.parse(await Deno.readTextFile(filePath));
const data = decoverto.type(ScriptFile).rawToInstance(
await Deno.readTextFile(filePath),
);
const content = await Deno.readTextFile(contentPath);
let language = contentPath.substring(contentPath.lastIndexOf("."));
if (language == ".ts") language = "deno";
if (language == ".py") language = "python3";
if (language == ".go") language = "go";
if (language != "python3" && language != "deno" && language != "go") {
throw new Error("Invalid language: " + language);
}
const language = inferContentTypeFromFilePath(contentPath);
let parent_hash = data.parent_hash;
if (!parent_hash) {
try {

969
cli/sync.ts Normal file
View File

@@ -0,0 +1,969 @@
import { requireLogin, resolveWorkspace } from "./context.ts";
import { getWorkspaceStream, Workspace } from "./workspace.ts";
import { decoverto, map, MapShape, model, property } from "./decoverto.ts";
import {
cbor,
colors,
Command,
Confirm,
copy,
ensureDir,
iterateReader,
microdiff,
nanoid,
objectHash,
path,
ScriptService,
} from "./deps.ts";
import {
Difference,
getTypeStrFromPath,
GlobalOptions,
inferTypeFromPath,
setValueByPath,
} from "./types.ts";
import { downloadTar } from "./pull.ts";
import { FolderFile } from "./folder.ts";
import { ResourceTypeFile } from "./resource-type.ts";
import {
findContentFile,
inferContentTypeFromFilePath,
pushScript,
ScriptFile,
} from "./script.ts";
import { ResourceFile } from "./resource.ts";
import { FlowFile } from "./flow.ts";
import { VariableFile } from "./variable.ts";
type TrackedId = string;
const TrackedId = String;
const CONTENT_ENCODER: cbor.Encoder = new cbor.Encoder({ pack: true });
export class Tracked {
#id: TrackedId;
#parent: State;
path: string;
#content_cache?: string;
constructor(id: TrackedId, parent: State, path: string) {
this.#id = id;
this.#parent = parent;
this.path = path;
}
async getContent(): Promise<unknown | undefined> {
if (this.#content_cache) {
return this.#content_cache;
}
if (!this.#parent.stateRoot) {
throw new Error("Parent uninitialized");
}
const f = this.#parent.contentFile(this.#id);
if (!f) {
return undefined;
}
const data = await Deno.readFile(
path.join(this.#parent.stateRoot, ".wmill", f),
);
const content = CONTENT_ENCODER.decode(data);
this.#content_cache = content;
return content;
}
getHash(): string {
return this.#parent.hashes.get(this.#id)!;
}
getId(): TrackedId {
return this.#id;
}
}
@model()
export class State {
// TODO: Handle script contents, as they are separate files & need to be tracked separately.
// I think the way this makes most sense is actually reating them as a separate kind of file everywhere
// And then move to using a diff-based push system everywhere. This should greatly simplify code & enable this easily
@property(map(() => TrackedId, () => String, { shape: MapShape.Object }))
hashes: Map<TrackedId, string>;
@property(map(() => TrackedId, () => String, { shape: MapShape.Object }))
contentFiles: Map<TrackedId, string>;
@property(map(() => String, () => TrackedId, { shape: MapShape.Object }))
tracked: Map<string, TrackedId>;
@property(() => String)
workspaceId: string;
@property(() => String)
remoteUrl: string;
stateRoot?: string;
constructor(
hashes: Map<TrackedId, string>,
tracked: Map<string, TrackedId>,
contentFiles: Map<TrackedId, string>,
workspaceId: string,
remoteUrl: string,
) {
this.hashes = hashes;
this.tracked = tracked;
this.contentFiles = contentFiles;
this.workspaceId = workspaceId;
this.remoteUrl = remoteUrl;
}
add(path: string) {
if (this.tracked.get(path)) {
throw new Error("Cannot newly track already tracked paths");
} else {
this.tracked.set(path, nanoid());
}
}
public forget(path: string) {
const id = this.tracked.get(path);
if (id) {
this.tracked.delete(path);
this.hashes.delete(id);
}
}
public contentFile(id: TrackedId): string | undefined {
return this.contentFiles.get(id);
}
public get(path: string): Tracked {
const id = this.tracked.get(path);
if (!id) {
throw new Error("Could not resolve path " + path);
}
return new Tracked(id, this, path);
}
public async save(): Promise<void> {
if (!this.stateRoot) {
throw new Error("Uninitialized state root");
}
const encoder = new cbor.Encoder({});
const plain = decoverto.type(State).instanceToPlain(this);
const result: Uint8Array = encoder.encode(plain, {});
await Deno.writeFile(path.join(this.stateRoot, ".wmill", "main"), result, {
create: true,
});
}
public static async loadState(dir: string): Promise<State> {
const source = await Deno.readFile(path.join(dir, ".wmill", "main"));
const encoder = new cbor.Encoder({});
const raw = encoder.decode(source);
const state = decoverto.type(State).plainToInstance(
raw,
);
state.stateRoot = dir;
return Object.freeze(state);
}
}
async function getState(opts: GlobalOptions) {
const existingState = await State.loadState(Deno.cwd());
const workspaceStream = await getWorkspaceStream();
const reader = workspaceStream.getReader();
while (true) {
const res = await reader.read();
if (res.value) {
if (
new URL(res.value.remote) == new URL(existingState.remoteUrl) &&
res.value.workspaceId == existingState.workspaceId
) {
opts.workspace = res.value.name;
(opts as any).__secret_workspace = res.value;
break;
}
}
if (res.done) {
break;
}
}
return existingState;
}
async function updateStateFromRemote(
workspace: Workspace,
state: State,
callback: (filename: string) => PromiseLike<boolean> | boolean,
) {
const untar = await downloadTar(workspace);
if (!untar) throw new Error("Failed to pull Tar");
const decoder = new TextDecoder();
for await (const entry of untar) {
const id = state.tracked.get(entry.fileName);
if (id) {
let val = "";
for await (const e of iterateReader(entry)) {
const tmp = decoder.decode(e);
val += tmp;
}
if (entry.fileName.endsWith(".json")) {
const parsed = JSON.parse(val);
const typed = inferTypeFromPath(entry.fileName, parsed);
const oldHash = state.hashes.get(id);
const newHash = objectHash(typed);
if (!oldHash || oldHash !== newHash) {
if (!await callback(entry.fileName)) {
return; // notice that we are not saving
}
state.hashes.set(id, newHash);
const encoded = CONTENT_ENCODER.encode(typed);
const fileName = nanoid();
await Deno.writeFile(
path.join(state.stateRoot!, ".wmill", fileName),
encoded,
{ create: true },
);
state.contentFiles.set(id, fileName);
}
} else {
const fileName = nanoid();
await Deno.writeTextFile(
path.join(state.stateRoot!, ".wmill", fileName),
val,
{ create: true },
);
state.contentFiles.set(id, fileName);
}
}
}
await state.save();
}
async function pull(
opts: GlobalOptions & { raw: boolean; rawOverride: boolean },
) {
if (opts.raw) {
const opts2 = opts as any;
opts2.override = opts2.rawOverride;
opts2.raw = undefined;
opts2.rawOverride = undefined;
await pullRaw(opts2, Deno.cwd());
return;
}
const state = await getState(opts);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
console.log("Pulling remote changes");
await updateStateFromRemote(workspace, state, (_) => true);
const diffs = diffState(state);
await copyNonJsonFiles(state);
console.log(`Applying changes to files`);
for await (const diff of diffs) {
await applyDiff(
diff.diff,
path.join(state.stateRoot!, diff.localPath),
);
}
console.log(colors.green.underline("Done! All changes applied."));
async function applyDiff(diffs: Difference[], file: string) {
ensureDir(path.dirname(file));
let json;
try {
json = JSON.parse(await Deno.readTextFile(file));
} catch {
json = {};
}
// TODO: Delegate the below to the object itself
// This would work by infering the type of `JSON` (which includes then statically typing it using decoverto) and then
// delegating the applying of the diffs to the object via an interface
for (const diff of diffs) {
if (diff.type === "CREATE") {
setValueByPath(json, diff.path, diff.value);
} else if (diff.type === "REMOVE") {
setValueByPath(json, diff.path, undefined);
} else if (diff.type === "CHANGE") {
setValueByPath(json, diff.path, diff.value);
}
}
// TODO: Use decoverto instanceToPlain below.
await Deno.writeTextFile(file, JSON.stringify(json, undefined, " "), {
create: true,
});
}
async function copyNonJsonFiles(state: State): Promise<void> {
for (const t of state.tracked.keys()) {
if (t.endsWith(".json")) {
continue;
}
const entry = state.get(t);
const target = await Deno.open(entry.path, { create: true, write: true });
const source = await Deno.open(
path.join(
state.stateRoot!,
".wmill",
state.contentFile(entry.getId())!,
),
{
read: true,
write: false,
},
);
await source.readable.pipeTo(target.writable);
}
}
}
async function* diffState(state: State): AsyncGenerator<StateDiff, void, void> {
for (const t of state.tracked.keys()) {
if (!t.endsWith(".json")) {
continue;
}
const entry = state.get(t);
let fileText;
try {
fileText = await Deno.readTextFile(
path.join(state.stateRoot!, entry.path),
);
} catch {
fileText = "{}";
}
const old = JSON.parse(fileText);
const fileHash = objectHash(old);
if (fileHash !== entry.getHash()) {
const stateContent = await entry.getContent() as any;
const diff = microdiff(
old,
stateContent ?? {},
{ cyclesFix: false },
);
yield new StateDiff(entry.getId(), entry.path, diff);
}
}
}
function prettyDiff(diffs: Difference[]) {
for (const diff of diffs) {
let pathString = "";
for (const pathSegment of diff.path) {
if (typeof pathSegment === "string") {
pathString += ".";
pathString += pathSegment;
} else {
pathString += "[";
pathString += pathSegment;
pathString += "]";
}
}
if (diff.type === "REMOVE" || diff.type === "CHANGE") {
console.log(colors.red("- " + pathString + " = " + diff.oldValue));
}
if (diff.type === "CREATE" || diff.type === "CHANGE") {
console.log(colors.green("+ " + pathString + " = " + diff.value));
}
}
}
async function push(opts: GlobalOptions & { raw: boolean }) {
if (opts.raw) {
const opts2 = opts as any;
opts2.raw = undefined;
await pushRaw(opts2, undefined);
return;
}
const state = await getState(opts);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
let error = false;
await updateStateFromRemote(workspace, state, async (filename) => {
const e = state.get(filename);
if (!e) {
throw new Error("!? State change on untracked file ?!");
}
let fileJSON;
try {
fileJSON = JSON.parse(
await Deno.readTextFile(path.join(state.stateRoot!, e.path)),
);
} catch {
fileJSON = {};
}
const file = inferTypeFromPath(e.path, fileJSON);
const eContent = (inferTypeFromPath(e.path, await e.getContent())) ?? {};
const fileHash = objectHash(file);
const eHash = objectHash(eContent);
if (fileHash !== eHash) {
console.log(
colors.red("!! Local and Remote change present. Local diff:"),
);
prettyDiff(microdiff(eContent as any, file, { cyclesFix: false }));
console.log(
colors.red(
"Consider comitting or otherwise saving your work and pulling to load any remote changes",
),
);
error = true;
return false;
}
return true;
});
if (error) {
return;
}
for (const p of state.tracked.keys()) {
if (!p.endsWith(".json")) continue;
const entry = state.get(p);
let fileJSON;
try {
fileJSON = JSON.parse(
await Deno.readTextFile(path.join(state.stateRoot!, entry.path)),
);
} catch {
fileJSON = {};
}
const file = inferTypeFromPath(entry.path, fileJSON);
const eContent =
(inferTypeFromPath(entry.path, await entry.getContent())) ?? {};
const fileHash = objectHash(file);
const eHash = objectHash(eContent);
if (fileHash !== eHash) {
const remotePath = entry.path.split(".")[0];
const type = getTypeStrFromPath(entry.path);
if (type === "script") {
// Diffing makes no sense for scripts - instead fetch parent hash & check hash again.
// If hash is still missmatched - create new script as child.
const typed = decoverto.type(ScriptFile).plainToInstance(file);
const contentPath = await findContentFile(entry.path);
const language = inferContentTypeFromFilePath(contentPath);
const content = await Deno.readTextFile(contentPath);
try {
const remote = await ScriptService.getScriptByPath({
workspace: workspace.workspaceId,
path: remotePath,
});
if (objectHash(remote) !== fileHash) {
await ScriptService.createScript({
workspace: workspace.workspaceId,
requestBody: {
content,
description: typed.description,
language,
path: remotePath,
summary: typed.summary,
is_template: typed.is_template,
kind: typed.kind,
lock: undefined,
parent_hash: remote.hash,
schema: typed.schema,
},
});
}
} catch {
// no parent hash
await ScriptService.createScript({
workspace: workspace.workspaceId,
requestBody: {
content,
description: typed.description,
language,
path: remotePath,
summary: typed.summary,
is_template: typed.is_template,
kind: typed.kind,
lock: undefined,
parent_hash: undefined,
schema: typed.schema,
},
});
}
} else {
const diff = microdiff(eContent as any, file, { cyclesFix: false });
await applyDiff(
workspace.workspaceId,
remotePath,
file,
diff,
);
}
}
}
let anyRemoteChanges = false;
await updateStateFromRemote(workspace, state, (_) => {
anyRemoteChanges = true;
return true;
});
if (anyRemoteChanges) {
console.log("New remote changes - consider pulling");
}
function applyDiff(
workspace: string,
remotePath: string,
file:
| ScriptFile
| VariableFile
| FlowFile
| ResourceFile
| ResourceTypeFile
| FolderFile,
diffs: Difference[],
) {
if (file instanceof ScriptFile) {
throw new Error(
"This code path should be unreachable - we should never generate diffs for scripts",
);
} else if (file instanceof FolderFile) {
const parts = remotePath.split("/");
if (parts[0] === "f") {
remotePath = parts[1];
} else {
remotePath = parts[0];
}
}
return file.pushDiffs(workspace, remotePath, diffs);
}
}
class StateDiff {
trackedId: TrackedId;
localPath: string;
diff: Difference[];
constructor(
trackedId: TrackedId,
localPath: string,
diff: Difference[],
) {
this.trackedId = trackedId;
this.localPath = localPath;
this.diff = diff;
}
}
async function add(opts: GlobalOptions, path: string) {
const state = await getState(opts);
// TODO: Automatically check whether this path exists either locally or on the remote
state.add(path);
if (path.endsWith(".script.json")) {
try {
const f = await findContentFile(path);
state.add(f);
} catch {
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
try {
const old = await ScriptService.getScriptByPath({
workspace: workspace.workspaceId,
path: path.split(".")[0],
});
if (old.language === "python3") {
state.add(path.replace(".script.json", ".py"));
} else if (old.language === "bash") {
state.add(path.replace(".script.json", ".sh"));
} else if (old.language === "deno") {
state.add(path.replace(".script.json", ".ts"));
} else if (old.language === "go") {
state.add(path.replace(".script.json", ".go"));
} else {
throw new Error("Remote returned invalid language?! " + old.language);
}
} catch {
throw new Error(
"Could not infer script language from local or remote. Exiting.",
);
}
}
}
await state.save();
}
async function init(opts: GlobalOptions) {
const root = Deno.cwd();
try {
await Deno.mkdir(path.join(root, ".wmill"));
} catch {
console.log(
colors.red(
"! Looks like this folder is already initialized or we are missing permissions to do so. Exiting.",
),
);
return;
}
const workspace = await resolveWorkspace(opts);
const newState = new State(
new Map(),
new Map(),
new Map(),
workspace.workspaceId,
workspace.remote,
);
newState.stateRoot = root;
await newState.save();
}
async function pullRaw(
opts: GlobalOptions & { override: boolean },
dir: string,
) {
const workspace = await resolveWorkspace(opts);
const untar = await downloadTar(workspace);
if (!untar) return;
for await (const entry of untar) {
console.log(entry.fileName);
const filePath = path.resolve(dir, entry.fileName);
if (entry.type === "directory") {
await ensureDir(filePath);
continue;
}
await ensureDir(path.dirname(filePath));
if (!opts.override) {
let exists = false;
try {
const _stat = await Deno.stat(filePath);
exists = true;
} catch {
exists = false;
}
if (exists) {
if (
!(await Confirm.prompt(
"Conflict at " +
filePath +
" do you want to override the local version?",
))
) {
continue;
}
}
}
const file = await Deno.open(filePath, { write: true, create: true });
const len = await copy(entry, file);
await file.truncate(len);
file.close();
}
console.log(colors.green("Done. Wrote all files to disk."));
}
type PushRawCandidate = {
path: string;
namespaceKind: "user" | "group" | "folder";
namespaceName: string;
};
type PushRawResourceTypeCandidate = {
path: string;
};
type PushRawFolderCandidate = {
path: string;
namespaceName: string;
};
async function pushRawFindCandidateFiles(
dir: string,
): Promise<
{
normal: PushRawCandidate[];
resourceTypes: PushRawResourceTypeCandidate[];
folders: PushRawFolderCandidate[];
}
> {
dir = path.resolve(dir);
if (path.dirname(dir).startsWith(".")) {
return { normal: [], resourceTypes: [], folders: [] };
}
const normalCandidates: PushRawCandidate[] = [];
const resourceTypeCandidates: PushRawResourceTypeCandidate[] = [];
const folderCandidates: PushRawFolderCandidate[] = [];
for await (const e of Deno.readDir(dir)) {
if (e.isDirectory) {
if (e.name == "u" || e.name == "g" || e.name == "f") { // TODO: Check version for f
const newDir = dir + (dir.endsWith("/") ? "" : "/") + e.name;
for await (const e2 of Deno.readDir(newDir)) {
if (e2.isDirectory) {
if (e2.name.startsWith(".")) continue;
const namespaceName = e2.name;
const stack: string[] = [];
{
const path = newDir + "/" + namespaceName + "/";
stack.push(path);
try {
await Deno.stat(path + "folder.meta.json");
folderCandidates.push({
namespaceName,
path: path + "folder.meta.json",
});
} catch {}
}
while (stack.length > 0) {
const dir2 = stack.pop()!;
for await (const e3 of Deno.readDir(dir2)) {
if (e3.isFile) {
if (e3.name === "folder.meta.json") continue;
normalCandidates.push({
path: dir2 + e3.name,
namespaceKind: e.name == "g"
? "group"
: e.name == "u"
? "user"
: "folder",
namespaceName: namespaceName,
});
} else {
stack.push(dir2 + e3.name + "/");
}
}
}
}
}
} else {
console.log(
colors.yellow(
"Including organizational folder " + e.name + " in push!",
),
);
const { normal, resourceTypes, folders } =
await pushRawFindCandidateFiles(
path.join(dir, e.name),
);
normalCandidates.push(...normal);
resourceTypeCandidates.push(...resourceTypes);
folderCandidates.push(...folders);
}
} else {
// handle root files
if (e.name.endsWith(".resource-type.json")) {
resourceTypeCandidates.push({
path: dir + (dir.endsWith("/") ? "" : "/") + e.name,
});
}
}
}
return {
normal: normalCandidates,
folders: folderCandidates,
resourceTypes: resourceTypeCandidates,
};
}
async function pushRaw(opts: GlobalOptions, dir?: string) {
dir = dir ?? Deno.cwd();
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
console.log(colors.blue("Searching Directory..."));
const { normal, resourceTypes, folders } = await pushRawFindCandidateFiles(
dir,
);
console.log(
colors.blue(
"Found " + (normal.length + resourceTypes.length + folders.length) +
" candidates",
),
);
for (const resourceType of resourceTypes) {
const fileName = resourceType.path.substring(
resourceType.path.lastIndexOf("/") + 1,
);
const fileNameParts = fileName.split(".");
// invalid file names, like my.cool.script.script.json. Not valid.
if (fileNameParts.length != 3) {
console.log(
colors.yellow("invalid file name found at " + resourceType.path),
);
continue;
}
// filter out non-json files. Note that we filter out script contents above, so this is really an error.
if (fileNameParts.at(-1) != "json") {
console.log(colors.yellow("non-JSON file found at " + resourceType.path));
continue;
}
console.log("pushing resource type " + fileNameParts.at(-3)!);
await decoverto.type(ResourceTypeFile).rawToInstance(
await Deno.readTextFile(resourceType.path),
).push(workspace.workspaceId, fileNameParts.at(-3)!);
}
for (const folder of folders) {
await decoverto.type(FolderFile).plainToInstance(
JSON.parse(await Deno.readTextFile(folder.path)),
).push(
workspace.workspaceId,
"f/" + folder.namespaceName,
);
}
for (const candidate of normal) {
// full file name. No leading /. includes .type.json
const fileName = candidate.path.substring(
candidate.path.lastIndexOf("/") + 1,
);
// figure out just the path after ...../u|g/username|group/ (in extra dir)
const dirParts = candidate.path.split("/").filter((x) => x.length > 0);
// TODO: check version for folder
const gIndex = dirParts.findIndex((x) => x == "u" || x == "g" || x == "f");
const extraDir = dirParts.slice(gIndex + 2, -1).join("/");
// file name parts has .json (hopefully) at -1, type at -2, and the actual name at -3. Dots in names are not allowed.
const fileNameParts = fileName.split(".");
// filter out script content files
if (
fileNameParts.at(-1) == "ts" ||
fileNameParts.at(-1) == "py" ||
fileNameParts.at(-1) == "go"
) {
// probably part of a script. Silent ignore.
continue;
}
// invalid file names, like my.cool.script.script.json. Not valid.
if (fileNameParts.length != 3) {
console.log(
colors.yellow("invalid file name found at " + candidate.path),
);
continue;
}
// filter out non-json files. Note that we filter out script contents above, so this is really an error.
if (fileNameParts.at(-1) != "json") {
console.log(colors.yellow("non-JSON file found at " + candidate.path));
continue;
}
// get the type & filter it for valid ones.
const type = fileNameParts.at(-2);
if (type == "resource-type") {
console.log(
colors.yellow(
"Found resource type file at " +
candidate.path +
" this appears to be inside a path folder. Resource types are not addressed by path. Place them at the root or inside only an organizational folder. Ignoring this file!",
),
);
continue;
}
if (
type != "flow" &&
type != "resource" &&
type != "script" &&
type != "variable"
) {
console.log(
colors.yellow(
"file with invalid type " + type + " found at " + candidate.path,
),
);
continue;
}
// create the remotePath for the API
const remotePath = (candidate.namespaceKind === "group"
? "g/"
: (candidate.namespaceKind === "user" ? "u/" : "f/")) +
candidate.namespaceName +
"/" +
(extraDir.length > 0 ? extraDir + "/" : "") +
fileNameParts.at(-3);
console.log("pushing " + type + " to " + remotePath);
const typed = inferTypeFromPath(
candidate.path,
JSON.parse(await Deno.readTextFile(candidate.path)),
);
if (typed instanceof ResourceTypeFile || typed instanceof FolderFile) {
throw new Error(
"Resource Types and Folders should be filtered out at this point!",
);
} else if (typed instanceof ScriptFile) {
let contentPath: string;
try {
contentPath = await findContentFile(candidate.path);
} catch (e) {
console.log(colors.red(e.toString()));
continue;
}
await pushScript(
candidate.path,
contentPath,
workspace.workspaceId,
remotePath,
);
} else {
typed.push(workspace.workspaceId, remotePath);
}
}
console.log(colors.underline.bold.green("Successfully Pushed all files."));
}
const command = new Command()
.command("init")
.description(
"Initialize this folder as sync root for the currently selected workspace & remote.",
)
.action(init as any)
.command("add")
.description("Add a local file for tracking")
.arguments("<path:string>")
.action(add as any)
.command("pull")
.description("Pull any remote changes and apply them locally")
.option("--raw", "Pull without using state.")
.option("--raw-override", "Always override local files with remote.", {
depends: ["raw"],
})
.action(pull as any)
.command("push")
.description("Push any local changes and apply them remotely")
.option("--raw", "Push without using state.")
.action(push as any);
export default command;

10
cli/tmp/f/my_folder/folder.meta.json
View File

@@ -1,10 +0,0 @@
{
"workspace_id": "admins",
"name": "my_folder",
"display_name": "my_folder",
"owners": [],
"extra_perms": {
"u/test": true,
"u/admin@windmill.dev": false
}
}

12
cli/tmp/u/admin/hub_sync.script.json
View File

@@ -1,12 +0,0 @@
{
"summary": "Syncronize Hub Resource types with starter workspace",
"description": "Basic administrative script to sync latest resource types from hub. Recommended to run at least once. On a schedule by default.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {},
"required": [],
"type": "object"
},
"is_template": false,
"lock": []
}

13
cli/tmp/u/admin/hub_sync.ts
View File

@@ -1,13 +0,0 @@
import wmill from "https://deno.land/x/wmill@v1.55.0/main.ts";
export async function main() {
await run(
"workspace", "add", "__automation", "starter", Deno.env.get("WM_BASE_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
await run("hub", "pull");
}
async function run(...cmd: string[]) {
console.log("Running \"" + cmd.join(' ') + "\"");
await wmill.parse(cmd);
}

133
cli/types.ts
View File

@@ -1,4 +1,137 @@
import { decoverto } from "./decoverto.ts";
import { FlowFile } from "./flow.ts";
import { ResourceTypeFile } from "./resource-type.ts";
import { ResourceFile } from "./resource.ts";
import { ScriptFile } from "./script.ts";
import { VariableFile } from "./variable.ts";
import { path } from "./deps.ts";
import { FolderFile } from "./folder.ts";
// TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
// Then combine those with diffing, which then gives the new push impl
export interface Resource {
push(workspace: string, remotePath: string): Promise<void>;
}
export interface PushDiffs {
pushDiffs(
workspace: string,
remotePath: string,
diffs: Difference[],
): Promise<void>;
}
export interface DifferenceCreate {
type: "CREATE";
path: (string | number)[];
value: any;
}
export interface DifferenceRemove {
type: "REMOVE";
path: (string | number)[];
oldValue: any;
}
export interface DifferenceChange {
type: "CHANGE";
path: (string | number)[];
value: any;
oldValue: any;
}
export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;
export function setValueByPath(
obj: any,
path: (string | number)[],
value: any,
) {
let i;
let lastObj = undefined;
for (i = 0; i < path.length - 1; i++) {
if (!obj) {
let oldNewObj;
if (typeof path[i] === "number") {
oldNewObj = [];
} else {
oldNewObj = {};
}
lastObj[path[i - 1]] = oldNewObj;
obj = oldNewObj;
}
lastObj = obj;
obj = obj[path[i]];
}
if (!obj) {
let oldNewObj;
if (typeof path[i] === "number") {
oldNewObj = [];
} else {
oldNewObj = {};
}
lastObj[path[i - 1]] = oldNewObj;
obj = oldNewObj;
}
obj[path[i]] = value;
}
export type GlobalOptions = {
workspace: string | undefined;
token: string | undefined;
};
export function inferTypeFromPath(
p: string,
obj: any,
):
| ScriptFile
| VariableFile
| FlowFile
| ResourceFile
| ResourceTypeFile
| FolderFile {
const typeEnding = getTypeStrFromPath(p);
if (typeEnding === "folder") {
return decoverto.type(FolderFile).plainToInstance(obj);
} else if (typeEnding === "script") {
return decoverto.type(ScriptFile).plainToInstance(obj);
} else if (typeEnding === "variable") {
return decoverto.type(VariableFile).plainToInstance(obj);
} else if (typeEnding === "flow") {
return decoverto.type(FlowFile).plainToInstance(obj);
} else if (typeEnding === "resource") {
return decoverto.type(ResourceFile).plainToInstance(obj);
} else if (typeEnding === "resource-type") {
return decoverto.type(ResourceTypeFile).plainToInstance(obj);
} else {
throw new Error("infer type unreachable");
}
}
export function getTypeStrFromPath(
p: string,
): "script" | "variable" | "flow" | "resource" | "resource-type" | "folder" {
const parsed = path.parse(p);
if (parsed.ext !== ".json") {
throw new Error(
"Cannot infer type of non-json file " + JSON.stringify(parsed),
);
}
if (parsed.name === "folder.meta") {
return "folder";
}
const typeEnding = parsed.name.split(".").at(-1);
if (
typeEnding === "script" || typeEnding === "variable" ||
typeEnding === "flow" || typeEnding === "resource" ||
typeEnding === "resource-type"
) {
return typeEnding;
} else {
throw new Error("Could not infer type of path " + JSON.stringify(parsed));
}
}

180
cli/variable.ts
View File

@@ -1,7 +1,22 @@
// deno-lint-ignore-file no-explicit-any
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import { GlobalOptions } from "./types.ts";
import { colors, Command, Table, VariableService } from "./deps.ts";
import {
Difference,
GlobalOptions,
PushDiffs,
Resource,
setValueByPath,
} from "./types.ts";
import {
colors,
Command,
EditVariable,
ListableVariable,
microdiff,
Table,
VariableService,
} from "./deps.ts";
import { decoverto, model, property } from "./decoverto.ts";
async function list(opts: GlobalOptions) {
const workspace = await resolveWorkspace(opts);
@@ -26,13 +41,100 @@ async function list(opts: GlobalOptions) {
.render();
}
type VariableFile = {
@model()
export class VariableFile implements Resource, PushDiffs {
@property(() => String)
value: string;
@property(() => Boolean)
is_secret: boolean;
@property(() => String)
description: string;
@property(() => Number)
account?: number;
@property(() => Boolean)
is_oauth?: boolean;
};
constructor(value: string, is_secret: boolean, description: string) {
this.value = value;
this.is_secret = is_secret;
this.description = description;
}
async pushDiffs(
workspace: string,
remotePath: string,
diffs: Difference[],
): Promise<void> {
if (await VariableService.existsVariable({ workspace, path: remotePath })) {
console.log(
colors.bold.yellow(
`Applying ${diffs.length} diffs to existing variable...`,
),
);
const changeset: EditVariable = {};
for (const diff of diffs) {
if (
diff.type !== "REMOVE" &&
(
diff.path.length !== 1 ||
!["path", "value", "is_secret", "description"].includes(
diff.path[0] as string,
)
)
) {
throw new Error("Invalid variable diff with path " + diff.path);
}
if (diff.type === "CREATE" || diff.type === "CHANGE") {
setValueByPath(changeset, diff.path, diff.value);
} else if (diff.type === "REMOVE") {
setValueByPath(changeset, diff.path, null);
}
}
const hasChanges = Object.values(changeset).some((v) =>
v !== null && typeof v !== "undefined"
);
if (!hasChanges) {
console.log(colors.yellow("! Skipping empty changeset"));
return;
}
await VariableService.updateVariable({
workspace,
path: remotePath,
requestBody: changeset,
});
} else {
console.log(colors.yellow("Creating new variable..."));
await VariableService.createVariable({
workspace,
requestBody: {
path: remotePath,
description: this.description,
is_secret: this.is_secret,
value: this.value,
account: this.account,
is_oauth: this.is_oauth,
},
});
}
}
async push(workspace: string, remotePath: string): Promise<void> {
let existing: ListableVariable | undefined;
try {
existing = await VariableService.getVariable({
workspace: workspace,
path: remotePath,
});
} catch {
existing = undefined;
}
await this.pushDiffs(
workspace,
remotePath,
microdiff(existing ?? {}, this, { cyclesFix: false }),
);
}
}
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
const workspace = await resolveWorkspace(opts);
@@ -58,72 +160,10 @@ export async function pushVariable(
filePath: string,
remotePath: string,
) {
const data: VariableFile = JSON.parse(await Deno.readTextFile(filePath));
if (await VariableService.existsVariable({ workspace, path: remotePath })) {
const existing = await VariableService.getVariable({
workspace: workspace,
path: remotePath,
});
if (existing.is_oauth != data.is_oauth) {
console.log(
colors.red.underline.bold(
"Remote variable at " +
remotePath +
" exists & has a different oauth state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
),
);
return;
}
if (existing.account != data.account) {
console.log(
colors.red.underline.bold(
"Remote variable at " +
remotePath +
" exists & has a different account state. This cannot be updated. If you wish to do this anyways, consider deleting the remote resource.",
),
);
return;
}
if (existing.is_secret && !data.is_secret) {
console.log(
colors.red.underline.bold(
"Remote variable at " +
remotePath +
" exists & is secret. Variables cannot be updated to be no longer secret. If you wish to do this anyways, consider deleting the remote resource.",
),
);
return;
}
const actual_secret = data.is_secret ? true : undefined;
console.log(colors.yellow("Updating existing variable..."));
await VariableService.updateVariable({
workspace,
path: remotePath,
requestBody: {
description: data.description,
is_secret: actual_secret,
path: remotePath,
value: data.value,
},
});
} else {
console.log(colors.yellow("Creating new variable..."));
await VariableService.createVariable({
workspace,
requestBody: {
path: remotePath,
description: data.description,
is_secret: data.is_secret,
value: data.value,
account: data.account,
is_oauth: data.is_oauth,
},
});
}
const data = decoverto.type(VariableFile).rawToInstance(
await Deno.readTextFile(filePath),
);
await data.push(workspace, remotePath);
}
const command = new Command()

60
cli/workspace.ts
View File

@@ -11,14 +11,31 @@ import {
Table,
WorkspaceService,
} from "./deps.ts";
import { requireLogin } from "./context.ts";
import { decoverto, model, property } from "./decoverto.ts";
export type Workspace = {
@model()
export class Workspace {
@property(() => String)
remote: string;
@property(() => String)
workspaceId: string;
@property(() => String)
name: string;
@property(() => String)
token: string;
};
constructor(
remote: string,
workspaceId: string,
name: string,
token: string,
) {
this.remote = remote;
this.workspaceId = workspaceId;
this.name = name;
this.token = token;
}
}
function makeWorkspaceStream(
readable: ReadableStream<Uint8Array>,
@@ -33,7 +50,9 @@ function makeWorkspaceStream(
if (line.length <= 2) {
return;
}
controller.enqueue(JSON.parse(line) as Workspace);
const workspace = decoverto.type(Workspace).rawToInstance(line);
workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
controller.enqueue(workspace);
} catch {
/* ignore */
}
@@ -42,14 +61,17 @@ function makeWorkspaceStream(
);
}
async function allWorkspaces(): Promise<Workspace[]> {
try {
const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
write: false,
read: true,
});
const workspaceStream = makeWorkspaceStream(file.readable);
export async function getWorkspaceStream() {
const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
write: false,
read: true,
});
return makeWorkspaceStream(file.readable);
}
export async function allWorkspaces(): Promise<Workspace[]> {
try {
const workspaceStream = await getWorkspaceStream();
const workspaces: Workspace[] = [];
for await (const workspace of workspaceStream) {
workspaces.push(workspace);
@@ -87,8 +109,7 @@ export async function getActiveWorkspace(
export async function getWorkspaceByName(
workspaceName: string,
): Promise<Workspace | undefined> {
const file = await Deno.open((await getRootStore()) + "remotes.ndjson");
const workspaceStream = makeWorkspaceStream(file.readable);
const workspaceStream = await getWorkspaceStream();
for await (const workspace of workspaceStream) {
if (workspace.name === workspaceName) {
return workspace;
@@ -185,6 +206,7 @@ export async function add(
remote = new URL(await Input.prompt("Enter the Remote URL")).toString();
}
}
remote = new URL(remote).toString(); // add trailing slash in all cases!
let token = await tryGetLoginInfo(opts);
while (!token) {
@@ -192,7 +214,10 @@ export async function add(
}
if (opts.create) {
setClient(token, remote.endsWith('/') ? remote.substring(0, remote.length - 1) : remote);
setClient(
token,
remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote,
);
if (
!await WorkspaceService.existsWorkspace({
@@ -224,6 +249,7 @@ export async function add(
}
export async function addWorkspace(workspace: Workspace) {
workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
const file = await Deno.open((await getRootStore()) + "remotes.ndjson", {
append: true,
write: true,
@@ -234,7 +260,7 @@ export async function addWorkspace(workspace: Workspace) {
file.close();
}
async function remove(_opts: GlobalOptions, name: string) {
export async function removeWorkspace(name: string) {
const orgWorkspaces = await allWorkspaces();
await Deno.writeTextFile(
(await getRootStore()) + "remotes.ndjson",
@@ -243,6 +269,10 @@ async function remove(_opts: GlobalOptions, name: string) {
.map((x) => JSON.stringify(x))
.join("\n"),
);
}
async function remove(_opts: GlobalOptions, name: string) {
await removeWorkspace(name);
console.log(colors.green.underline("Succesfully removed workspace!"));
}