kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: use windmill-utils-internal for cli (#6297)

Browse Source 
* add utils package

* naming

* cleaning

* simplify assignPath

* rename old files

* same for locks

* create on confirm

* default true

* use replaceinlinescripts from utils

* use extractscriptfromflows

* make it compile

* cleaning

* use argsigtojson

* fix

* fix missing await

* cleaner

* cleaning

* cleaning

* use in frontend

* add docs

* testing

* remove log

* use autogenerated types

* remove old

* fix

* cleaning

* adapt usage

* draft

* better build script

* fix build

* revert to default creation

* add docs

* remove and rename

* make everything work

* add await

* only if not installed

* add vs code setting

* add to publish action

* fix bc

* safer use of sep

* fix

* do not rename on push

* no publish on release

* use published package on frontend

* nit
This commit is contained in:
centdix
2025-07-31 15:23:39 +02:00
committed by GitHub
parent 9457d64266
commit 5b0ea4d2de
24 changed files with 511 additions and 2165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -7,7 +7,7 @@ CaddyfileRemoteMalo
*.swp
**/.idea/
.direnv
.vscode
/.vscode
.dev-docker-wrapper*
backend/.minio-data
.aider*

3
cli/.gitignore vendored
View File

@@ -1,2 +1,3 @@
npm/
gen/
gen/
node_modules/

42
cli/build-mac.sh Executable file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# Generate client files
./gen_wm_client-mac.sh
# Generate utils client files
./windmill-utils-internal/gen_wm_client-mac.sh
# Function to add .ts extensions to relative imports
add_ts_extensions() {
find windmill-utils-internal/src -name "*.ts" -type f | while read -r file; do
# Create backup of original
cp "$file" "$file.orig"
# Add .ts to relative imports that don't already have extensions
gsed -E \
-e 's/(from[[:space:]]+["'"'"'])(\.[^"'"'"']*[^./][^"'"'"']*)(["'"'"'])/\1\2.ts\3/g' \
-e 's/(import[[:space:]]+["'"'"'])(\.[^"'"'"']*[^./][^"'"'"']*)(["'"'"'])/\1\2.ts\3/g' \
"$file.orig" > "$file"
done
}
# Function to revert to original files
revert_extensions() {
find windmill-utils-internal/src -name "*.orig" -type f | while read -r backup_file; do
original_file="${backup_file%.orig}"
mv "$backup_file" "$original_file"
done
}
# Set up trap to ensure cleanup happens on exit, error, or interruption
trap 'echo "Cleaning up..."; revert_extensions' EXIT ERR INT TERM
# Add .ts extensions for Deno
echo "Adding .ts extensions for Deno build..."
add_ts_extensions
# Run dnt
echo "Running dnt..."
deno run -A dnt.ts
echo "Build complete!"

43
cli/build.sh
View File

@@ -1,4 +1,45 @@
#!/usr/bin/env bash
# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`.
# Set script to exit on any error
set -e
# Generate client files
./gen_wm_client.sh
# Generate utils client files
./windmill-utils-internal/gen_wm_client.sh
# Function to add .ts extensions to relative imports
add_ts_extensions() {
find windmill-utils-internal/src -name "*.ts" -type f | while read -r file; do
# Create backup of original
cp "$file" "$file.orig"
# Add .ts to relative imports that don't already have extensions
gsed -E \
-e 's/(from[[:space:]]+["'"'"'])(\.[^"'"'"']*[^./][^"'"'"']*)(["'"'"'])/\1\2.ts\3/g' \
-e 's/(import[[:space:]]+["'"'"'])(\.[^"'"'"']*[^./][^"'"'"']*)(["'"'"'])/\1\2.ts\3/g' \
"$file.orig" > "$file"
done
}
# Function to revert to original files
revert_extensions() {
find windmill-utils-internal/src -name "*.orig" -type f | while read -r backup_file; do
original_file="${backup_file%.orig}"
mv "$backup_file" "$original_file"
done
}
# Set up trap to ensure cleanup happens on exit, error, or interruption
trap 'echo "Cleaning up..."; revert_extensions' EXIT ERR INT TERM
# Add .ts extensions for Deno
echo "Adding .ts extensions for Deno build..."
add_ts_extensions
# Run dnt
echo "Running dnt..."
deno run -A dnt.ts
echo "Build complete!"

1767
cli/deno.lock generated
View File

File diff suppressed because it is too large Load Diff

14
cli/dev.ts
View File

@@ -22,7 +22,8 @@ import {
import { exts, findGlobalDeps, removeExtensionToPath } from "./script.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { OpenFlow } from "./gen/types.gen.ts";
import { FlowFile, replaceInlineScripts } from "./flow.ts";
import { FlowFile } from "./flow.ts";
import { replaceInlineScripts } from "./windmill-utils-internal/src/inline-scripts/replacer.ts";
import { parseMetadataFile } from "./metadata.ts";
const PORT = 3001;
@@ -74,7 +75,16 @@ async function dev(opts: GlobalOptions & SyncOptions) {
const localFlow = (await yamlParseFile(
localPath + "flow.yaml"
)) as FlowFile;
replaceInlineScripts(localFlow.value.modules, localPath, undefined);
await replaceInlineScripts(
localFlow.value.modules,
async (path: string) => await Deno.readTextFile(localPath + path),
log,
localPath,
SEP,
undefined,
(path: string, newPath: string) => Deno.renameSync(path, newPath),
(path: string) => Deno.removeSync(path),
);
currentLastEdit = {
type: "flow",
flow: localFlow,

64
cli/flow.ts
View File

@@ -8,11 +8,12 @@ import { requireLogin } from "./auth.ts";
import { resolveWorkspace, validatePath } from "./context.ts";
import { resolve, track_job } from "./script.ts";
import { defaultFlowDefinition } from "./bootstrap/flow_bootstrap.ts";
import { blueColor, generateFlowLockInternal } from "./metadata.ts";
import { generateFlowLockInternal } from "./metadata.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "./conf.ts";
import { FSFSElement, elementsToMap, ignoreF } from "./sync.ts";
import { readInlinePathSync } from "./utils.ts";
import { Flow, FlowModule } from "./gen/types.gen.ts";
import { Flow } from "./gen/types.gen.ts";
import { replaceInlineScripts } from "./windmill-utils-internal/src/inline-scripts/replacer.ts";
export interface FlowFile {
summary: string;
@@ -23,55 +24,6 @@ export interface FlowFile {
const alreadySynced: string[] = [];
export function replaceInlineScripts(
modules: FlowModule[],
localPath: string,
removeLocks: string[] | undefined
) {
modules.forEach((m, i) => {
if (!m.value) {
throw Error(
`Module value is undefined for flow module ${i} in ${localPath}`
);
return;
}
if (m.value.type == "rawscript") {
if (m.value.content.startsWith("!inline")) {
const path = m.value.content.split(" ")[1];
m.value.content = Deno.readTextFileSync(localPath + path);
const lock = m.value.lock;
if (removeLocks && removeLocks.includes(path)) {
m.value.lock = undefined;
} else if (
lock &&
typeof lock == "string" &&
lock.trimStart().startsWith("!inline ")
) {
const path = lock.split(" ")[1];
try {
m.value.lock = readInlinePathSync(localPath + path);
} catch {
log.error(`Lock file ${path} not found`);
}
}
}
} else if (m.value.type == "forloopflow") {
replaceInlineScripts(m.value.modules, localPath, removeLocks);
} else if (m.value.type == "whileloopflow") {
replaceInlineScripts(m.value.modules, localPath, removeLocks);
} else if (m.value.type == "branchall") {
m.value.branches.forEach((b) =>
replaceInlineScripts(b.modules, localPath, removeLocks)
);
} else if (m.value.type == "branchone") {
m.value.branches.forEach((b) =>
replaceInlineScripts(b.modules, localPath, removeLocks)
);
replaceInlineScripts(m.value.default, localPath, removeLocks);
}
});
}
export async function pushFlow(
workspace: string,
remotePath: string,
@@ -98,7 +50,13 @@ export async function pushFlow(
}
const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile;
replaceInlineScripts(localFlow.value.modules, localPath, undefined);
await replaceInlineScripts(
localFlow.value.modules,
async (path: string) => await Deno.readTextFile(localPath + path),
log,
localPath,
SEP,
);
if (flow) {
if (isSuperset(localFlow, flow)) {

28
cli/gen_wm_client-mac.sh Executable file
View File

@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -eou pipefail
script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# only install gnu-sed if not already installed
if ! command -v gsed &> /dev/null; then
brew install gnu-sed
fi
rm -rf "${script_dirpath}/gen"
npx --yes @hey-api/openapi-ts@0.53.1 --input "${script_dirpath}/../backend/windmill-api/openapi.yaml" --output "${script_dirpath}/gen" --useOptions --client legacy/fetch --schemas false
cat <<EOF - gen/core/OpenAPI.ts > temp_file && mv temp_file gen/core/OpenAPI.ts
const getEnv = (key: string) => {
return Deno.env.get(key)
};
const baseUrl = getEnv("BASE_INTERNAL_URL") ?? getEnv("BASE_URL") ?? "http://localhost:8000";
const baseUrlApi = (baseUrl ?? '') + "/api";
EOF
gsed -i 's/WITH_CREDENTIALS: false/WITH_CREDENTIALS: true/g' gen/core/OpenAPI.ts
gsed -i 's/TOKEN: undefined/TOKEN: getEnv("WM_TOKEN")/g' gen/core/OpenAPI.ts
gsed -i "s/BASE: '\/api'/BASE: baseUrlApi/g" gen/core/OpenAPI.ts
find gen/ -name "*.ts" -exec gsed -i -E "s/(import.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;
find gen/ -name "*.ts" -exec gsed -i -E "s/(export.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;

5
cli/main.ts
View File

@@ -265,8 +265,9 @@ const command = new Command()
}
} catch (error) {
// If there's an error checking backend settings, just continue with defaults
const errorMessage = error instanceof Error ? error.message : String(error);
log.warn(
`Could not check backend for git-sync settings: ${error.message}`,
`Could not check backend for git-sync settings: ${errorMessage}`,
);
log.info("Continuing with default settings");
}
@@ -373,7 +374,7 @@ const command = new Command()
"upgrade",
new UpgradeCommand({
provider: new NpmProvider({ package: "windmill-cli" }),
}).error((e) => {
}).error((e: any) => {
log.error(e);
log.info(
"Try running with sudo and otherwise check the result of the command: npm uninstall windmill-cli && npm install -g windmill-cli",

250
cli/metadata.ts
View File

@@ -13,7 +13,6 @@ import {
defaultScriptMetadata,
} from "./bootstrap/script_bootstrap.ts";
import { Workspace } from "./workspace.ts";
import { SchemaProperty } from "./bootstrap/common.ts";
import {
languagesWithRawReqsSupport,
LanguageWithRawReqsSupport,
@@ -23,14 +22,15 @@ import { inferContentTypeFromFilePath } from "./script_common.ts";
import { GlobalDeps, exts, findGlobalDeps } from "./script.ts";
import {
FSFSElement,
extractInlineScriptsForFlows,
findCodebase,
newPathAssigner,
yamlOptions,
} from "./sync.ts";
import { generateHash, readInlinePathSync } from "./utils.ts";
import { SyncCodebase } from "./codebase.ts";
import { FlowFile, replaceInlineScripts } from "./flow.ts";
import { FlowFile } from "./flow.ts";
import { replaceInlineScripts } from "./windmill-utils-internal/src/inline-scripts/replacer.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "./windmill-utils-internal/src/inline-scripts/extractor.ts";
import { argSigToJsonSchemaType } from "./windmill-utils-internal/src/parse/parse-schema.ts";
import { getIsWin } from "./main.ts";
import { FlowValue } from "./gen/types.gen.ts";
@@ -172,10 +172,15 @@ export async function generateFlowLockInternal(
}
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
replaceInlineScripts(
await replaceInlineScripts(
flowValue.value.modules,
async (path: string) => await Deno.readTextFile(folder + SEP + path),
log,
folder + SEP!,
changedScripts
SEP,
changedScripts,
(path: string, newPath: string) => Deno.renameSync(path, newPath),
(path: string) => Deno.removeSync(path),
);
//removeChangedLocks
@@ -186,10 +191,7 @@ export async function generateFlowLockInternal(
rawReqs
);
const inlineScripts = extractInlineScriptsForFlows(
flowValue.value.modules,
newPathAssigner(opts.defaultTs ?? "bun")
);
const inlineScripts = extractInlineScriptsForFlows(flowValue.value.modules, {}, SEP);
inlineScripts
.filter((s) => s.path.endsWith(".lock"))
.forEach((s) => {
@@ -704,234 +706,6 @@ function sortObject(obj: any): any {
);
}
//copied straight fron frontend /src/utils/inferArgs.ts
export function argSigToJsonSchemaType(
t:
| string
| { resource: string | null }
| {
list:
| (
| string
| {
object: {
name?: string;
props?: { key: string; typ: any }[];
};
}
)
| { str: any }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| null;
}
| { dynselect: string }
| { str: string[] | null }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| {
oneof: {
label: string;
properties: { key: string; typ: any }[];
}[];
},
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: "" };
if (t === "int") {
newS.type = "integer";
} else if (t === "float") {
newS.type = "number";
} else if (t === "bool") {
newS.type = "boolean";
} else if (t === "email") {
newS.type = "string";
newS.format = "email";
} else if (t === "sql") {
newS.type = "string";
newS.format = "sql";
} else if (t === "yaml") {
newS.type = "string";
newS.format = "yaml";
} else if (t === "bytes") {
newS.type = "string";
newS.contentEncoding = "base64";
newS.originalType = "bytes";
} else if (t === "datetime") {
newS.type = "string";
newS.format = "date-time";
} else if (typeof t !== "string" && "oneof" in t) {
newS.type = "object";
if (t.oneof) {
newS.oneOf = t.oneof.map((obj) => {
const oldObjS =
oldS.oneOf?.find((o) => o?.title === obj.label) ?? undefined;
const properties: Record<string, any> = {};
for (const prop of obj.properties) {
if (oldObjS?.properties && prop.key in oldObjS?.properties) {
properties[prop.key] = oldObjS?.properties[prop.key];
} else {
properties[prop.key] = { description: "", type: "" };
}
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
return {
type: "object",
title: obj.label,
properties,
order: oldObjS?.order ?? undefined,
};
});
}
} else if (typeof t !== "string" && `object` in t) {
newS.type = "object";
if (t.object.name) {
newS.format = `resource-${t.object.name}`;
}
if (t.object.props) {
const properties: Record<string, any> = {};
for (const prop of t.object.props) {
if (oldS.properties && prop.key in oldS.properties) {
properties[prop.key] = oldS.properties[prop.key];
} else {
properties[prop.key] = { description: "", type: "" };
}
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
newS.properties = properties;
}
} else if (typeof t !== "string" && `str` in t) {
newS.type = "string";
if (t.str) {
newS.originalType = "enum";
newS.enum = t.str;
} else if (oldS.originalType == "string" && oldS.enum) {
newS.originalType = "string";
newS.enum = oldS.enum;
} else {
newS.originalType = "string";
newS.enum = undefined;
}
} else if (typeof t !== "string" && `resource` in t) {
newS.type = "object";
newS.format = `resource-${t.resource}`;
} else if (typeof t !== "string" && `dynselect` in t) {
newS.type = "object";
newS.format = `dynselect-${t.dynselect}`;
} else if (typeof t !== "string" && `list` in t) {
newS.type = "array";
if (t.list === "int" || t.list === "float") {
newS.items = { type: "number" };
newS.originalType = "number[]";
} else if (t.list === "bytes") {
newS.items = { type: "string", contentEncoding: "base64" };
newS.originalType = "bytes[]";
} else if (
t.list &&
typeof t.list == "object" &&
"str" in t.list &&
t.list.str
) {
newS.items = { type: "string", enum: t.list.str };
newS.originalType = "enum[]";
} else if (
t.list == "string" ||
(t.list && typeof t.list == "object" && "str" in t.list)
) {
newS.items = { type: "string", enum: oldS.items?.enum };
newS.originalType = "string[]";
} else if (
t.list &&
typeof t.list == "object" &&
"resource" in t.list &&
t.list.resource
) {
newS.items = {
type: "resource",
resourceType: t.list.resource as string,
};
newS.originalType = "resource[]";
} else if (
t.list &&
typeof t.list == "object" &&
"object" in t.list &&
t.list.object
) {
if (t.list.object.name) {
newS.format = `resource-${t.list.object.name}`;
}
if (t.list.object.props && t.list.object.props.length > 0) {
const properties: Record<string, any> = {};
for (const prop of t.list.object.props) {
properties[prop.key] = { description: "", type: "" };
argSigToJsonSchemaType(prop.typ, properties[prop.key]);
}
newS.items = { type: "object", properties: properties };
} else {
newS.items = { type: "object" };
}
newS.originalType = "record[]";
} else {
newS.items = { type: "object" };
newS.originalType = "object[]";
}
} else {
newS.type = "object";
}
const preservedFields = [
"description",
"pattern",
"min",
"max",
"currency",
"currencyLocale",
"multiselect",
"customErrorMessage",
"required",
"showExpr",
"password",
"order",
"dateFormat",
"title",
"placeholder",
];
preservedFields.forEach((field) => {
// @ts-ignore
if (oldS[field] !== undefined) {
// @ts-ignore
newS[field] = oldS[field];
}
});
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != "description") {
// @ts-ignore
delete oldS[prop];
}
}
} else if (
(oldS.format == "date" || oldS.format === "date-time") &&
newS.format == "string"
) {
newS.format = oldS.format;
} else if (newS.format == "date-time" && oldS.format == "date") {
newS.format = "date";
} else if (oldS.items?.type != newS.items?.type) {
delete oldS.items;
}
if (oldS.format && !newS.format) {
oldS.format = undefined
}
Object.assign(oldS, newS);
// if (sameItems && savedItems != undefined && savedItems.enum != undefined) {
// sendUserToast(JSON.stringify(savedItems))
// oldS.items = savedItems
// }
}
////////////////////////////////////////////////////////////////////////////////////////////
// end of refactoring TODO //
////////////////////////////////////////////////////////////////////////////////////////////

129
cli/sync.ts
View File

@@ -4,7 +4,6 @@ import {
colors,
Command,
Confirm,
Input,
Select,
ensureDir,
minimatch,
@@ -37,8 +36,8 @@ import {
} from "./script.ts";
import { handleFile } from "./script.ts";
import { deepEqual, isFileResource, Repository, selectRepository } from "./utils.ts";
import { SyncOptions, mergeConfigWithConfigFile, readConfigFile, getEffectiveSettings } from "./conf.ts";
import { deepEqual, isFileResource } from "./utils.ts";
import { SyncOptions, readConfigFile, getEffectiveSettings } from "./conf.ts";
import { Workspace } from "./workspace.ts";
import { removePathPrefix } from "./types.ts";
import { SyncCodebase, listSyncCodebases } from "./codebase.ts";
@@ -47,9 +46,10 @@ import {
generateScriptMetadataInternal,
readLockfile,
} from "./metadata.ts";
import { FlowModule, OpenFlow, RawScript } from "./gen/types.gen.ts";
import { OpenFlow } from "./gen/types.gen.ts";
import { pushResource } from "./resource.ts";
import { assignPath } from "./windmill-utils-internal/src/path-utils/path-assigner.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "./windmill-utils-internal/src/inline-scripts/extractor.ts";
// Merge CLI options with effective settings, preserving CLI flags as overrides
function mergeCliWithEffectiveOptions<T extends GlobalOptions & SyncOptions & { repository?: string }>(
@@ -310,56 +310,8 @@ export interface InlineScript {
content: string;
}
export function extractInlineScriptsForFlows(
modules: FlowModule[],
pathAssigner: PathAssigner
): InlineScript[] {
return modules.flatMap((m) => {
if (m.value.type == "rawscript") {
const [basePath, ext] = pathAssigner.assignPath(
m.summary,
m.value.language
);
const path = basePath + ext;
const content = m.value.content;
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path.replaceAll(SEP, "/");
const lock = m.value.lock;
if (lock && lock != "") {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath.replaceAll(SEP, "/");
r.push({ path: lockPath, content: lock });
}
return r;
} else if (m.value.type == "forloopflow") {
return extractInlineScriptsForFlows(m.value.modules, pathAssigner);
} else if (m.value.type == "branchall") {
return m.value.branches.flatMap((b) =>
extractInlineScriptsForFlows(b.modules, pathAssigner)
);
} else if (m.value.type == "whileloopflow") {
return extractInlineScriptsForFlows(m.value.modules, pathAssigner);
} else if (m.value.type == "branchone") {
return [
...m.value.branches.flatMap((b) =>
extractInlineScriptsForFlows(b.modules, pathAssigner)
),
...extractInlineScriptsForFlows(m.value.default, pathAssigner),
];
} else {
return [];
}
});
}
interface PathAssigner {
assignPath(summary: string | undefined, language: string): [string, string];
}
const INLINE_SCRIPT = "inline_script";
export function extractInlineScriptsForApps(
rec: any,
pathAssigner: PathAssigner
): InlineScript[] {
if (!rec) {
return [];
@@ -368,8 +320,7 @@ export function extractInlineScriptsForApps(
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "inlineScript" && typeof v == "object") {
const o: Record<string, any> = v as any;
const name = rec["name"];
const [basePath, ext] = pathAssigner.assignPath(name, o["language"]);
const [basePath, ext] = assignPath(rec["id"], o["language"]);
const r = [];
if (o["content"]) {
const content = o["content"];
@@ -389,69 +340,13 @@ export function extractInlineScriptsForApps(
}
return r;
} else {
return extractInlineScriptsForApps(v, pathAssigner);
return extractInlineScriptsForApps(v);
}
});
}
return [];
}
export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner {
let counter = 0;
const seen_names = new Set<string>();
function assignPath(
summary: string | undefined,
language: RawScript["language"] | "frontend" | "bunnative"
): [string, string] {
let name;
name = summary?.toLowerCase()?.replaceAll(" ", "_") ?? "";
let original_name = name;
if (name == "") {
original_name = INLINE_SCRIPT;
name = `${INLINE_SCRIPT}_0`;
}
while (seen_names.has(name)) {
counter++;
name = `${original_name}_${counter}`;
}
seen_names.add(name);
let ext;
if (language == "python3") ext = "py";
else if (language == defaultTs || language == "bunnative") ext = "ts";
else if (language == "bun") ext = "bun.ts";
else if (language == "deno") ext = "deno.ts";
else if (language == "go") ext = "go";
else if (language == "bash") ext = "sh";
else if (language == "powershell") ext = "ps1";
else if (language == "postgresql") ext = "pg.sql";
else if (language == "mysql") ext = "my.sql";
else if (language == "bigquery") ext = "bq.sql";
else if (language == "oracledb") ext = "odb.sql";
else if (language == "snowflake") ext = "sf.sql";
else if (language == "mssql") ext = "ms.sql";
else if (language == "graphql") ext = "gql";
else if (language == "nativets") ext = "native.ts";
else if (language == "frontend") ext = "frontend.js";
else if (language == "php") ext = "php";
else if (language == "rust") ext = "rs";
else if (language == "csharp") ext = "cs";
else if (language == "nu") ext = "nu";
else if (language == "ansible") ext = "playbook.yml";
else if (language == "java") ext = "java";
else if (language == "duckdb") ext = "duckdb.sql";
// for related places search: ADD_NEW_LANG
else ext = "no_ext";
return [`${name}.inline_script.`, ext];
}
return { assignPath };
}
function ZipFSElement(
zip: JSZip,
useYaml: boolean,
@@ -495,10 +390,7 @@ function ZipFSElement(
async *getChildren(): AsyncIterable<DynFSElement> {
if (kind == "flow") {
const flow: OpenFlow = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScriptsForFlows(
flow.value.modules,
newPathAssigner(defaultTs)
);
const inlineScripts = extractInlineScriptsForFlows(flow.value.modules, {}, SEP);
for (const s of inlineScripts) {
yield {
isDirectory: false,
@@ -522,10 +414,7 @@ function ZipFSElement(
};
} else if (kind == "app") {
const app = JSON.parse(await f.async("text"));
const inlineScripts = extractInlineScriptsForApps(
app?.["value"],
newPathAssigner(defaultTs)
);
const inlineScripts = extractInlineScriptsForApps(app?.["value"]);
for (const s of inlineScripts) {
yield {
isDirectory: false,

1
cli/windmill-utils-internal/.gitignore vendored
View File

@@ -18,7 +18,6 @@ src/gen/
.env.*.local
# IDE/Editor files
.vscode/
.idea/
*.swp
*.swo

3
cli/windmill-utils-internal/.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"deno.enable": false
}

9
cli/windmill-utils-internal/gen_wm_client-mac.sh
View File

@@ -1,6 +1,9 @@
#!/usr/bin/env bash
brew install gnu-sed
# only install gnu-sed if not already installed
if ! command -v gsed &> /dev/null; then
brew install gnu-sed
fi
script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
output_dirpath="${script_dirpath}/src/gen"
@@ -20,7 +23,3 @@ EOF
gsed -i 's/WITH_CREDENTIALS: false/WITH_CREDENTIALS: true/g' "${output_dirpath}/core/OpenAPI.ts"
gsed -i 's/TOKEN: undefined/TOKEN: getEnv("WM_TOKEN")/g' "${output_dirpath}/core/OpenAPI.ts"
gsed -i "s/BASE: '\/api'/BASE: baseUrlApi/g" "${output_dirpath}/core/OpenAPI.ts"
find "${output_dirpath}" -name "*.ts" -exec gsed -i -E "s/(import.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;
find "${output_dirpath}" -name "*.ts" -exec gsed -i -E "s/(export.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;

4
cli/windmill-utils-internal/gen_wm_client.sh
View File

@@ -18,7 +18,3 @@ EOF
sed -i 's/WITH_CREDENTIALS: false/WITH_CREDENTIALS: true/g' "${output_dirpath}/core/OpenAPI.ts"
sed -i 's/TOKEN: undefined/TOKEN: getEnv("WM_TOKEN")/g' "${output_dirpath}/core/OpenAPI.ts"
sed -i "s/BASE: '\/api'/BASE: baseUrlApi/g" "${output_dirpath}/core/OpenAPI.ts"
find "${output_dirpath}" -name "*.ts" -exec sed -i -E "s/(import.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;
find "${output_dirpath}" -name "*.ts" -exec sed -i -E "s/(export.*from[[:space:]]*['\"][^'\"]+)(['\"])/\1.ts\2/g" {} \;

11
cli/windmill-utils-internal/publish.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/bin/bash
set -eou pipefail
script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
args=${1:-}
rm -rf "${script_dirpath}/dist"
npm install
npm run build
npm publish ${args}

18
cli/windmill-utils-internal/src/inline-scripts/extractor.ts
View File

@@ -1,5 +1,4 @@
import { assignPath } from "../path-utils";
import { SEP } from "../constants";
import { assignPath } from "../path-utils/path-assigner";
import { FlowModule } from "../gen/types.gen";
/**
@@ -24,6 +23,7 @@ interface InlineScript {
export function extractInlineScripts(
modules: FlowModule[],
mapping: Record<string, string> = {},
separator: string = "/",
defaultTs?: "bun" | "deno"
): InlineScript[] {
return modules.flatMap((m) => {
@@ -36,28 +36,28 @@ export function extractInlineScripts(
const path = mapping[m.id] ?? basePath + ext;
const content = m.value.content;
const r = [{ path: path, content: content }];
m.value.content = "!inline " + path.replace(SEP, "/");
m.value.content = "!inline " + path.replaceAll(separator, "/");
const lock = m.value.lock;
if (lock && lock != "") {
const lockPath = basePath + "lock";
m.value.lock = "!inline " + lockPath.replace(SEP, "/");
m.value.lock = "!inline " + lockPath.replaceAll(separator, "/");
r.push({ path: lockPath, content: lock });
}
return r;
} else if (m.value.type == "forloopflow") {
return extractInlineScripts(m.value.modules, mapping, defaultTs);
return extractInlineScripts(m.value.modules, mapping, separator, defaultTs);
} else if (m.value.type == "branchall") {
return m.value.branches.flatMap((b) =>
extractInlineScripts(b.modules, mapping, defaultTs)
extractInlineScripts(b.modules, mapping, separator, defaultTs)
);
} else if (m.value.type == "whileloopflow") {
return extractInlineScripts(m.value.modules, mapping, defaultTs);
return extractInlineScripts(m.value.modules, mapping, separator, defaultTs);
} else if (m.value.type == "branchone") {
return [
...m.value.branches.flatMap((b) =>
extractInlineScripts(b.modules, mapping, defaultTs)
extractInlineScripts(b.modules, mapping, separator, defaultTs)
),
...extractInlineScripts(m.value.default, mapping, defaultTs),
...extractInlineScripts(m.value.default, mapping, separator, defaultTs),
];
} else {
return [];

64
cli/windmill-utils-internal/src/inline-scripts/replacer.ts
View File

@@ -1,4 +1,4 @@
import { FlowModule } from "../gen/types.gen.ts";
import { FlowModule } from "../gen/types.gen";
/**
* Replaces inline script references with actual file content from the filesystem.
@@ -13,33 +13,72 @@ import { FlowModule } from "../gen/types.gen.ts";
*/
export async function replaceInlineScripts(
modules: FlowModule[],
fileReader: (...args: any[]) => any,
fileReader: (path: string) => Promise<string>,
logger: {
info: (...args: any[]) => void,
error: (...args: any[]) => void,
info: (message: string) => void,
error: (message: string) => void,
} = {
info: () => {},
error: () => {},
},
localPath: string,
removeLocks?: string[]
separator: string = "/",
removeLocks?: string[],
renamer?: (path: string, newPath: string) => void,
deleter?: (path: string) => void
): Promise<void> {
await Promise.all(modules.map(async (module) => {
if (!module.value) {
throw new Error(`Module value is undefined for module ${module.id}`);
}
if (module.value.type === "rawscript") {
if (module.value.content.startsWith("!inline")) {
if (module.value.type === "rawscript" && module.value.content && module.value.content.startsWith("!inline")) {
const path = module.value.content.split(" ")[1];
const pathPrefix = path.split(".")[0];
const pathSuffix = path.split(".").slice(1).join(".");
// new path is the module id with the same suffix
const newPath = module.id + "." + pathSuffix;
try {
module.value.content = await fileReader(path);
} catch {
logger.error(`Script file ${path} not found`);
// try new path
try {
module.value.content = await fileReader(newPath);
} catch {
logger.error(`Script file ${newPath} not found`);
}
}
// rename the file if the prefix is different from the module id (fix old naming)
if (pathPrefix != module.id && renamer) {
logger.info(`Renaming ${path} to ${module.id}.${pathSuffix}`);
try {
renamer(localPath + path, localPath + module.id + "." + pathSuffix);
} catch {
logger.info(`Failed to rename ${path} to ${module.id}.${pathSuffix}`);
}
}
const lock = module.value.lock;
if (removeLocks && removeLocks.includes(path)) {
module.value.lock = undefined;
// delete the file if the prefix is different from the module id (fix old naming)
if (lock && lock != "") {
const path = lock.split(" ")[1];
const pathPrefix = path.split(".")[0];
if (pathPrefix != module.id && deleter) {
logger.info(`Deleting ${path}`);
try {
deleter(localPath + path);
} catch {
logger.error(`Failed to delete ${path}`);
}
}
}
} else if (
lock &&
typeof lock == "string" &&
@@ -47,23 +86,22 @@ export async function replaceInlineScripts(
) {
const path = lock.split(" ")[1];
try {
module.value.lock = await fileReader(path);
module.value.lock = await fileReader(path.replaceAll("/", separator));
} catch {
logger.error(`Lock file ${path} not found`);
}
}
}
} else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") {
await replaceInlineScripts(module.value.modules, fileReader, logger, localPath, removeLocks);
await replaceInlineScripts(module.value.modules, fileReader, logger, localPath, separator, removeLocks);
} else if (module.value.type === "branchall") {
await Promise.all(module.value.branches.map(async (branch) => {
await replaceInlineScripts(branch.modules, fileReader, logger, localPath, removeLocks);
await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks);
}));
} else if (module.value.type === "branchone") {
await Promise.all(module.value.branches.map(async (branch) => {
await replaceInlineScripts(branch.modules, fileReader, logger, localPath, removeLocks);
await replaceInlineScripts(branch.modules, fileReader, logger, localPath, separator, removeLocks);
}));
await replaceInlineScripts(module.value.default, fileReader, logger, localPath, removeLocks);
await replaceInlineScripts(module.value.default, fileReader, logger, localPath, separator, removeLocks);
}
}));
}

8
cli/windmill-utils-internal/tsconfig.json
View File

@@ -1,8 +1,8 @@
{
"compilerOptions": {
"target": "ES2020",
"target": "ES2022",
"module": "commonjs",
"lib": ["ES2020"],
"lib": ["ES2022"],
"declaration": true,
"outDir": "./dist",
"rootDir": "./src",
@@ -17,9 +17,7 @@
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"noEmit": true,
"allowImportingTsExtensions": true
"forceConsistentCasingInFileNames": true
},
"include": [
"src/**/*"

7
frontend/package-lock.json generated
View File

@@ -85,6 +85,7 @@
"windmill-parser-wasm-ts": "1.514.1",
"windmill-parser-wasm-yaml": "1.510.1",
"windmill-sql-datatype-parser-wasm": "1.318.0",
"windmill-utils-internal": "^1.0.0",
"xterm": "^5.3.0",
"xterm-readline": "^1.1.2",
"y-monaco": "^0.1.4",
@@ -12994,6 +12995,12 @@
"resolved": "https://registry.npmjs.org/windmill-sql-datatype-parser-wasm/-/windmill-sql-datatype-parser-wasm-1.318.0.tgz",
"integrity": "sha512-jlRw6abUJi4vDm+7xDSjhb7dvm4tC+lBXv0EEwn52Veadwcl5EB1yGHb9XVqQEfcYr9JU62xfJlN6DoGQmYE/g=="
},
"node_modules/windmill-utils-internal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.0.0.tgz",
"integrity": "sha512-S93XgzdM8WNmDt+MUqiBzyA/3GacZV3Wo9DxxYpAuDHvIP4s8c9zKBhNIZqB008i1opI4tTxB8SwhsYBhTaRuw==",
"license": "Apache 2.0"
},
"node_modules/word-wrap": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz",

1
frontend/package.json
View File

@@ -152,6 +152,7 @@
"windmill-parser-wasm-ts": "1.514.1",
"windmill-parser-wasm-yaml": "1.510.1",
"windmill-sql-datatype-parser-wasm": "1.318.0",
"windmill-utils-internal": "^1.0.0",
"xterm": "^5.3.0",
"xterm-readline": "^1.1.2",
"y-monaco": "^0.1.4",

2
frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte
View File

@@ -13,7 +13,7 @@
init(wasmUrl)
import { argSigToJsonSchemaType } from '$lib/inferArgSig'
import { argSigToJsonSchemaType } from 'windmill-utils-internal'
import SchemaForm from '$lib/components/SchemaForm.svelte'
import { untrack } from 'svelte'

2
frontend/src/lib/infer.ts
View File

@@ -39,7 +39,7 @@ import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.w
import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url'
import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url'
import { workspaceStore } from './stores.js'
import { argSigToJsonSchemaType } from './inferArgSig.js'
import { argSigToJsonSchemaType } from 'windmill-utils-internal'
import { type AssetWithAccessType } from './components/assets/lib.js'
const loadSchemaLastRun =

199
frontend/src/lib/inferArgSig.ts
View File

@@ -1,199 +0,0 @@
import type { SchemaProperty } from './common'
export function argSigToJsonSchemaType(
t:
| string
| { resource: string | null }
| {
list:
| (string | { name?: string; props?: { key: string; typ: any }[] })
| { str: any }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| null
}
| { dynselect: string }
| { str: string[] | null }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| {
oneof: {
label: string
properties: { key: string; typ: any }[]
}[]
},
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: '' }
if (t === 'int') {
newS.type = 'integer'
} else if (t === 'float') {
newS.type = 'number'
} else if (t === 'bool') {
newS.type = 'boolean'
} else if (t === 'email') {
newS.type = 'string'
newS.format = 'email'
} else if (t === 'sql') {
newS.type = 'string'
newS.format = 'sql'
} else if (t === 'yaml') {
newS.type = 'string'
newS.format = 'yaml'
} else if (t === 'bytes') {
newS.type = 'string'
newS.contentEncoding = 'base64'
newS.originalType = 'bytes'
} else if (t === 'datetime') {
newS.type = 'string'
newS.format = 'date-time'
} else if (typeof t !== 'string' && 'oneof' in t) {
newS.type = 'object'
if (t.oneof) {
newS.oneOf = t.oneof.map((obj) => {
const oldObjS = oldS.oneOf?.find((o) => o?.title === obj.label) ?? undefined
const properties: Record<string, any> = {}
for (const prop of obj.properties) {
if (oldObjS?.properties && prop.key in oldObjS?.properties) {
properties[prop.key] = oldObjS?.properties[prop.key]
} else {
properties[prop.key] = { description: '', type: '' }
}
argSigToJsonSchemaType(prop.typ, properties[prop.key])
}
return {
type: 'object',
title: obj.label,
properties,
order: oldObjS?.order ?? undefined
}
})
}
} else if (typeof t !== 'string' && `object` in t) {
newS.type = 'object'
if (t.object.name) {
newS.format = `resource-${t.object.name}`
}
if (t.object.props) {
const properties: Record<string, any> = {}
for (const prop of t.object.props) {
if (oldS.properties && prop.key in oldS.properties) {
properties[prop.key] = oldS.properties[prop.key]
} else {
properties[prop.key] = { description: '', type: '' }
}
argSigToJsonSchemaType(prop.typ, properties[prop.key])
}
newS.properties = properties
}
} else if (typeof t !== 'string' && `str` in t) {
newS.type = 'string'
if (t.str) {
newS.originalType = 'enum'
newS.enum = t.str
} else if (oldS.originalType == 'string' && oldS.enum) {
newS.originalType = 'string'
newS.enum = oldS.enum
} else {
newS.originalType = 'string'
newS.enum = undefined
}
} else if (typeof t !== 'string' && `resource` in t) {
newS.type = 'object'
newS.format = `resource-${t.resource}`
} else if (typeof t !== 'string' && `dynselect` in t) {
newS.type = 'object'
newS.format = `dynselect-${t.dynselect}`
} else if (typeof t !== 'string' && `list` in t) {
newS.type = 'array'
if (t.list === 'int' || t.list === 'float') {
newS.items = { type: 'number' }
newS.originalType = 'number[]'
} else if (t.list === 'bytes') {
newS.items = { type: 'string', contentEncoding: 'base64' }
newS.originalType = 'bytes[]'
} else if (t.list && typeof t.list == 'object' && 'str' in t.list && t.list.str) {
newS.items = { type: 'string', enum: t.list.str }
newS.originalType = 'enum[]'
} else if (t.list == 'string' || (t.list && typeof t.list == 'object' && 'str' in t.list)) {
newS.items = { type: 'string', enum: oldS.items?.enum }
newS.originalType = 'string[]'
} else if (t.list && typeof t.list == 'object' && 'resource' in t.list && t.list.resource) {
newS.items = {
type: 'resource',
resourceType: t.list.resource as string
}
newS.originalType = 'resource[]'
} else if (t.list && typeof t.list == 'object' && 'object' in t.list && t.list.object) {
if (t.list.object.name) {
newS.format = `resource-${t.list.object.name}`
}
if (t.list.object.props && t.list.object.props.length > 0) {
const properties: Record<string, any> = {}
for (const prop of t.list.object.props) {
properties[prop.key] = { description: '', type: '' }
argSigToJsonSchemaType(prop.typ, properties[prop.key])
}
newS.items = { type: 'object', properties: properties }
} else {
newS.items = { type: 'object' }
}
newS.originalType = 'record[]'
} else {
newS.items = { type: 'object' }
newS.originalType = 'object[]'
}
} else {
newS.type = 'object'
}
const preservedFields = [
'description',
'pattern',
'min',
'max',
'currency',
'currencyLocale',
'multiselect',
'customErrorMessage',
'required',
'showExpr',
'password',
'order',
'dateFormat',
'title',
'placeholder'
]
preservedFields.forEach((field) => {
// @ts-ignore
if (oldS[field] !== undefined) {
// @ts-ignore
newS[field] = oldS[field]
}
})
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != 'description') {
// @ts-ignore
delete oldS[prop]
}
}
} else if ((oldS.format == 'date' || oldS.format === 'date-time') && newS.format == 'string') {
newS.format = oldS.format
} else if (newS.format == 'date-time' && oldS.format == 'date') {
newS.format = 'date'
} else if (oldS.items?.type != newS.items?.type) {
delete oldS.items
}
if (oldS.format && !newS.format) {
oldS.format = undefined
}
Object.assign(oldS, newS)
// if (sameItems && savedItems != undefined && savedItems.enum != undefined) {
// sendUserToast(JSON.stringify(savedItems))
// oldS.items = savedItems
// }
}