kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(cli): specific items for file resource type (#6464)

Browse Source 
* cli file resource specific items

* improvement

* resource command + correct order of context

* no dynamic imports

* support trigger types for branch specific items

* also update trigger cli function to be branch aware

* hubscript path
This commit is contained in:
Alexander Petric
2025-08-27 14:49:46 +02:00
committed by GitHub
parent 2c29079fd5
commit dba5c95d6f
10 changed files with 278 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
cli/src/commands/resource/resource.ts
View File

@@ -11,6 +11,8 @@ import { colors, Command, log, SEP, Table } from "../../../deps.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { Resource } from "../../../gen/types.gen.ts";
import { readInlinePathSync } from "../../utils/utils.ts";
import { isBranchSpecificFile } from "../../core/specific_items.ts";
import { getCurrentGitBranch } from "../../utils/git.ts";
export interface ResourceFile {
value: any;
@@ -23,7 +25,8 @@ export async function pushResource(
workspace: string,
remotePath: string,
resource: ResourceFile | Resource | undefined,
localResource: ResourceFile
localResource: ResourceFile,
originalLocalPath?: string
): Promise<void> {
remotePath = removeType(remotePath, "resource");
try {
@@ -35,21 +38,49 @@ export async function pushResource(
// flow doesn't exist
}
if (localResource.value["content"]?.startsWith("!inline ")) {
const basePath = localResource.value["content"].split(" ")[1];
localResource.value["content"] = readInlinePathSync(basePath);
}
// Helper function to resolve inline content
const resolveInlineContent = async () => {
if (localResource.value["content"]?.startsWith("!inline ")) {
const basePath = localResource.value["content"].split(" ")[1];
// If we're processing a branch-specific metadata file, read from branch-specific resource file
let pathToRead = basePath;
if (originalLocalPath && isBranchSpecificFile(originalLocalPath)) {
const currentBranch = getCurrentGitBranch();
if (currentBranch) {
// Directly construct branch-specific resource file path
const resourcePathSegments = basePath.split(".");
if (resourcePathSegments.length >= 4 && resourcePathSegments[resourcePathSegments.length - 3] === "resource" && resourcePathSegments[resourcePathSegments.length - 2] === "file") {
const fileBaseParts = resourcePathSegments.slice(0, -3);
const fileExt = resourcePathSegments.slice(-3);
pathToRead = [...fileBaseParts, currentBranch, ...fileExt].join(".");
}
}
}
localResource.value["content"] = readInlinePathSync(pathToRead);
}
};
if (resource) {
if (isSuperset(localResource, resource)) {
return;
}
// Only resolve inline content if we're actually updating
await resolveInlineContent();
await wmill.updateResource({
workspace: workspace,
path: remotePath.replaceAll(SEP, "/"),
requestBody: { ...localResource },
});
} else {
// New resource - resolve inline content
await resolveInlineContent();
if (localResource.is_oauth) {
log.info(
colors.yellow(
@@ -89,7 +120,8 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
workspace.workspaceId,
remotePath,
undefined,
parseFromFile(filePath)
parseFromFile(filePath),
filePath // Pass the local file path for branch-specific inline content resolution
);
log.info(colors.bold.underline.green(`Resource ${remotePath} pushed`));
}

24
cli/src/commands/script/script.ts
View File

@@ -15,6 +15,8 @@ import {
} from "../../../deps.ts";
import { deepEqual } from "../../utils/utils.ts";
import * as wmill from "../../../gen/services.gen.ts";
import * as specificItems from "../../core/specific_items.ts";
import { getCurrentGitBranch } from "../../utils/git.ts";
import {
defaultScriptMetadata,
@@ -102,12 +104,24 @@ async function push(opts: PushOptions, filePath: string) {
export async function findResourceFile(path: string) {
const splitPath = path.split(".");
const contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json";
const contentBasePathYAML = splitPath[0] + "." + splitPath[1] + ".yaml";
let contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json";
let contentBasePathYAML = splitPath[0] + "." + splitPath[1] + ".yaml";
// Check for branch-specific metadata files first
const currentBranch = getCurrentGitBranch();
const candidates = [contentBasePathJSON, contentBasePathYAML];
if (currentBranch) {
// Add branch-specific candidates at the beginning (higher priority)
const branchSpecificJSON = specificItems.toBranchSpecificPath(contentBasePathJSON, currentBranch);
const branchSpecificYAML = specificItems.toBranchSpecificPath(contentBasePathYAML, currentBranch);
candidates.unshift(branchSpecificJSON, branchSpecificYAML);
}
const validCandidates = (
await Promise.all(
[contentBasePathJSON, contentBasePathYAML].map((x) => {
candidates.map((x) => {
return Deno.stat(x)
.catch(() => undefined)
.then((x) => x?.isFile)
@@ -580,7 +594,7 @@ export function filePathExtensionFromContentType(
return ".java";
} else if (language === "ruby") {
return ".rb";
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
} else {
throw new Error("Invalid language: " + language);
}
@@ -611,7 +625,7 @@ export const exts = [
".playbook.yml",
".java",
".rb"
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
];
export function removeExtensionToPath(path: string): string {

37
cli/src/commands/sync/sync.ts
View File

@@ -1930,11 +1930,21 @@ export async function push(
await Deno.readTextFile(resourceFilePath)
);
// For branch-specific resources, push to the base path on the workspace server
// This ensures branch-specific files are stored with their base names in the workspace
let serverPath = resourceFilePath;
const currentBranch = getCurrentGitBranch();
if (currentBranch && isBranchSpecificFile(resourceFilePath)) {
serverPath = fromBranchSpecificPath(resourceFilePath, currentBranch);
}
await pushResource(
workspace.workspaceId,
resourceFilePath,
serverPath,
undefined,
newObj
newObj,
resourceFilePath
);
if (stateTarget) {
await Deno.writeTextFile(stateTarget, change.after);
@@ -1945,6 +1955,12 @@ export async function push(
const oldObj = parseFromPath(change.path, change.before);
const newObj = parseFromPath(change.path, change.after);
// Check if this is a branch-specific item and get the original branch-specific path
let originalBranchSpecificPath: string | undefined;
if (specificItems && isSpecificItem(change.path, specificItems)) {
originalBranchSpecificPath = getBranchSpecificPath(change.path, specificItems);
}
await pushObj(
workspace.workspaceId,
change.path,
@@ -1952,7 +1968,8 @@ export async function push(
newObj,
opts.plainSecrets ?? false,
alreadySynced,
opts.message
opts.message,
originalBranchSpecificPath
);
if (stateTarget) {
@@ -1986,6 +2003,17 @@ export async function push(
);
}
const obj = parseFromPath(change.path, change.content);
// Determine the actual local file path for this change
// For branch-specific items, we read from branch-specific files but push to base server paths
let localFilePath = change.path;
if (specificItems && isSpecificItem(change.path, specificItems)) {
const branchSpecificPath = getBranchSpecificPath(change.path, specificItems);
if (branchSpecificPath) {
localFilePath = branchSpecificPath;
}
}
await pushObj(
workspace.workspaceId,
change.path,
@@ -1993,7 +2021,8 @@ export async function push(
obj,
opts.plainSecrets ?? false,
[],
opts.message
opts.message,
localFilePath // Pass the actual local file path
);
if (stateTarget) {

35
cli/src/commands/trigger/trigger.ts
View File

@@ -15,7 +15,10 @@ import {
isSuperset,
parseFromFile,
removeType,
TRIGGER_TYPES,
} from "../../types.ts";
import { fromBranchSpecificPath, isBranchSpecificFile } from "../../core/specific_items.ts";
import { getCurrentGitBranch } from "../../utils/git.ts";
import { requireLogin } from "../../core/auth.ts";
import { validatePath, resolveWorkspace } from "../../core/context.ts";
@@ -222,25 +225,29 @@ async function list(opts: GlobalOptions) {
}
function checkIfValidTrigger(kind: string | undefined): kind is TriggerType {
if (
kind &&
[
"http",
"websocket",
"kafka",
"nats",
"postgres",
"mqtt",
"sqs",
"gcp",
].includes(kind)
) {
if (kind && (TRIGGER_TYPES as readonly string[]).includes(kind)) {
return true;
} else {
return false;
}
}
function extractTriggerKindFromPath(filePath: string): string | undefined {
let pathToAnalyze = filePath;
// If this is a branch-specific file, convert it to the base path first
if (isBranchSpecificFile(filePath)) {
const currentBranch = getCurrentGitBranch();
if (currentBranch) {
pathToAnalyze = fromBranchSpecificPath(filePath, currentBranch);
}
}
// Now extract trigger type from the base path: "something.kafka_trigger.yaml" -> "kafka"
const triggerMatch = pathToAnalyze.match(/\.(\w+)_trigger\.yaml$/);
return triggerMatch ? triggerMatch[1] : undefined;
}
async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
@@ -256,7 +263,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
console.log(colors.bold.yellow("Pushing trigger..."));
const triggerKind = filePath.split(".")[1].split("_")[0];
const triggerKind = extractTriggerKindFromPath(filePath);
if (!checkIfValidTrigger(triggerKind)) {
throw new Error("Invalid trigger kind: " + triggerKind);
}

4
cli/src/core/conf.ts
View File

@@ -44,6 +44,7 @@ export interface SyncOptions {
commonSpecificItems?: {
variables?: string[];
resources?: string[];
triggers?: string[];
};
} & {
[branchName: string]: SyncOptions & {
@@ -54,6 +55,7 @@ export interface SyncOptions {
specificItems?: {
variables?: string[];
resources?: string[];
triggers?: string[];
};
};
};
@@ -62,6 +64,7 @@ export interface SyncOptions {
commonSpecificItems?: {
variables?: string[];
resources?: string[];
triggers?: string[];
};
} & {
[branchName: string]: SyncOptions & {
@@ -72,6 +75,7 @@ export interface SyncOptions {
specificItems?: {
variables?: string[];
resources?: string[];
triggers?: string[];
};
};
};

33
cli/src/core/context.ts
View File

@@ -109,15 +109,11 @@ async function tryResolveWorkspace(
return { isError: false, value: e };
}
const defaultWorkspace = await getActiveWorkspace(opts);
if (!defaultWorkspace) {
return {
isError: true,
error: colors.red.underline("No workspace given and no default set."),
};
}
return { isError: false, value: defaultWorkspace };
// Only check for explicit workspace, don't fallback to active workspace here
return {
isError: true,
error: colors.red.underline("No explicit workspace given."),
};
}
async function tryResolveBranchWorkspace(
@@ -259,6 +255,9 @@ async function tryResolveBranchWorkspace(
export async function resolveWorkspace(
opts: GlobalOptions
): Promise<Workspace> {
const cache = (opts as any).__secret_workspace;
if (cache) return cache;
if (opts.baseUrl) {
if (opts.workspace && opts.token) {
let normalizedBaseUrl: string;
@@ -328,20 +327,28 @@ export async function resolveWorkspace(
}
}
// Try explicit workspace flag first (should override branch-based resolution)
// Try explicit workspace flag first (highest priority)
const res = await tryResolveWorkspace(opts);
if (!res.isError) {
return res.value;
}
// Fall back to branch-based resolution if no explicit workspace
// Try branch-based resolution (medium priority)
const branchWorkspace = await tryResolveBranchWorkspace(opts);
if (branchWorkspace) {
(opts as any).__secret_workspace = branchWorkspace;
return branchWorkspace;
}
// If both failed, show the original error from explicit workspace resolution
log.info(colors.red.bold(res.error));
// Fall back to active workspace (lowest priority)
const activeWorkspace = await getActiveWorkspace(opts);
if (activeWorkspace) {
(opts as any).__secret_workspace = activeWorkspace;
return activeWorkspace;
}
// If everything failed, show error
log.info(colors.red.bold("No workspace given and no default set."));
return Deno.exit(-1);
}

128
cli/src/core/specific_items.ts
View File

@@ -1,10 +1,59 @@
import { minimatch } from "../../deps.ts";
import { getCurrentGitBranch, isGitRepository } from "../utils/git.ts";
import { isFileResource } from "../utils/utils.ts";
import { SyncOptions } from "./conf.ts";
import { TRIGGER_TYPES } from "../types.ts";
export interface SpecificItemsConfig {
variables?: string[];
resources?: string[];
triggers?: string[];
}
// Define all branch-specific file types (computed lazily)
function getBranchSpecificTypes() {
return {
variable: '.variable.yaml',
resource: '.resource.yaml',
// Generate trigger patterns from the list
...Object.fromEntries(
TRIGGER_TYPES.map(t => [`${t}_trigger`, `.${t}_trigger.yaml`])
)
} as const;
}
/**
* Check if a path ends with any trigger type
*/
function isTriggerFile(path: string): boolean {
return TRIGGER_TYPES.some(type => path.endsWith(`.${type}_trigger.yaml`));
}
/**
* Extract the file type suffix from a path
*/
function getFileTypeSuffix(path: string): string | null {
for (const [_, suffix] of Object.entries(getBranchSpecificTypes())) {
if (path.endsWith(suffix)) {
return suffix;
}
}
const resourceFileMatch = path.match(/(\\.resource\\.file\\..+)$/);
if (resourceFileMatch) {
return resourceFileMatch[1];
}
return null;
}
/**
* Build regex pattern for all supported yaml file types
*/
function buildYamlTypePattern(): string {
const basicTypes = ['variable', 'resource'];
const triggerTypes = TRIGGER_TYPES.map(t => `${t}_trigger`);
return `((${basicTypes.join('|')})|(${triggerTypes.join('|')}))`;
}
/**
@@ -39,6 +88,9 @@ export function getSpecificItemsForCurrentBranch(config: SyncOptions): SpecificI
if (commonItems?.resources) {
merged.resources = [...commonItems.resources];
}
if (commonItems?.triggers) {
merged.triggers = [...commonItems.triggers];
}
// Add branch-specific items (extending common items)
if (branchItems?.variables) {
@@ -47,6 +99,9 @@ export function getSpecificItemsForCurrentBranch(config: SyncOptions): SpecificI
if (branchItems?.resources) {
merged.resources = [...(merged.resources || []), ...branchItems.resources];
}
if (branchItems?.triggers) {
merged.triggers = [...(merged.triggers || []), ...branchItems.triggers];
}
return merged;
}
@@ -75,6 +130,21 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig
return specificItems.resources ? matchesPatterns(path, specificItems.resources) : false;
}
// Check for any trigger type
if (isTriggerFile(path)) {
return specificItems.triggers ? matchesPatterns(path, specificItems.triggers) : false;
}
// Check for resource files using the standard detection function
if (isFileResource(path)) {
// Extract the base path without the file extension to match against patterns
const basePathMatch = path.match(/^(.+?)\.resource\.file\./);
if (basePathMatch && specificItems.resources) {
const basePath = basePathMatch[1] + '.resource.yaml';
return matchesPatterns(basePath, specificItems.resources);
}
}
return false;
}
@@ -82,14 +152,24 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig
* Convert a base path to a branch-specific path
*/
export function toBranchSpecificPath(basePath: string, branchName: string): string {
// Extract the extension (e.g., ".variable.yaml" or ".resource.yaml")
const extensionMatch = basePath.match(/(\.(variable|resource)\.yaml)$/);
if (!extensionMatch) {
return basePath; // Return unchanged if no recognized extension
}
// Check for resource file pattern (e.g., .resource.file.ini)
const resourceFileMatch = basePath.match(/^(.+?)(\.resource\.file\..+)$/);
const extension = extensionMatch[1];
const pathWithoutExtension = basePath.substring(0, basePath.length - extension.length);
let extension: string;
let pathWithoutExtension: string;
if (resourceFileMatch) {
// Handle resource files
extension = resourceFileMatch[2];
pathWithoutExtension = resourceFileMatch[1];
} else {
const suffix = getFileTypeSuffix(basePath);
if (!suffix) {
return basePath;
}
extension = suffix;
pathWithoutExtension = basePath.substring(0, basePath.length - extension.length);
}
// Sanitize branch name to be filesystem-safe
const sanitizedBranchName = branchName.replace(/[\/\\:*?"<>|.]/g, '_');
@@ -108,17 +188,29 @@ export function toBranchSpecificPath(basePath: string, branchName: string): stri
export function fromBranchSpecificPath(branchSpecificPath: string, branchName: string): string {
// Sanitize branch name the same way as in toBranchSpecificPath
const sanitizedBranchName = branchName.replace(/[\/\\:*?"<>|.]/g, '_');
// Pattern: path.sanitizedBranchName.extension
const escapedBranchName = sanitizedBranchName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pattern = new RegExp(`\\.${escapedBranchName}(\\.(variable|resource)\\.yaml)$`);
const match = branchSpecificPath.match(pattern);
if (!match) {
// Check for resource file pattern first
const resourceFilePattern = new RegExp(`\\.${escapedBranchName}(\\.resource\\.file\\..+)$`);
const resourceFileMatch = branchSpecificPath.match(resourceFilePattern);
if (resourceFileMatch) {
const extension = resourceFileMatch[1];
const pathWithoutBranchAndExtension = branchSpecificPath.substring(
0,
branchSpecificPath.length - `.${sanitizedBranchName}${extension}`.length
);
return `${pathWithoutBranchAndExtension}${extension}`;
}
const yamlPattern = new RegExp(`\\.${escapedBranchName}(\\.${buildYamlTypePattern()}\\.yaml)$`);
const yamlMatch = branchSpecificPath.match(yamlPattern);
if (!yamlMatch) {
return branchSpecificPath; // Return unchanged if not a branch-specific path
}
const extension = match[1];
const extension = yamlMatch[1];
const pathWithoutBranchAndExtension = branchSpecificPath.substring(
0,
branchSpecificPath.length - `.${sanitizedBranchName}${extension}`.length
@@ -166,10 +258,14 @@ export function isCurrentBranchFile(path: string): boolean {
return false;
}
// Sanitize branch name to match what would be used in file naming
const sanitizedBranchName = currentBranch.replace(/[\/\\:*?"<>|.]/g, '_');
const escapedBranchName = sanitizedBranchName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
// Use cached pattern or create and cache new one
let pattern = branchPatternCache.get(currentBranch);
if (!pattern) {
pattern = new RegExp(`\\.${currentBranch}\\.(variable|resource)\\.yaml$`);
pattern = new RegExp(`\\.${escapedBranchName}\\.${buildYamlTypePattern()}\\.yaml$|\\.${escapedBranchName}\\.resource\\.file\\..+$`);
branchPatternCache.set(currentBranch, pattern);
}
@@ -181,6 +277,6 @@ export function isCurrentBranchFile(path: string): boolean {
* Used to identify and skip files from other branches during sync operations
*/
export function isBranchSpecificFile(path: string): boolean {
// Pattern: *.branchName.variable.yaml or *.branchName.resource.yaml
return /\.[^.]+\.(variable|resource)\.yaml$/.test(path);
const yamlTypePattern = buildYamlTypePattern();
return new RegExp(`\\.[^.]+\\.${yamlTypePattern}\\.yaml$|\\.[^.]+\\.resource\\.file\\..+$`).test(path);
}

27
cli/src/types.ts
View File

@@ -45,6 +45,17 @@ export interface DifferenceChange {
export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;
export const TRIGGER_TYPES = [
'http',
'websocket',
'kafka',
'nats',
'postgres',
'mqtt',
'sqs',
'gcp'
] as const;
export type GlobalOptions = {
baseUrl: string | undefined;
workspace: string | undefined;
@@ -111,6 +122,17 @@ export function showConflict(path: string, local: string, remote: string) {
log.info("\n");
}
/**
* Pushes an object to the workspace server based on its type
* @param workspace - The workspace ID to push to
* @param p - The server path (base path for branch-specific items)
* @param befObj - The previous object state (for updates)
* @param newObj - The new object state to push
* @param plainSecrets - Whether to store secrets in plain text
* @param alreadySynced - Array to track already synced items
* @param message - Optional commit/update message
* @param originalLocalPath - The original local file path (used for branch-specific resource file resolution)
*/
export async function pushObj(
workspace: string,
p: string,
@@ -118,7 +140,8 @@ export async function pushObj(
newObj: any,
plainSecrets: boolean,
alreadySynced: string[],
message?: string
message?: string,
originalLocalPath?: string
) {
const typeEnding = getTypeStrFromPath(p);
@@ -135,7 +158,7 @@ export async function pushObj(
} else if (typeEnding === "resource") {
if (!alreadySynced.includes(p)) {
alreadySynced.push(p);
await pushResource(workspace, p, befObj, newObj);
await pushResource(workspace, p, befObj, newObj, originalLocalPath || p);
}
} else if (typeEnding === "resource-type") {
await pushResourceType(workspace, p, befObj, newObj);

6
cli/src/utils/utils.ts
View File

@@ -136,10 +136,12 @@ export function sleep(ms: number) {
export function isFileResource(path: string): boolean {
const splitPath = path.split(".");
// Check for pattern: *.resource.file.* (handles both base and branch-specific)
return (
splitPath.length >= 4 &&
splitPath[1] == "resource" &&
splitPath[2] == "file"
splitPath[splitPath.length - 3] == "resource" &&
splitPath[splitPath.length - 2] == "file"
);
}

3
frontend/src/lib/hubPaths.json
View File

@@ -12,7 +12,8 @@
"gitSync_10": "hub/19785/sync-script-to-git-repo-windmill",
"gitSync_11": "hub/19789/sync-script-to-git-repo-windmill",
"gitSync_12": "hub/19798/sync-script-to-git-repo-windmill",
"gitSync": "hub/19801/sync-script-to-git-repo-windmill",
"gitSync_13": "hub/19801/sync-script-to-git-repo-windmill",
"gitSync": "hub/19803/sync-script-to-git-repo-windmill",
"gitSyncTest_0": "hub/9073/git-repo-test-read-write-windmill",
"gitSyncTest_1": "hub/11499/git-repo-test-read-write-windmill",
"gitSyncTest_2": "hub/11667/git-repo-test-read-write-windmill",