diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 6b54625e4e..d3a272cc6a 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -199,7 +199,6 @@ components: enum: - static required: - - value - type JavascriptTransform: diff --git a/windmill-yaml-validator/README.md b/windmill-yaml-validator/README.md new file mode 100644 index 0000000000..0921482b36 --- /dev/null +++ b/windmill-yaml-validator/README.md @@ -0,0 +1,159 @@ +# Windmill YAML Validator + +A TypeScript-based YAML validator for Windmill flow files. This package validates flow.yaml files against the OpenFlow JSON schema to ensure they conform to the Windmill flow specification. + +## Overview + +The windmill-yaml-validator provides runtime validation for Windmill flow YAML files. It is currently used in the **Windmill VSCode extension** to show syntax errors and validation issues on `flow.yaml` files in real-time as developers edit them. + +## Features + +- **Schema-based validation**: Validates against the official OpenFlow JSON schema +- **Detailed error reporting**: Returns comprehensive error information with specific paths to invalid fields + +## Installation + +```bash +npm install windmill-yaml-validator +``` + +## Usage + +### Basic Validation + +```typescript +import { FlowValidator } from "windmill-yaml-validator"; + +const validator = new FlowValidator(); + +const yamlContent = ` +summary: Test Flow +value: + modules: [] +`; + +const result = validator.validateFlow(yamlContent); + +if (result.errors.length === 0) { + console.log("Flow is valid!"); +} else { + console.log("Validation errors:", result.errors); +} +``` + +### Error Handling + +The validator returns detailed error information for invalid flows: + +```typescript +const invalidYaml = ` +summary: 123 # Should be a string +value: + modules: + - id: step1 + value: + type: rawscript + language: invalid_language # Invalid enum value +`; + +const result = validator.validateFlow(invalidYaml); + +result.errors.forEach((error) => { + console.log(`Error at ${error.instancePath}: ${error.message}`); + // Example output: + // Error at /summary: must be string + // Error at /value/modules/0/value/language: must be equal to one of the allowed values +}); +``` + +## API + +### `FlowValidator` + +Main validator class that validates Windmill flow YAML files. + +#### Constructor + +```typescript +new FlowValidator(); +``` + +Creates a new validator instance. The constructor initializes the AJV validator with the OpenFlow schema. + +#### Methods + +##### `validateFlow(doc: string)` + +Validates a flow document against the OpenFlow schema. + +**Parameters:** + +- `doc` (string): The YAML flow document as a string + +**Returns:** + +```typescript +{ + parsed: YamlParserResult; // Parsed YAML with source pointers + errors: ErrorObject[]; // Array of validation errors (empty if valid) +} +``` + +**Throws:** + +- Error if `doc` is not a string + +## Development + +### Building + +```bash +npm run build +``` + +The build process: + +1. Runs `gen_openflow_schema.sh` to generate the OpenFlow JSON schema from `openflow.openapi.yaml` +2. Removes discriminator mappings (not supported by AJV) +3. Compiles TypeScript to JavaScript + +### Testing + +```bash +npm test +``` + +Run tests in watch mode: + +```bash +npm test:watch +``` + +### Schema Generation + +The validator uses a JSON schema generated from the OpenAPI specification: + +```bash +./gen_openflow_schema.sh +``` + +This script: + +- Bundles `openflow.openapi.yaml` into a single JSON schema +- Removes discriminator mappings for AJV compatibility +- Removes the `ToolValue` discriminator entirely (see below) +- Outputs to `src/gen/openflow.json` + +#### Why Remove Discriminators? + +The OpenFlow schema uses OpenAPI discriminators for efficient type resolution in `oneOf` schemas. However, AJV's discriminator support has limitations: + +1. **Discriminator Mappings**: Not fully supported by AJV, so they are removed from all schemas +2. **ToolValue Discriminator**: Completely removed because `FlowModuleTool` uses `allOf` composition, which prevents AJV from finding the discriminator property (`tool_type`) at the expected location + +**Impact**: Without discriminators, AJV falls back to standard `oneOf` validation, which: + +- Tests each alternative until one matches +- Is slightly slower but still performant for our use case +- Provides the same validation correctness +- Works correctly with complex schema compositions like `allOf` diff --git a/windmill-yaml-validator/gen_openflow_schema.sh b/windmill-yaml-validator/gen_openflow_schema.sh index fc179fedfe..f2c9ecf2a2 100755 --- a/windmill-yaml-validator/gen_openflow_schema.sh +++ b/windmill-yaml-validator/gen_openflow_schema.sh @@ -22,6 +22,13 @@ try { } removeMapping(schema); + + // Remove discriminator entirely from ToolValue as it doesn't work with allOf in FlowModuleTool + if (schema.components?.schemas?.ToolValue?.discriminator) { + delete schema.components.schemas.ToolValue.discriminator; + console.log('Removed discriminator from ToolValue schema'); + } + fs.writeFileSync(filePath, JSON.stringify(schema, null, 2)); console.log('Removed discriminator mappings from openflow.json'); } catch (e) { diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index fb9a84198f..1aa6ebf542 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.0.0", + "version": "1.0.1", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index 68d89d27fe..a1994aecb1 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.0.0", + "version": "1.0.1", "description": "YAML validator for Windmill", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts b/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts index 046a3f2c16..0efee8d189 100644 --- a/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts +++ b/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts @@ -1,285 +1,421 @@ -import { FlowValidator } from '../flow-validator'; -import * as fs from 'fs'; -import * as path from 'path'; +import { FlowValidator } from "../flow-validator"; +import * as fs from "fs"; +import * as path from "path"; -describe('FlowValidator', () => { +describe("FlowValidator", () => { let validator: FlowValidator; - const samplesDir = path.join(__dirname, 'test-samples'); + const samplesDir = path.join(__dirname, "test-samples"); const readSample = (filename: string): string => { - return fs.readFileSync(path.join(samplesDir, filename), 'utf-8'); + return fs.readFileSync(path.join(samplesDir, filename), "utf-8"); }; beforeEach(() => { validator = new FlowValidator(); }); - describe('constructor', () => { - it('should create a validator instance', () => { + describe("constructor", () => { + it("should create a validator instance", () => { expect(validator).toBeInstanceOf(FlowValidator); }); - it('should initialize without throwing', () => { + it("should initialize without throwing", () => { expect(() => new FlowValidator()).not.toThrow(); }); }); - describe('validateFlow', () => { - it('should throw error for non-string input', () => { - expect(() => validator.validateFlow(null as any)).toThrow('Document must be a string'); - expect(() => validator.validateFlow(123 as any)).toThrow('Document must be a string'); - expect(() => validator.validateFlow({} as any)).toThrow('Document must be a string'); - expect(() => validator.validateFlow([] as any)).toThrow('Document must be a string'); + describe("validateFlow", () => { + it("should throw error for non-string input", () => { + expect(() => validator.validateFlow(null as any)).toThrow( + "Document must be a string" + ); + expect(() => validator.validateFlow(123 as any)).toThrow( + "Document must be a string" + ); + expect(() => validator.validateFlow({} as any)).toThrow( + "Document must be a string" + ); + expect(() => validator.validateFlow([] as any)).toThrow( + "Document must be a string" + ); }); - describe('valid flows', () => { - it('should validate a valid minimal flow from sample file', () => { - const validFlow = readSample('valid-minimal.yaml'); - + describe("valid flows", () => { + it("should validate a valid minimal flow from sample file", () => { + const validFlow = readSample("valid-minimal.yaml"); + const result = validator.validateFlow(validFlow); - + expect(result.errors).toHaveLength(0); expect(result.parsed).toBeDefined(); expect(result.parsed.data).toMatchObject({ - summary: 'Test Flow', + summary: "Test Flow", value: { - modules: [] - } + modules: [], + }, }); }); - it('should validate a script flow from sample file', () => { - const validFlow = readSample('valid-script-flow.yaml'); - + it("should validate a script flow from sample file", () => { + const validFlow = readSample("valid-script-flow.yaml"); + const result = validator.validateFlow(validFlow); - + expect(result.errors).toHaveLength(0); expect(result.parsed.data).toMatchObject({ - summary: 'Simple Script Flow', - description: 'A basic flow that runs a TypeScript script', + summary: "Simple Script Flow", + description: "A basic flow that runs a TypeScript script", schema: expect.objectContaining({ - type: 'object', + type: "object", properties: expect.objectContaining({ message: expect.objectContaining({ - type: 'string', - default: 'Hello World' - }) - }) + type: "string", + default: "Hello World", + }), + }), }), value: { modules: expect.arrayContaining([ expect.objectContaining({ - id: 'script_step', + id: "script_step", value: expect.objectContaining({ - type: 'rawscript', - language: 'deno', - input_transforms: {} - }) - }) - ]) - } + type: "rawscript", + language: "deno", + input_transforms: {}, + }), + }), + ]), + }, }); }); }); - describe('invalid flows', () => { - it('should return errors for missing summary from sample file', () => { - const invalidFlow = readSample('invalid-missing-summary.yaml'); - + describe("invalid flows", () => { + it("should return errors for missing summary from sample file", () => { + const invalidFlow = readSample("invalid-missing-summary.yaml"); + const result = validator.validateFlow(invalidFlow); - + expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors.some(error => - error.instancePath === '' && - error.keyword === 'required' && - error.params?.missingProperty === 'summary' - )).toBe(true); + expect( + result.errors.some( + (error) => + error.instancePath === "" && + error.keyword === "required" && + error.params?.missingProperty === "summary" + ) + ).toBe(true); }); - it('should return errors for invalid types from sample file', () => { - const invalidFlow = readSample('invalid-wrong-types.yaml'); - + it("should return errors for invalid types from sample file", () => { + const invalidFlow = readSample("invalid-wrong-types.yaml"); + const result = validator.validateFlow(invalidFlow); - + expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors.some(error => - error.instancePath === '/summary' && - error.keyword === 'type' - )).toBe(true); + expect( + result.errors.some( + (error) => + error.instancePath === "/summary" && error.keyword === "type" + ) + ).toBe(true); }); - it('should return errors for invalid language from sample file', () => { - const invalidFlow = readSample('invalid-language.yaml'); - + it("should return errors for invalid language from sample file", () => { + const invalidFlow = readSample("invalid-language.yaml"); + const result = validator.validateFlow(invalidFlow); - + expect(result.errors.length).toBeGreaterThan(0); - expect(result.errors.some(error => - error.instancePath === '/value/modules/0/value/language' && - error.keyword === 'enum' - )).toBe(true); + expect( + result.errors.some( + (error) => + error.instancePath === "/value/modules/0/value/language" && + error.keyword === "enum" + ) + ).toBe(true); }); - it('should handle empty file from sample', () => { - const emptyFlow = readSample('empty.yaml'); - + it("should handle empty file from sample", () => { + const emptyFlow = readSample("empty.yaml"); + const result = validator.validateFlow(emptyFlow); - + expect(result.parsed).toBeDefined(); expect(result.errors.length).toBeGreaterThan(0); }); - it('should handle complex invalid flow with comprehensive error detection', () => { - const complexInvalidFlow = readSample('invalid-complex-flow.yaml'); - + it("should handle complex invalid flow with comprehensive error detection", () => { + const complexInvalidFlow = readSample("invalid-complex-flow.yaml"); + const result = validator.validateFlow(complexInvalidFlow); - + expect(result.errors.length).toBeGreaterThan(20); // Should have many errors - + // Check for missing required fields errors - const missingRequiredErrors = result.errors.filter(error => - error.keyword === 'required' + const missingRequiredErrors = result.errors.filter( + (error) => error.keyword === "required" ); expect(missingRequiredErrors.length).toBeGreaterThan(10); - + // Check for invalid enum errors (invalid language) - const enumErrors = result.errors.filter(error => - error.keyword === 'enum' + const enumErrors = result.errors.filter( + (error) => error.keyword === "enum" ); expect(enumErrors.length).toBeGreaterThan(0); - + // Check for type errors (string vs number, boolean vs string, etc.) - const typeErrors = result.errors.filter(error => - error.keyword === 'type' + const typeErrors = result.errors.filter( + (error) => error.keyword === "type" ); expect(typeErrors.length).toBeGreaterThan(5); - + // Should detect invalid forloop structure (by checking for forloop-related required fields) - const forloopRequiredErrors = result.errors.filter(error => - error.keyword === 'required' && error.message && - (error.message.includes('modules') || error.message.includes('iterator') || error.message.includes('skip_failures')) + const forloopRequiredErrors = result.errors.filter( + (error) => + error.keyword === "required" && + error.message && + (error.message.includes("modules") || + error.message.includes("iterator") || + error.message.includes("skip_failures")) ); expect(forloopRequiredErrors.length).toBeGreaterThan(0); - + // Should detect invalid branch structure - const branchRequiredErrors = result.errors.filter(error => - error.keyword === 'required' && error.message && - (error.message.includes('branches') || error.message.includes('default') || error.message.includes('expr')) + const branchRequiredErrors = result.errors.filter( + (error) => + error.keyword === "required" && + error.message && + (error.message.includes("branches") || + error.message.includes("default") || + error.message.includes("expr")) ); expect(branchRequiredErrors.length).toBeGreaterThan(0); - + // Should detect discriminator errors for invalid transform types - const discriminatorErrors = result.errors.filter(error => - error.keyword === 'discriminator' + const discriminatorErrors = result.errors.filter( + (error) => error.keyword === "discriminator" ); expect(discriminatorErrors.length).toBeGreaterThan(0); }); - it('should handle deeply nested invalid structures with detailed error reporting', () => { - const nestedInvalidFlow = readSample('invalid-nested-structures.yaml'); - + it("should handle deeply nested invalid structures with detailed error reporting", () => { + const nestedInvalidFlow = readSample("invalid-nested-structures.yaml"); + const result = validator.validateFlow(nestedInvalidFlow); - + expect(result.errors.length).toBeGreaterThan(10); // Should have many nested errors - + // Check for deeply nested path errors (paths with many levels) - const deepNestedErrors = result.errors.filter(error => - error.instancePath.split('/').length > 6 // Deep nesting + const deepNestedErrors = result.errors.filter( + (error) => error.instancePath.split("/").length > 6 // Deep nesting ); expect(deepNestedErrors.length).toBeGreaterThan(0); - + // Check for transform-related errors - const transformErrors = result.errors.filter(error => - error.instancePath.includes('input_transforms') || - error.instancePath.includes('iterator') + const transformErrors = result.errors.filter( + (error) => + error.instancePath.includes("input_transforms") || + error.instancePath.includes("iterator") ); expect(transformErrors.length).toBeGreaterThan(0); - + // Check for type errors in general - const typeErrors = result.errors.filter(error => - error.keyword === 'type' + const typeErrors = result.errors.filter( + (error) => error.keyword === "type" ); expect(typeErrors.length).toBeGreaterThan(0); - + // Check for missing required fields in nested structures - const nestedRequiredErrors = result.errors.filter(error => - error.keyword === 'required' && - error.instancePath.includes('/modules/') + const nestedRequiredErrors = result.errors.filter( + (error) => + error.keyword === "required" && + error.instancePath.includes("/modules/") ); expect(nestedRequiredErrors.length).toBeGreaterThan(0); - + // Check for discriminator errors in nested transforms - const nestedDiscriminatorErrors = result.errors.filter(error => - error.keyword === 'discriminator' + const nestedDiscriminatorErrors = result.errors.filter( + (error) => error.keyword === "discriminator" ); expect(nestedDiscriminatorErrors.length).toBeGreaterThan(0); }); - it('should provide specific error locations for complex validation failures', () => { - const complexInvalidFlow = readSample('invalid-complex-flow.yaml'); - + it("should provide specific error locations for complex validation failures", () => { + const complexInvalidFlow = readSample("invalid-complex-flow.yaml"); + const result = validator.validateFlow(complexInvalidFlow); - + // Verify that errors have meaningful instance paths - const errorsWithPaths = result.errors.filter(error => - error.instancePath && error.instancePath.length > 0 + const errorsWithPaths = result.errors.filter( + (error) => error.instancePath && error.instancePath.length > 0 ); expect(errorsWithPaths.length).toBeGreaterThan(5); - + // Check that we can identify specific problematic modules - const moduleSpecificErrors = result.errors.filter(error => - error.instancePath.includes('/value/modules/') + const moduleSpecificErrors = result.errors.filter((error) => + error.instancePath.includes("/value/modules/") ); expect(moduleSpecificErrors.length).toBeGreaterThan(0); - + // Verify error messages are descriptive - const descriptiveErrors = result.errors.filter(error => - error.message && error.message.length > 0 + const descriptiveErrors = result.errors.filter( + (error) => error.message && error.message.length > 0 ); expect(descriptiveErrors.length).toBe(result.errors.length); }); - it('should handle all major flow control structures with validation errors', () => { - const complexInvalidFlow = readSample('invalid-complex-flow.yaml'); - + it("should handle all major flow control structures with validation errors", () => { + const complexInvalidFlow = readSample("invalid-complex-flow.yaml"); + const result = validator.validateFlow(complexInvalidFlow); - + expect(result.errors.length).toBeGreaterThan(20); - + // Should find errors related to flow control structures by checking instance paths - const flowControlErrors = result.errors.filter(error => - error.instancePath.includes('/modules/2/') || // forloop module - error.instancePath.includes('/modules/3/') || // forloop module - error.instancePath.includes('/modules/4/') || // branch module - error.instancePath.includes('/modules/5/') || // branch module - error.instancePath.includes('/modules/6/') || // branch all module - error.instancePath.includes('/modules/7/') // while loop module + const flowControlErrors = result.errors.filter( + (error) => + error.instancePath.includes("/modules/2/") || // forloop module + error.instancePath.includes("/modules/3/") || // forloop module + error.instancePath.includes("/modules/4/") || // branch module + error.instancePath.includes("/modules/5/") || // branch module + error.instancePath.includes("/modules/6/") || // branch all module + error.instancePath.includes("/modules/7/") // while loop module ); expect(flowControlErrors.length).toBeGreaterThan(10); - + // Should find errors in script and flow references - const pathReferenceErrors = result.errors.filter(error => - error.instancePath.includes('/modules/8/') || // script path module - error.instancePath.includes('/modules/9/') // flow path module + const pathReferenceErrors = result.errors.filter( + (error) => + error.instancePath.includes("/modules/8/") || // script path module + error.instancePath.includes("/modules/9/") // flow path module ); expect(pathReferenceErrors.length).toBeGreaterThan(0); - + // Should detect modules with missing IDs - const missingIdErrors = result.errors.filter(error => - error.keyword === 'required' && error.message && - error.message.includes('id') + const missingIdErrors = result.errors.filter( + (error) => + error.keyword === "required" && + error.message && + error.message.includes("id") ); expect(missingIdErrors.length).toBeGreaterThan(0); - + // Should detect type mismatches at the flow level - const flowLevelTypeErrors = result.errors.filter(error => - error.instancePath.startsWith('/value/') && - !error.instancePath.includes('/modules/') && - error.keyword === 'type' + const flowLevelTypeErrors = result.errors.filter( + (error) => + error.instancePath.startsWith("/value/") && + !error.instancePath.includes("/modules/") && + error.keyword === "type" ); expect(flowLevelTypeErrors.length).toBeGreaterThan(0); }); }); + describe("AI agent flows", () => { + describe("valid AI agent flows", () => { + it("should validate a basic AI agent flow with FlowModule tools", () => { + const validFlow = readSample("valid-aiagent-basic.yaml"); + + const result = validator.validateFlow(validFlow); + + expect(result.errors).toHaveLength(0); + }); + + it("should validate an AI agent flow with MCP tools", () => { + const validFlow = readSample("valid-aiagent-mcp.yaml"); + + const result = validator.validateFlow(validFlow); + + expect(result.errors).toHaveLength(0); + }); + + it("should validate an AI agent flow with mixed FlowModule and MCP tools", () => { + const validFlow = readSample("valid-aiagent-mixed.yaml"); + + const result = validator.validateFlow(validFlow); + + expect(result.errors).toHaveLength(0); + }); + + it("should validate an AI agent flow with parallel execution enabled", () => { + const validFlow = readSample("valid-aiagent-parallel.yaml"); + + const result = validator.validateFlow(validFlow); + + expect(result.errors).toHaveLength(0); + // Verify tools array has expected structure + const agentModule = (result.parsed.data as any).value.modules[0]; + expect(agentModule.value.tools).toHaveLength(2); + }); + }); + + describe("invalid AI agent flows", () => { + it("should return errors for AI agent missing required tools field", () => { + const invalidFlow = readSample("invalid-aiagent-missing-tools.yaml"); + + const result = validator.validateFlow(invalidFlow); + + expect(result.errors.length).toBeGreaterThan(0); + expect( + result.errors.some( + (error) => + error.keyword === "required" && + error.params?.missingProperty === "tools" + ) + ).toBe(true); + }); + + it("should return errors for AI agent missing type field", () => { + const invalidFlow = readSample("invalid-aiagent-missing-type.yaml"); + + const result = validator.validateFlow(invalidFlow); + + expect(result.errors.length).toBeGreaterThan(0); + // Should fail discriminator validation since type is missing + expect( + result.errors.some( + (error) => + error.keyword === "required" || + error.keyword === "discriminator" + ) + ).toBe(true); + }); + + it("should return errors for AI agent with invalid tool_type", () => { + const invalidFlow = readSample( + "invalid-aiagent-invalid-tool-type.yaml" + ); + + const result = validator.validateFlow(invalidFlow); + + expect(result.errors.length).toBeGreaterThan(0); + // Should fail discriminator validation for invalid tool_type + expect( + result.errors.some( + (error) => + error.keyword === "discriminator" || error.keyword === "enum" + ) + ).toBe(true); + }); + + it("should return errors for MCP tool missing resource_path", () => { + const invalidFlow = readSample( + "invalid-aiagent-mcp-missing-resource.yaml" + ); + + const result = validator.validateFlow(invalidFlow); + + expect(result.errors.length).toBeGreaterThan(0); + expect( + result.errors.some( + (error) => + error.keyword === "required" && + error.params?.missingProperty === "resource_path" + ) + ).toBe(true); + }); + }); + }); }); -}); \ No newline at end of file +}); diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-invalid-tool-type.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-invalid-tool-type.yaml new file mode 100644 index 0000000000..171de33af7 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-invalid-tool-type.yaml @@ -0,0 +1,20 @@ +summary: Invalid AI Agent - Invalid Tool Type +description: An AI agent with an invalid tool_type value +value: + modules: + - id: invalid_tool_agent + value: + type: aiagent + input_transforms: + task: + type: static + value: "Perform task" + tools: + - id: bad_tool + summary: Tool with invalid type + value: + tool_type: invalid_type # Should be 'flowmodule' or 'mcp' + type: rawscript + input_transforms: {} + content: "export async function main() { return {}; }" + language: deno diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-mcp-missing-resource.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-mcp-missing-resource.yaml new file mode 100644 index 0000000000..1ebb783280 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-mcp-missing-resource.yaml @@ -0,0 +1,27 @@ +summary: Invalid AI Agent - MCP Missing Resource Path +description: An AI agent with MCP tool missing required resource_path +value: + modules: + - id: mcp_broken_agent + value: + type: aiagent + input_transforms: + instruction: + type: static + value: "Execute MCP tools" + tools: + - id: broken_mcp_tool + summary: MCP tool without resource_path + value: + tool_type: mcp + # Missing required 'resource_path' field + include_tools: + - search + - fetch + - id: valid_tool + value: + tool_type: flowmodule + type: rawscript + input_transforms: {} + content: "export async function main() { return {}; }" + language: deno diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-tools.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-tools.yaml new file mode 100644 index 0000000000..8d6d4a272c --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-tools.yaml @@ -0,0 +1,12 @@ +summary: Invalid AI Agent - Missing Tools +description: An AI agent without the required tools field +value: + modules: + - id: broken_agent + value: + type: aiagent + input_transforms: + prompt: + type: static + value: "Test prompt" + # Missing required 'tools' field diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-type.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-type.yaml new file mode 100644 index 0000000000..14c9aa2dc0 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-type.yaml @@ -0,0 +1,19 @@ +summary: Invalid AI Agent - Missing Type +description: An AI agent module without the required type field +value: + modules: + - id: typeless_agent + value: + # Missing required 'type' field + input_transforms: + query: + type: static + value: "Search query" + tools: + - id: tool_1 + value: + tool_type: flowmodule + type: rawscript + input_transforms: {} + content: "export async function main() { return {}; }" + language: deno diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml new file mode 100644 index 0000000000..83758cbb48 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml @@ -0,0 +1,39 @@ +summary: Basic AI Agent Flow +description: A flow with an AI agent using FlowModule tools +schema: + type: object + properties: + prompt: + type: string + default: "Analyze the data" +value: + modules: + - id: agent_step + value: + type: aiagent + input_transforms: + prompt: + type: static + value: "Please analyze the data" + tools: + - id: tool_1 + summary: Data fetcher + value: + tool_type: flowmodule + type: rawscript + input_transforms: {} + content: | + export async function main() { + return { data: [1, 2, 3] }; + } + language: deno + - id: tool_2 + summary: Calculator + value: + tool_type: flowmodule + type: script + path: f/scripts/calculator + input_transforms: + operation: + type: javascript + expr: "previous.result" diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml new file mode 100644 index 0000000000..c8d9fceb55 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml @@ -0,0 +1,27 @@ +summary: AI Agent with MCP Tools +description: A flow with an AI agent using MCP server tools +value: + modules: + - id: mcp_agent + value: + type: aiagent + input_transforms: + task: + type: static + value: "Search for information" + tools: + - id: mcp_search + summary: Search tool from MCP + value: + tool_type: mcp + resource_path: f/resources/mcp_server + include_tools: + - search + - fetch + - id: mcp_calculator + summary: Math operations + value: + tool_type: mcp + resource_path: f/resources/calculator_mcp + exclude_tools: + - deprecated_function diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml new file mode 100644 index 0000000000..902aef8dd7 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml @@ -0,0 +1,46 @@ +summary: AI Agent with Mixed Tools +description: An AI agent using both FlowModule and MCP tools +schema: + type: object + properties: + user_query: + type: string +value: + modules: + - id: mixed_agent + value: + type: aiagent + input_transforms: + query: + type: javascript + expr: "flow_input.user_query" + tools: + - id: custom_script + value: + tool_type: flowmodule + type: rawscript + input_transforms: + input: + type: static + value: "test" + content: | + export async function main(input: string) { + return { processed: input }; + } + language: python3 + - id: mcp_tool + value: + tool_type: mcp + resource_path: f/resources/mcp_integration + include_tools: + - api_call + - id: flow_tool + summary: Nested flow + value: + tool_type: flowmodule + type: flow + path: f/flows/data_processor + input_transforms: + data: + type: javascript + expr: "previous_result.output" diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml new file mode 100644 index 0000000000..5053b4f9ee --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml @@ -0,0 +1,38 @@ +summary: AI Agent with Parallel Execution +description: An AI agent configured to execute tools in parallel +value: + modules: + - id: parallel_agent + value: + type: aiagent + parallel: true + input_transforms: + instruction: + type: static + value: "Process data in parallel" + model: + type: javascript + expr: "'gpt-4'" + tools: + - id: parallel_tool_1 + summary: First parallel tool + value: + tool_type: flowmodule + type: rawscript + input_transforms: {} + content: | + export async function main() { + return { result: "tool_1" }; + } + language: bun + - id: parallel_tool_2 + summary: Second parallel tool + value: + tool_type: flowmodule + type: rawscript + input_transforms: {} + content: | + export async function main() { + return { result: "tool_2" }; + } + language: deno