fix(yaml-validator): update openflow for aiagents (#6895)

* update openflow for aiagents

* remove value from required in openflow.json

* all

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

windmill-yaml-validator/README.md

@@ -0,0 +1,159 @@
+# Windmill YAML Validator
+
+A TypeScript-based YAML validator for Windmill flow files. This package validates flow.yaml files against the OpenFlow JSON schema to ensure they conform to the Windmill flow specification.
+
+## Overview
+
+The windmill-yaml-validator provides runtime validation for Windmill flow YAML files. It is currently used in the **Windmill VSCode extension** to show syntax errors and validation issues on `flow.yaml` files in real-time as developers edit them.
+
+## Features
+
+- **Schema-based validation**: Validates against the official OpenFlow JSON schema
+- **Detailed error reporting**: Returns comprehensive error information with specific paths to invalid fields
+
+## Installation
+
+```bash
+npm install windmill-yaml-validator
+```
+
+## Usage
+
+### Basic Validation
+
+```typescript
+import { FlowValidator } from "windmill-yaml-validator";
+
+const validator = new FlowValidator();
+
+const yamlContent = `
+summary: Test Flow
+value:
+  modules: []
+`;
+
+const result = validator.validateFlow(yamlContent);
+
+if (result.errors.length === 0) {
+  console.log("Flow is valid!");
+} else {
+  console.log("Validation errors:", result.errors);
+}
+```
+
+### Error Handling
+
+The validator returns detailed error information for invalid flows:
+
+```typescript
+const invalidYaml = `
+summary: 123  # Should be a string
+value:
+  modules:
+    - id: step1
+      value:
+        type: rawscript
+        language: invalid_language  # Invalid enum value
+`;
+
+const result = validator.validateFlow(invalidYaml);
+
+result.errors.forEach((error) => {
+  console.log(`Error at ${error.instancePath}: ${error.message}`);
+  // Example output:
+  // Error at /summary: must be string
+  // Error at /value/modules/0/value/language: must be equal to one of the allowed values
+});
+```
+
+## API
+
+### `FlowValidator`
+
+Main validator class that validates Windmill flow YAML files.
+
+#### Constructor
+
+```typescript
+new FlowValidator();
+```
+
+Creates a new validator instance. The constructor initializes the AJV validator with the OpenFlow schema.
+
+#### Methods
+
+##### `validateFlow(doc: string)`
+
+Validates a flow document against the OpenFlow schema.
+
+**Parameters:**
+
+- `doc` (string): The YAML flow document as a string
+
+**Returns:**
+
+```typescript
+{
+  parsed: YamlParserResult<unknown>;  // Parsed YAML with source pointers
+  errors: ErrorObject[];               // Array of validation errors (empty if valid)
+}
+```
+
+**Throws:**
+
+- Error if `doc` is not a string
+
+## Development
+
+### Building
+
+```bash
+npm run build
+```
+
+The build process:
+
+1. Runs `gen_openflow_schema.sh` to generate the OpenFlow JSON schema from `openflow.openapi.yaml`
+2. Removes discriminator mappings (not supported by AJV)
+3. Compiles TypeScript to JavaScript
+
+### Testing
+
+```bash
+npm test
+```
+
+Run tests in watch mode:
+
+```bash
+npm test:watch
+```
+
+### Schema Generation
+
+The validator uses a JSON schema generated from the OpenAPI specification:
+
+```bash
+./gen_openflow_schema.sh
+```
+
+This script:
+
+- Bundles `openflow.openapi.yaml` into a single JSON schema
+- Removes discriminator mappings for AJV compatibility
+- Removes the `ToolValue` discriminator entirely (see below)
+- Outputs to `src/gen/openflow.json`
+
+#### Why Remove Discriminators?
+
+The OpenFlow schema uses OpenAPI discriminators for efficient type resolution in `oneOf` schemas. However, AJV's discriminator support has limitations:
+
+1. **Discriminator Mappings**: Not fully supported by AJV, so they are removed from all schemas
+2. **ToolValue Discriminator**: Completely removed because `FlowModuleTool` uses `allOf` composition, which prevents AJV from finding the discriminator property (`tool_type`) at the expected location
+
+**Impact**: Without discriminators, AJV falls back to standard `oneOf` validation, which:
+
+- Tests each alternative until one matches
+- Is slightly slower but still performant for our use case
+- Provides the same validation correctness
+- Works correctly with complex schema compositions like `allOf`

windmill-yaml-validator/gen_openflow_schema.sh

@@ -22,6 +22,13 @@ try {
     }
 
     removeMapping(schema);
+
+    // Remove discriminator entirely from ToolValue as it doesn't work with allOf in FlowModuleTool
+    if (schema.components?.schemas?.ToolValue?.discriminator) {
+        delete schema.components.schemas.ToolValue.discriminator;
+        console.log('Removed discriminator from ToolValue schema');
+    }
+
     fs.writeFileSync(filePath, JSON.stringify(schema, null, 2));
     console.log('Removed discriminator mappings from openflow.json');
 } catch (e) {

windmill-yaml-validator/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "windmill-yaml-validator",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "windmill-yaml-validator",
-      "version": "1.0.0",
+      "version": "1.0.1",
       "license": "Apache 2.0",
       "dependencies": {
         "@stoplight/yaml": "^4.3.0",

windmill-yaml-validator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "windmill-yaml-validator",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "YAML validator for Windmill",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts

@@ -1,285 +1,421 @@
-import { FlowValidator } from '../flow-validator';
-import * as fs from 'fs';
-import * as path from 'path';
+import { FlowValidator } from "../flow-validator";
+import * as fs from "fs";
+import * as path from "path";
 
-describe('FlowValidator', () => {
+describe("FlowValidator", () => {
   let validator: FlowValidator;
-  const samplesDir = path.join(__dirname, 'test-samples');
+  const samplesDir = path.join(__dirname, "test-samples");
 
   const readSample = (filename: string): string => {
-    return fs.readFileSync(path.join(samplesDir, filename), 'utf-8');
+    return fs.readFileSync(path.join(samplesDir, filename), "utf-8");
   };
 
   beforeEach(() => {
     validator = new FlowValidator();
   });
 
-  describe('constructor', () => {
-    it('should create a validator instance', () => {
+  describe("constructor", () => {
+    it("should create a validator instance", () => {
       expect(validator).toBeInstanceOf(FlowValidator);
     });
 
-    it('should initialize without throwing', () => {
+    it("should initialize without throwing", () => {
       expect(() => new FlowValidator()).not.toThrow();
     });
   });
 
-  describe('validateFlow', () => {
-    it('should throw error for non-string input', () => {
-      expect(() => validator.validateFlow(null as any)).toThrow('Document must be a string');
-      expect(() => validator.validateFlow(123 as any)).toThrow('Document must be a string');
-      expect(() => validator.validateFlow({} as any)).toThrow('Document must be a string');
-      expect(() => validator.validateFlow([] as any)).toThrow('Document must be a string');
+  describe("validateFlow", () => {
+    it("should throw error for non-string input", () => {
+      expect(() => validator.validateFlow(null as any)).toThrow(
+        "Document must be a string"
+      );
+      expect(() => validator.validateFlow(123 as any)).toThrow(
+        "Document must be a string"
+      );
+      expect(() => validator.validateFlow({} as any)).toThrow(
+        "Document must be a string"
+      );
+      expect(() => validator.validateFlow([] as any)).toThrow(
+        "Document must be a string"
+      );
     });
 
-    describe('valid flows', () => {
-      it('should validate a valid minimal flow from sample file', () => {
-        const validFlow = readSample('valid-minimal.yaml');
-        
+    describe("valid flows", () => {
+      it("should validate a valid minimal flow from sample file", () => {
+        const validFlow = readSample("valid-minimal.yaml");
+
         const result = validator.validateFlow(validFlow);
-        
+
         expect(result.errors).toHaveLength(0);
         expect(result.parsed).toBeDefined();
         expect(result.parsed.data).toMatchObject({
-          summary: 'Test Flow',
+          summary: "Test Flow",
           value: {
-            modules: []
-          }
+            modules: [],
+          },
         });
       });
 
-      it('should validate a script flow from sample file', () => {
-        const validFlow = readSample('valid-script-flow.yaml');
-        
+      it("should validate a script flow from sample file", () => {
+        const validFlow = readSample("valid-script-flow.yaml");
+
         const result = validator.validateFlow(validFlow);
-        
+
         expect(result.errors).toHaveLength(0);
         expect(result.parsed.data).toMatchObject({
-          summary: 'Simple Script Flow',
-          description: 'A basic flow that runs a TypeScript script',
+          summary: "Simple Script Flow",
+          description: "A basic flow that runs a TypeScript script",
           schema: expect.objectContaining({
-            type: 'object',
+            type: "object",
             properties: expect.objectContaining({
               message: expect.objectContaining({
-                type: 'string',
-                default: 'Hello World'
-              })
-            })
+                type: "string",
+                default: "Hello World",
+              }),
+            }),
           }),
           value: {
             modules: expect.arrayContaining([
               expect.objectContaining({
-                id: 'script_step',
+                id: "script_step",
                 value: expect.objectContaining({
-                  type: 'rawscript',
-                  language: 'deno',
-                  input_transforms: {}
-                })
-              })
-            ])
-          }
+                  type: "rawscript",
+                  language: "deno",
+                  input_transforms: {},
+                }),
+              }),
+            ]),
+          },
         });
       });
     });
 
-    describe('invalid flows', () => {
-      it('should return errors for missing summary from sample file', () => {
-        const invalidFlow = readSample('invalid-missing-summary.yaml');
-        
+    describe("invalid flows", () => {
+      it("should return errors for missing summary from sample file", () => {
+        const invalidFlow = readSample("invalid-missing-summary.yaml");
+
         const result = validator.validateFlow(invalidFlow);
-        
+
         expect(result.errors.length).toBeGreaterThan(0);
-        expect(result.errors.some(error => 
-          error.instancePath === '' && 
-          error.keyword === 'required' && 
-          error.params?.missingProperty === 'summary'
-        )).toBe(true);
+        expect(
+          result.errors.some(
+            (error) =>
+              error.instancePath === "" &&
+              error.keyword === "required" &&
+              error.params?.missingProperty === "summary"
+          )
+        ).toBe(true);
       });
 
-      it('should return errors for invalid types from sample file', () => {
-        const invalidFlow = readSample('invalid-wrong-types.yaml');
-        
+      it("should return errors for invalid types from sample file", () => {
+        const invalidFlow = readSample("invalid-wrong-types.yaml");
+
         const result = validator.validateFlow(invalidFlow);
-        
+
         expect(result.errors.length).toBeGreaterThan(0);
-        expect(result.errors.some(error => 
-          error.instancePath === '/summary' && 
-          error.keyword === 'type'
-        )).toBe(true);
+        expect(
+          result.errors.some(
+            (error) =>
+              error.instancePath === "/summary" && error.keyword === "type"
+          )
+        ).toBe(true);
       });
 
-      it('should return errors for invalid language from sample file', () => {
-        const invalidFlow = readSample('invalid-language.yaml');
-        
+      it("should return errors for invalid language from sample file", () => {
+        const invalidFlow = readSample("invalid-language.yaml");
+
         const result = validator.validateFlow(invalidFlow);
-        
+
         expect(result.errors.length).toBeGreaterThan(0);
-        expect(result.errors.some(error => 
-          error.instancePath === '/value/modules/0/value/language' && 
-          error.keyword === 'enum'
-        )).toBe(true);
+        expect(
+          result.errors.some(
+            (error) =>
+              error.instancePath === "/value/modules/0/value/language" &&
+              error.keyword === "enum"
+          )
+        ).toBe(true);
       });
 
-      it('should handle empty file from sample', () => {
-        const emptyFlow = readSample('empty.yaml');
-        
+      it("should handle empty file from sample", () => {
+        const emptyFlow = readSample("empty.yaml");
+
         const result = validator.validateFlow(emptyFlow);
-        
+
         expect(result.parsed).toBeDefined();
         expect(result.errors.length).toBeGreaterThan(0);
       });
 
-      it('should handle complex invalid flow with comprehensive error detection', () => {
-        const complexInvalidFlow = readSample('invalid-complex-flow.yaml');
-        
+      it("should handle complex invalid flow with comprehensive error detection", () => {
+        const complexInvalidFlow = readSample("invalid-complex-flow.yaml");
+
         const result = validator.validateFlow(complexInvalidFlow);
-        
+
         expect(result.errors.length).toBeGreaterThan(20); // Should have many errors
-        
+
         // Check for missing required fields errors
-        const missingRequiredErrors = result.errors.filter(error => 
-          error.keyword === 'required'
+        const missingRequiredErrors = result.errors.filter(
+          (error) => error.keyword === "required"
         );
         expect(missingRequiredErrors.length).toBeGreaterThan(10);
-        
+
         // Check for invalid enum errors (invalid language)
-        const enumErrors = result.errors.filter(error => 
-          error.keyword === 'enum'
+        const enumErrors = result.errors.filter(
+          (error) => error.keyword === "enum"
         );
         expect(enumErrors.length).toBeGreaterThan(0);
-        
+
         // Check for type errors (string vs number, boolean vs string, etc.)
-        const typeErrors = result.errors.filter(error => 
-          error.keyword === 'type'
+        const typeErrors = result.errors.filter(
+          (error) => error.keyword === "type"
         );
         expect(typeErrors.length).toBeGreaterThan(5);
-        
+
         // Should detect invalid forloop structure (by checking for forloop-related required fields)
-        const forloopRequiredErrors = result.errors.filter(error => 
-          error.keyword === 'required' && error.message &&
-          (error.message.includes('modules') || error.message.includes('iterator') || error.message.includes('skip_failures'))
+        const forloopRequiredErrors = result.errors.filter(
+          (error) =>
+            error.keyword === "required" &&
+            error.message &&
+            (error.message.includes("modules") ||
+              error.message.includes("iterator") ||
+              error.message.includes("skip_failures"))
         );
         expect(forloopRequiredErrors.length).toBeGreaterThan(0);
-        
+
         // Should detect invalid branch structure
-        const branchRequiredErrors = result.errors.filter(error => 
-          error.keyword === 'required' && error.message &&
-          (error.message.includes('branches') || error.message.includes('default') || error.message.includes('expr'))
+        const branchRequiredErrors = result.errors.filter(
+          (error) =>
+            error.keyword === "required" &&
+            error.message &&
+            (error.message.includes("branches") ||
+              error.message.includes("default") ||
+              error.message.includes("expr"))
         );
         expect(branchRequiredErrors.length).toBeGreaterThan(0);
-        
+
         // Should detect discriminator errors for invalid transform types
-        const discriminatorErrors = result.errors.filter(error => 
-          error.keyword === 'discriminator'
+        const discriminatorErrors = result.errors.filter(
+          (error) => error.keyword === "discriminator"
         );
         expect(discriminatorErrors.length).toBeGreaterThan(0);
       });
 
-      it('should handle deeply nested invalid structures with detailed error reporting', () => {
-        const nestedInvalidFlow = readSample('invalid-nested-structures.yaml');
-        
+      it("should handle deeply nested invalid structures with detailed error reporting", () => {
+        const nestedInvalidFlow = readSample("invalid-nested-structures.yaml");
+
         const result = validator.validateFlow(nestedInvalidFlow);
-        
+
         expect(result.errors.length).toBeGreaterThan(10); // Should have many nested errors
-        
+
         // Check for deeply nested path errors (paths with many levels)
-        const deepNestedErrors = result.errors.filter(error => 
-          error.instancePath.split('/').length > 6 // Deep nesting
+        const deepNestedErrors = result.errors.filter(
+          (error) => error.instancePath.split("/").length > 6 // Deep nesting
         );
         expect(deepNestedErrors.length).toBeGreaterThan(0);
-        
+
         // Check for transform-related errors
-        const transformErrors = result.errors.filter(error => 
-          error.instancePath.includes('input_transforms') || 
-          error.instancePath.includes('iterator')
+        const transformErrors = result.errors.filter(
+          (error) =>
+            error.instancePath.includes("input_transforms") ||
+            error.instancePath.includes("iterator")
         );
         expect(transformErrors.length).toBeGreaterThan(0);
-        
+
         // Check for type errors in general
-        const typeErrors = result.errors.filter(error => 
-          error.keyword === 'type'
+        const typeErrors = result.errors.filter(
+          (error) => error.keyword === "type"
         );
         expect(typeErrors.length).toBeGreaterThan(0);
-        
+
         // Check for missing required fields in nested structures
-        const nestedRequiredErrors = result.errors.filter(error => 
-          error.keyword === 'required' && 
-          error.instancePath.includes('/modules/')
+        const nestedRequiredErrors = result.errors.filter(
+          (error) =>
+            error.keyword === "required" &&
+            error.instancePath.includes("/modules/")
         );
         expect(nestedRequiredErrors.length).toBeGreaterThan(0);
-        
+
         // Check for discriminator errors in nested transforms
-        const nestedDiscriminatorErrors = result.errors.filter(error => 
-          error.keyword === 'discriminator'
+        const nestedDiscriminatorErrors = result.errors.filter(
+          (error) => error.keyword === "discriminator"
         );
         expect(nestedDiscriminatorErrors.length).toBeGreaterThan(0);
       });
 
-      it('should provide specific error locations for complex validation failures', () => {
-        const complexInvalidFlow = readSample('invalid-complex-flow.yaml');
-        
+      it("should provide specific error locations for complex validation failures", () => {
+        const complexInvalidFlow = readSample("invalid-complex-flow.yaml");
+
         const result = validator.validateFlow(complexInvalidFlow);
-        
+
         // Verify that errors have meaningful instance paths
-        const errorsWithPaths = result.errors.filter(error => 
-          error.instancePath && error.instancePath.length > 0
+        const errorsWithPaths = result.errors.filter(
+          (error) => error.instancePath && error.instancePath.length > 0
         );
         expect(errorsWithPaths.length).toBeGreaterThan(5);
-        
+
         // Check that we can identify specific problematic modules
-        const moduleSpecificErrors = result.errors.filter(error => 
-          error.instancePath.includes('/value/modules/')
+        const moduleSpecificErrors = result.errors.filter((error) =>
+          error.instancePath.includes("/value/modules/")
         );
         expect(moduleSpecificErrors.length).toBeGreaterThan(0);
-        
+
         // Verify error messages are descriptive
-        const descriptiveErrors = result.errors.filter(error => 
-          error.message && error.message.length > 0
+        const descriptiveErrors = result.errors.filter(
+          (error) => error.message && error.message.length > 0
         );
         expect(descriptiveErrors.length).toBe(result.errors.length);
       });
 
-      it('should handle all major flow control structures with validation errors', () => {
-        const complexInvalidFlow = readSample('invalid-complex-flow.yaml');
-        
+      it("should handle all major flow control structures with validation errors", () => {
+        const complexInvalidFlow = readSample("invalid-complex-flow.yaml");
+
         const result = validator.validateFlow(complexInvalidFlow);
-        
+
         expect(result.errors.length).toBeGreaterThan(20);
-        
+
         // Should find errors related to flow control structures by checking instance paths
-        const flowControlErrors = result.errors.filter(error => 
-          error.instancePath.includes('/modules/2/') || // forloop module
-          error.instancePath.includes('/modules/3/') || // forloop module  
-          error.instancePath.includes('/modules/4/') || // branch module
-          error.instancePath.includes('/modules/5/') || // branch module
-          error.instancePath.includes('/modules/6/') || // branch all module
-          error.instancePath.includes('/modules/7/')    // while loop module
+        const flowControlErrors = result.errors.filter(
+          (error) =>
+            error.instancePath.includes("/modules/2/") || // forloop module
+            error.instancePath.includes("/modules/3/") || // forloop module
+            error.instancePath.includes("/modules/4/") || // branch module
+            error.instancePath.includes("/modules/5/") || // branch module
+            error.instancePath.includes("/modules/6/") || // branch all module
+            error.instancePath.includes("/modules/7/") // while loop module
         );
         expect(flowControlErrors.length).toBeGreaterThan(10);
-        
+
         // Should find errors in script and flow references
-        const pathReferenceErrors = result.errors.filter(error => 
-          error.instancePath.includes('/modules/8/') || // script path module
-          error.instancePath.includes('/modules/9/')    // flow path module
+        const pathReferenceErrors = result.errors.filter(
+          (error) =>
+            error.instancePath.includes("/modules/8/") || // script path module
+            error.instancePath.includes("/modules/9/") // flow path module
         );
         expect(pathReferenceErrors.length).toBeGreaterThan(0);
-        
+
         // Should detect modules with missing IDs
-        const missingIdErrors = result.errors.filter(error => 
-          error.keyword === 'required' && error.message &&
-          error.message.includes('id')
+        const missingIdErrors = result.errors.filter(
+          (error) =>
+            error.keyword === "required" &&
+            error.message &&
+            error.message.includes("id")
         );
         expect(missingIdErrors.length).toBeGreaterThan(0);
-        
+
         // Should detect type mismatches at the flow level
-        const flowLevelTypeErrors = result.errors.filter(error => 
-          error.instancePath.startsWith('/value/') && 
-          !error.instancePath.includes('/modules/') &&
-          error.keyword === 'type'
+        const flowLevelTypeErrors = result.errors.filter(
+          (error) =>
+            error.instancePath.startsWith("/value/") &&
+            !error.instancePath.includes("/modules/") &&
+            error.keyword === "type"
         );
         expect(flowLevelTypeErrors.length).toBeGreaterThan(0);
       });
     });
 
+    describe("AI agent flows", () => {
+      describe("valid AI agent flows", () => {
+        it("should validate a basic AI agent flow with FlowModule tools", () => {
+          const validFlow = readSample("valid-aiagent-basic.yaml");
+
+          const result = validator.validateFlow(validFlow);
+
+          expect(result.errors).toHaveLength(0);
+        });
+
+        it("should validate an AI agent flow with MCP tools", () => {
+          const validFlow = readSample("valid-aiagent-mcp.yaml");
+
+          const result = validator.validateFlow(validFlow);
+
+          expect(result.errors).toHaveLength(0);
+        });
+
+        it("should validate an AI agent flow with mixed FlowModule and MCP tools", () => {
+          const validFlow = readSample("valid-aiagent-mixed.yaml");
+
+          const result = validator.validateFlow(validFlow);
+
+          expect(result.errors).toHaveLength(0);
+        });
+
+        it("should validate an AI agent flow with parallel execution enabled", () => {
+          const validFlow = readSample("valid-aiagent-parallel.yaml");
+
+          const result = validator.validateFlow(validFlow);
+
+          expect(result.errors).toHaveLength(0);
+          // Verify tools array has expected structure
+          const agentModule = (result.parsed.data as any).value.modules[0];
+          expect(agentModule.value.tools).toHaveLength(2);
+        });
+      });
+
+      describe("invalid AI agent flows", () => {
+        it("should return errors for AI agent missing required tools field", () => {
+          const invalidFlow = readSample("invalid-aiagent-missing-tools.yaml");
+
+          const result = validator.validateFlow(invalidFlow);
+
+          expect(result.errors.length).toBeGreaterThan(0);
+          expect(
+            result.errors.some(
+              (error) =>
+                error.keyword === "required" &&
+                error.params?.missingProperty === "tools"
+            )
+          ).toBe(true);
+        });
+
+        it("should return errors for AI agent missing type field", () => {
+          const invalidFlow = readSample("invalid-aiagent-missing-type.yaml");
+
+          const result = validator.validateFlow(invalidFlow);
+
+          expect(result.errors.length).toBeGreaterThan(0);
+          // Should fail discriminator validation since type is missing
+          expect(
+            result.errors.some(
+              (error) =>
+                error.keyword === "required" ||
+                error.keyword === "discriminator"
+            )
+          ).toBe(true);
+        });
+
+        it("should return errors for AI agent with invalid tool_type", () => {
+          const invalidFlow = readSample(
+            "invalid-aiagent-invalid-tool-type.yaml"
+          );
+
+          const result = validator.validateFlow(invalidFlow);
+
+          expect(result.errors.length).toBeGreaterThan(0);
+          // Should fail discriminator validation for invalid tool_type
+          expect(
+            result.errors.some(
+              (error) =>
+                error.keyword === "discriminator" || error.keyword === "enum"
+            )
+          ).toBe(true);
+        });
+
+        it("should return errors for MCP tool missing resource_path", () => {
+          const invalidFlow = readSample(
+            "invalid-aiagent-mcp-missing-resource.yaml"
+          );
+
+          const result = validator.validateFlow(invalidFlow);
+
+          expect(result.errors.length).toBeGreaterThan(0);
+          expect(
+            result.errors.some(
+              (error) =>
+                error.keyword === "required" &&
+                error.params?.missingProperty === "resource_path"
+            )
+          ).toBe(true);
+        });
+      });
+    });
   });
-});
+});

windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-invalid-tool-type.yaml

@@ -0,0 +1,20 @@
+summary: Invalid AI Agent - Invalid Tool Type
+description: An AI agent with an invalid tool_type value
+value:
+  modules:
+    - id: invalid_tool_agent
+      value:
+        type: aiagent
+        input_transforms:
+          task:
+            type: static
+            value: "Perform task"
+        tools:
+          - id: bad_tool
+            summary: Tool with invalid type
+            value:
+              tool_type: invalid_type  # Should be 'flowmodule' or 'mcp'
+              type: rawscript
+              input_transforms: {}
+              content: "export async function main() { return {}; }"
+              language: deno

windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-mcp-missing-resource.yaml

@@ -0,0 +1,27 @@
+summary: Invalid AI Agent - MCP Missing Resource Path
+description: An AI agent with MCP tool missing required resource_path
+value:
+  modules:
+    - id: mcp_broken_agent
+      value:
+        type: aiagent
+        input_transforms:
+          instruction:
+            type: static
+            value: "Execute MCP tools"
+        tools:
+          - id: broken_mcp_tool
+            summary: MCP tool without resource_path
+            value:
+              tool_type: mcp
+              # Missing required 'resource_path' field
+              include_tools:
+                - search
+                - fetch
+          - id: valid_tool
+            value:
+              tool_type: flowmodule
+              type: rawscript
+              input_transforms: {}
+              content: "export async function main() { return {}; }"
+              language: deno

windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-tools.yaml

@@ -0,0 +1,12 @@
+summary: Invalid AI Agent - Missing Tools
+description: An AI agent without the required tools field
+value:
+  modules:
+    - id: broken_agent
+      value:
+        type: aiagent
+        input_transforms:
+          prompt:
+            type: static
+            value: "Test prompt"
+        # Missing required 'tools' field

windmill-yaml-validator/src/validation/__tests__/test-samples/invalid-aiagent-missing-type.yaml

@@ -0,0 +1,19 @@
+summary: Invalid AI Agent - Missing Type
+description: An AI agent module without the required type field
+value:
+  modules:
+    - id: typeless_agent
+      value:
+        # Missing required 'type' field
+        input_transforms:
+          query:
+            type: static
+            value: "Search query"
+        tools:
+          - id: tool_1
+            value:
+              tool_type: flowmodule
+              type: rawscript
+              input_transforms: {}
+              content: "export async function main() { return {}; }"
+              language: deno

windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml

@@ -0,0 +1,39 @@
+summary: Basic AI Agent Flow
+description: A flow with an AI agent using FlowModule tools
+schema:
+  type: object
+  properties:
+    prompt:
+      type: string
+      default: "Analyze the data"
+value:
+  modules:
+    - id: agent_step
+      value:
+        type: aiagent
+        input_transforms:
+          prompt:
+            type: static
+            value: "Please analyze the data"
+        tools:
+          - id: tool_1
+            summary: Data fetcher
+            value:
+              tool_type: flowmodule
+              type: rawscript
+              input_transforms: {}
+              content: |
+                export async function main() {
+                  return { data: [1, 2, 3] };
+                }
+              language: deno
+          - id: tool_2
+            summary: Calculator
+            value:
+              tool_type: flowmodule
+              type: script
+              path: f/scripts/calculator
+              input_transforms:
+                operation:
+                  type: javascript
+                  expr: "previous.result"

windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml

@@ -0,0 +1,27 @@
+summary: AI Agent with MCP Tools
+description: A flow with an AI agent using MCP server tools
+value:
+  modules:
+    - id: mcp_agent
+      value:
+        type: aiagent
+        input_transforms:
+          task:
+            type: static
+            value: "Search for information"
+        tools:
+          - id: mcp_search
+            summary: Search tool from MCP
+            value:
+              tool_type: mcp
+              resource_path: f/resources/mcp_server
+              include_tools:
+                - search
+                - fetch
+          - id: mcp_calculator
+            summary: Math operations
+            value:
+              tool_type: mcp
+              resource_path: f/resources/calculator_mcp
+              exclude_tools:
+                - deprecated_function

windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml

@@ -0,0 +1,46 @@
+summary: AI Agent with Mixed Tools
+description: An AI agent using both FlowModule and MCP tools
+schema:
+  type: object
+  properties:
+    user_query:
+      type: string
+value:
+  modules:
+    - id: mixed_agent
+      value:
+        type: aiagent
+        input_transforms:
+          query:
+            type: javascript
+            expr: "flow_input.user_query"
+        tools:
+          - id: custom_script
+            value:
+              tool_type: flowmodule
+              type: rawscript
+              input_transforms:
+                input:
+                  type: static
+                  value: "test"
+              content: |
+                export async function main(input: string) {
+                  return { processed: input };
+                }
+              language: python3
+          - id: mcp_tool
+            value:
+              tool_type: mcp
+              resource_path: f/resources/mcp_integration
+              include_tools:
+                - api_call
+          - id: flow_tool
+            summary: Nested flow
+            value:
+              tool_type: flowmodule
+              type: flow
+              path: f/flows/data_processor
+              input_transforms:
+                data:
+                  type: javascript
+                  expr: "previous_result.output"

windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml

@@ -0,0 +1,38 @@
+summary: AI Agent with Parallel Execution
+description: An AI agent configured to execute tools in parallel
+value:
+  modules:
+    - id: parallel_agent
+      value:
+        type: aiagent
+        parallel: true
+        input_transforms:
+          instruction:
+            type: static
+            value: "Process data in parallel"
+          model:
+            type: javascript
+            expr: "'gpt-4'"
+        tools:
+          - id: parallel_tool_1
+            summary: First parallel tool
+            value:
+              tool_type: flowmodule
+              type: rawscript
+              input_transforms: {}
+              content: |
+                export async function main() {
+                  return { result: "tool_1" };
+                }
+              language: bun
+          - id: parallel_tool_2
+            summary: Second parallel tool
+            value:
+              tool_type: flowmodule
+              type: rawscript
+              input_transforms: {}
+              content: |
+                export async function main() {
+                  return { result: "tool_2" };
+                }
+              language: deno