diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 379abdcf71..5f73780a10 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -19697,6 +19697,7 @@ components: description: True if script_path points to a flow, false if it points to a script args: $ref: "#/components/schemas/ScriptArgs" + nullable: true extra_perms: type: object additionalProperties: @@ -19707,57 +19708,74 @@ components: description: Email of the user who owns this schedule, used for permissioned_as error: type: string + nullable: true description: Last error message if the schedule failed to trigger on_failure: type: string + nullable: true description: Path to a script or flow to run when the scheduled job fails on_failure_times: type: number + nullable: true description: Number of consecutive failures before the on_failure handler is triggered (default 1) on_failure_exact: type: boolean + nullable: true description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N on_failure_extra_args: $ref: "#/components/schemas/ScriptArgs" + nullable: true on_recovery: type: string + nullable: true description: Path to a script or flow to run when the schedule recovers after failures on_recovery_times: type: number + nullable: true description: Number of consecutive successes before the on_recovery handler is triggered (default 1) on_recovery_extra_args: $ref: "#/components/schemas/ScriptArgs" + nullable: true on_success: type: string + nullable: true description: Path to a script or flow to run after each successful execution on_success_extra_args: $ref: "#/components/schemas/ScriptArgs" + nullable: true ws_error_handler_muted: type: boolean description: If true, the workspace-level error handler will not be triggered for this schedule's failures retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + nullable: true summary: type: string + nullable: true description: Short summary describing the purpose of this schedule description: type: string + nullable: true description: Detailed description of what this schedule does no_flow_overlap: type: boolean description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs) tag: type: string + nullable: true description: Worker tag to route jobs to specific worker groups paused_until: type: string format: date-time + nullable: true description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time cron_version: type: string + nullable: true description: Cron parser version. Use 'v2' for extended syntax with additional features dynamic_skip: type: string + nullable: true description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false) required: - path @@ -19819,60 +19837,77 @@ components: type: boolean description: True if script_path points to a flow, false if it points to a script args: + nullable: true $ref: "#/components/schemas/ScriptArgs" enabled: type: boolean description: Whether the schedule is currently active and will trigger jobs on_failure: type: string + nullable: true description: Path to a script or flow to run when the scheduled job fails on_failure_times: type: number + nullable: true description: Number of consecutive failures before the on_failure handler is triggered (default 1) on_failure_exact: type: boolean + nullable: true description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N on_failure_extra_args: + nullable: true $ref: "#/components/schemas/ScriptArgs" on_recovery: type: string + nullable: true description: Path to a script or flow to run when the schedule recovers after failures on_recovery_times: type: number + nullable: true description: Number of consecutive successes before the on_recovery handler is triggered (default 1) on_recovery_extra_args: + nullable: true $ref: "#/components/schemas/ScriptArgs" on_success: type: string + nullable: true description: Path to a script or flow to run after each successful execution on_success_extra_args: + nullable: true $ref: "#/components/schemas/ScriptArgs" ws_error_handler_muted: type: boolean description: If true, the workspace-level error handler will not be triggered for this schedule's failures retry: + nullable: true $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" no_flow_overlap: type: boolean description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs) summary: type: string + nullable: true description: Short summary describing the purpose of this schedule description: type: string + nullable: true description: Detailed description of what this schedule does tag: type: string + nullable: true description: Worker tag to route jobs to specific worker groups paused_until: type: string + nullable: true format: date-time description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time cron_version: type: string + nullable: true description: Cron parser version. Use 'v2' for extended syntax with additional features dynamic_skip: type: string + nullable: true description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false) required: - path @@ -19892,57 +19927,74 @@ components: type: string description: IANA timezone for the schedule (e.g., 'UTC', 'Europe/Paris', 'America/New_York') args: + nullable: true $ref: "#/components/schemas/ScriptArgs" on_failure: type: string + nullable: true description: Path to a script or flow to run when the scheduled job fails on_failure_times: type: number + nullable: true description: Number of consecutive failures before the on_failure handler is triggered (default 1) on_failure_exact: type: boolean + nullable: true description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N on_failure_extra_args: + nullable: true $ref: "#/components/schemas/ScriptArgs" on_recovery: type: string + nullable: true description: Path to a script or flow to run when the schedule recovers after failures on_recovery_times: type: number + nullable: true description: Number of consecutive successes before the on_recovery handler is triggered (default 1) on_recovery_extra_args: + nullable: true $ref: "#/components/schemas/ScriptArgs" on_success: type: string + nullable: true description: Path to a script or flow to run after each successful execution on_success_extra_args: + nullable: true $ref: "#/components/schemas/ScriptArgs" ws_error_handler_muted: type: boolean description: If true, the workspace-level error handler will not be triggered for this schedule's failures retry: + nullable: true $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" no_flow_overlap: type: boolean description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs) summary: type: string + nullable: true description: Short summary describing the purpose of this schedule description: type: string + nullable: true description: Detailed description of what this schedule does tag: type: string + nullable: true description: Worker tag to route jobs to specific worker groups paused_until: type: string + nullable: true format: date-time description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time cron_version: type: string + nullable: true description: Cron parser version. Use 'v2' for extended syntax with additional features dynamic_skip: type: string + nullable: true description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false) required: - schedule @@ -20154,6 +20206,7 @@ components: description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /. static_asset_config: type: object + nullable: true description: Configuration for serving static assets (s3 bucket, storage path, filename) properties: s3: @@ -20172,12 +20225,15 @@ components: description: HTTP method (get, post, put, delete, patch) that triggers this endpoint authentication_resource_path: type: string + nullable: true description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods) summary: type: string + nullable: true description: Short summary describing the purpose of this trigger description: type: string + nullable: true description: Detailed description of what this trigger does request_type: $ref: "#/components/schemas/HttpRequestType" @@ -20234,12 +20290,15 @@ components: description: If true, the route includes the workspace ID in the path summary: type: string + nullable: true description: Short summary describing the purpose of this trigger description: type: string + nullable: true description: Detailed description of what this trigger does static_asset_config: type: object + nullable: true description: Configuration for serving static assets (s3 bucket, storage path, filename) properties: s3: @@ -20261,6 +20320,7 @@ components: description: HTTP method (get, post, put, delete, patch) that triggers this endpoint authentication_resource_path: type: string + nullable: true description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods) is_async: type: boolean @@ -20315,15 +20375,18 @@ components: description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /. summary: type: string + nullable: true description: Short summary describing the purpose of this trigger description: type: string + nullable: true description: Detailed description of what this trigger does workspaced_route: type: boolean description: If true, the route includes the workspace ID in the path static_asset_config: type: object + nullable: true description: Configuration for serving static assets (s3 bucket, storage path, filename) properties: s3: @@ -20339,6 +20402,7 @@ components: - s3 authentication_resource_path: type: string + nullable: true description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods) is_flow: type: boolean @@ -20449,12 +20513,14 @@ components: - value initial_messages: type: array + nullable: true description: Messages to send immediately after connecting (can be raw strings or computed by runnables) items: $ref: "#/components/schemas/WebsocketTriggerInitialMessage" url_runnable_args: description: Arguments to pass to the script/flow that computes the WebSocket URL $ref: "#/components/schemas/ScriptArgs" + nullable: true can_return_message: type: boolean description: If true, the script can return a message to send back through the WebSocket @@ -20508,11 +20574,13 @@ components: - value initial_messages: type: array + nullable: true description: Messages to send immediately after connecting (can be raw strings or computed by runnables) items: $ref: "#/components/schemas/WebsocketTriggerInitialMessage" url_runnable_args: description: Arguments to pass to the script/flow that computes the WebSocket URL + nullable: true $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean @@ -20568,11 +20636,13 @@ components: - value initial_messages: type: array + nullable: true description: Messages to send immediately after connecting (can be raw strings or computed by runnables) items: $ref: "#/components/schemas/WebsocketTriggerInitialMessage" url_runnable_args: description: Arguments to pass to the script/flow that computes the WebSocket URL + nullable: true $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean @@ -20674,15 +20744,19 @@ components: description: Array of MQTT topics to subscribe to, each with topic name and QoS level v3_config: $ref: "#/components/schemas/MqttV3Config" + nullable: true description: MQTT v3 specific configuration (clean_session) v5_config: $ref: "#/components/schemas/MqttV5Config" + nullable: true description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval) client_id: type: string + nullable: true description: MQTT client ID for this connection client_version: $ref: "#/components/schemas/MqttClientVersion" + nullable: true description: MQTT protocol version ('v3' or 'v5') server_id: type: string @@ -20720,14 +20794,18 @@ components: description: Array of MQTT topics to subscribe to, each with topic name and QoS level client_id: type: string + nullable: true description: MQTT client ID for this connection v3_config: + nullable: true $ref: "#/components/schemas/MqttV3Config" description: MQTT v3 specific configuration (clean_session) v5_config: + nullable: true $ref: "#/components/schemas/MqttV5Config" description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval) client_version: + nullable: true $ref: "#/components/schemas/MqttClientVersion" description: MQTT protocol version ('v3' or 'v5') path: @@ -20770,14 +20848,18 @@ components: description: Array of MQTT topics to subscribe to, each with topic name and QoS level client_id: type: string + nullable: true description: MQTT client ID for this connection v3_config: + nullable: true $ref: "#/components/schemas/MqttV3Config" description: MQTT v3 specific configuration (clean_session) v5_config: + nullable: true $ref: "#/components/schemas/MqttV5Config" description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval) client_version: + nullable: true $ref: "#/components/schemas/MqttClientVersion" description: MQTT protocol version ('v3' or 'v5') path: @@ -20851,6 +20933,7 @@ components: $ref: "#/components/schemas/DeliveryType" delivery_config: $ref: "#/components/schemas/PushConfig" + nullable: true subscription_mode: $ref: "#/components/schemas/SubscriptionMode" last_server_ping: @@ -20904,6 +20987,7 @@ components: delivery_type: $ref: "#/components/schemas/DeliveryType" delivery_config: + nullable: true $ref: "#/components/schemas/PushConfig" path: type: string @@ -20980,6 +21064,7 @@ components: description: Path to the AWS resource containing credentials or OIDC configuration message_attributes: type: array + nullable: true items: type: string description: Array of SQS message attribute names to include with each message @@ -21071,6 +21156,7 @@ components: description: Path to the AWS resource containing credentials or OIDC configuration message_attributes: type: array + nullable: true items: type: string description: Array of SQS message attribute names to include with each message @@ -21116,6 +21202,7 @@ components: description: Path to the AWS resource containing credentials or OIDC configuration message_attributes: type: array + nullable: true items: type: string description: Array of SQS message attribute names to include with each message @@ -21522,9 +21609,11 @@ components: description: If true, uses NATS JetStream for durable message delivery stream_name: type: string + nullable: true description: JetStream stream name (required when use_jetstream is true) consumer_name: type: string + nullable: true description: JetStream consumer name (required when use_jetstream is true) subjects: type: array @@ -21576,9 +21665,11 @@ components: description: If true, uses NATS JetStream for durable message delivery stream_name: type: string + nullable: true description: JetStream stream name (required when use_jetstream is true) consumer_name: type: string + nullable: true description: JetStream consumer name (required when use_jetstream is true) subjects: type: array @@ -21616,9 +21707,11 @@ components: description: If true, uses NATS JetStream for durable message delivery stream_name: type: string + nullable: true description: JetStream stream name (required when use_jetstream is true) consumer_name: type: string + nullable: true description: JetStream consumer name (required when use_jetstream is true) subjects: type: array diff --git a/cli/README.md b/cli/README.md index 3b16447989..c7e3fda45b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -110,6 +110,43 @@ source <(wmill completions zsh) ## Development +### Testing with a local `windmill-yaml-validator` + +The CLI imports `windmill-yaml-validator` from npm (`npm:windmill-yaml-validator@1.1.0`). +To test local changes to the validator before publishing, use the Deno compatibility +script and import map override: + +1. Make the validator sources Deno-compatible: + +```bash +cd ../windmill-yaml-validator +./deno-compat.sh +``` + +2. Add the following entries to `cli/deno.json` imports: + +```json +"npm:windmill-yaml-validator@1.1.0": "../windmill-yaml-validator/src/index.ts", +"ajv": "npm:ajv@^8.17.1", +"@stoplight/yaml": "npm:@stoplight/yaml@^4.3.0" +``` + +3. Run the CLI directly with Deno: + +```bash +deno run -A src/main.ts lint +``` + +4. When done, restore everything: + +```bash +# Restore validator sources +cd ../windmill-yaml-validator +./deno-compat.sh -r + +# Remove the 3 import map lines from cli/deno.json +``` + ### Running Tests **Prerequisites:** diff --git a/cli/deno.json b/cli/deno.json index 01299b9802..0c14793da7 100644 --- a/cli/deno.json +++ b/cli/deno.json @@ -17,4 +17,4 @@ "ws": "npm:ws@8.18.0" }, "nodeModulesDir": "auto" -} +} \ No newline at end of file diff --git a/cli/deno.lock b/cli/deno.lock index 536c8960c8..9fcd2b7a64 100644 --- a/cli/deno.lock +++ b/cli/deno.lock @@ -86,6 +86,8 @@ "npm:open@*": "10.2.0", "npm:svelte-preprocess@6.0.3": "6.0.3_svelte@5.45.2__acorn@8.14.1", "npm:svelte@5.45.2": "5.45.2_acorn@8.14.1", + "npm:windmill-yaml-validator@1.1.0": "1.1.0", + "npm:windmill-yaml-validator@1.1.1": "1.1.1", "npm:ws@*": "8.18.3", "npm:ws@8.18.0": "8.18.0", "npm:ws@8.18.3": "8.18.3" @@ -540,6 +542,28 @@ "@jridgewell/sourcemap-codec" ] }, + "@stoplight/ordered-object-literal@1.0.5": { + "integrity": "sha512-COTiuCU5bgMUtbIFBuyyh2/yVVzlr5Om0v5utQDgBCuQUOPgU1DwoffkTfg4UBQOvByi5foF4w4T+H9CoRe5wg==" + }, + "@stoplight/types@14.1.1": { + "integrity": "sha512-/kjtr+0t0tjKr+heVfviO9FrU/uGLc+QNX3fHJc19xsCNYqU7lVhaXxDmEID9BZTjG+/r9pK9xP/xU02XGg65g==", + "dependencies": [ + "@types/json-schema", + "utility-types" + ] + }, + "@stoplight/yaml-ast-parser@0.0.50": { + "integrity": "sha512-Pb6M8TDO9DtSVla9yXSTAxmo9GVEouq5P40DWXdOie69bXogZTkgvopCq+yEvTMA0F6PEvdJmbtTV3ccIp11VQ==" + }, + "@stoplight/yaml@4.3.0": { + "integrity": "sha512-JZlVFE6/dYpP9tQmV0/ADfn32L9uFarHWxfcRhReKUnljz1ZiUM5zpX+PH8h5CJs6lao3TuFqnPm9IJJCEkE2w==", + "dependencies": [ + "@stoplight/ordered-object-literal", + "@stoplight/types", + "@stoplight/yaml-ast-parser", + "tslib" + ] + }, "@sveltejs/acorn-typescript@1.0.7_acorn@8.14.1": { "integrity": "sha512-znp1A/Y1Jj4l/Zy7PX5DZKBE0ZNY+5QBngiE21NJkfSTyzzC5iKNWOtwFXKtIrn7MXEFBck4jD95iBNkGjK92Q==", "dependencies": [ @@ -552,6 +576,9 @@ "@types/estree@1.0.8": { "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==" }, + "@types/json-schema@7.0.15": { + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" + }, "@types/node@24.2.0": { "integrity": "sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==", "dependencies": [ @@ -652,6 +679,15 @@ "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "bin": true }, + "ajv@8.17.1": { + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "dependencies": [ + "fast-deep-equal", + "fast-uri", + "json-schema-traverse", + "require-from-string" + ] + }, "aria-query@5.3.2": { "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==" }, @@ -882,6 +918,12 @@ "vary" ] }, + "fast-deep-equal@3.1.3": { + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "fast-uri@3.1.0": { + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==" + }, "finalhandler@2.1.0": { "integrity": "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q==", "dependencies": [ @@ -993,6 +1035,9 @@ "isarray@1.0.0": { "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, + "json-schema-traverse@1.0.0": { + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "jszip@3.7.1": { "integrity": "sha512-ghL0tz1XG9ZEmRMcEN2vt7xabrDdqHHeykgARpmZ0BiIctWxM47Vt63ZO2dnp4QYt/xJVLLy5Zv1l/xRdh2byg==", "dependencies": [ @@ -1144,6 +1189,9 @@ "util-deprecate" ] }, + "require-from-string@2.0.2": { + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" + }, "router@2.2.0": { "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", "dependencies": [ @@ -1279,6 +1327,9 @@ "toidentifier@1.0.1": { "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" }, + "tslib@2.8.1": { + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, "type-is@2.0.1": { "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "dependencies": [ @@ -1300,6 +1351,9 @@ "util-deprecate@1.0.2": { "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "utility-types@3.11.0": { + "integrity": "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw==" + }, "vary@1.1.2": { "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, @@ -1320,6 +1374,20 @@ "windmill-client@1.515.1": { "integrity": "sha512-o6qynOEbPubZTZUOLLs2Z9f+uBZQJUCw/+YWgvI6p8nu5BJ6J3N/wEfbY1X5TTnJNuqahQ0UgimYzhurT5XQFw==" }, + "windmill-yaml-validator@1.1.0": { + "integrity": "sha512-TM9rl6NycP4eXYOzi4Y8/EXHU4phzFUJWN28IlHDx4eRDNPEkj+6jAF4xUaBvLeFEJl0CfznEdhtM61vDgomKQ==", + "dependencies": [ + "@stoplight/yaml", + "ajv" + ] + }, + "windmill-yaml-validator@1.1.1": { + "integrity": "sha512-CVgAwEoBdJhF39q2N012QffhlGPRIyIWd8gj7NnfG+/lMWgH2k5CBLtKIt6cPF8Bxz+6DGC3st1ARSsecDtbTg==", + "dependencies": [ + "@stoplight/yaml", + "ajv" + ] + }, "wrappy@1.0.2": { "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts new file mode 100644 index 0000000000..58d456bed0 --- /dev/null +++ b/cli/src/commands/lint/lint.ts @@ -0,0 +1,273 @@ +import { colors, Command, log, path, SEP } from "../../../deps.ts"; +import { GlobalOptions } from "../../types.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import { + FSFSElement, + ignoreF, + readDirRecursiveWithIgnore, +} from "../sync/sync.ts"; +import { + getValidationTargetFromFilename, + type ValidationTarget, + WindmillYamlValidator, +} from "npm:windmill-yaml-validator@1.1.1"; + +interface LintOptions extends GlobalOptions { + json?: boolean; + failOnWarn?: boolean; +} + +interface FileIssue { + path: string; + target: string; + errors: string[]; +} + +interface LintWarning { + path: string; + message: string; +} + +export interface LintReport { + scannedFiles: number; + validatedFiles: number; + validFiles: number; + invalidFiles: number; + skippedUnsupportedFiles: number; + warnings: LintWarning[]; + issues: FileIssue[]; + success: boolean; + exitCode: number; +} + +const YAML_FILE_REGEX = /\.ya?ml$/i; +const NATIVE_TRIGGER_REGEX = /\.[^.]+_native_trigger\.ya?ml$/i; + +function normalizePath(p: string): string { + return p.replaceAll(SEP, "/"); +} + +function isUnsupportedTriggerPath(filePath: string): boolean { + return NATIVE_TRIGGER_REGEX.test(filePath); +} + +function formatTarget(target: ValidationTarget): string { + if (target.type === "trigger") { + return `${target.triggerKind}_trigger`; + } + return target.type; +} + +export function formatValidationError(error: { + instancePath?: string; + keyword?: string; + message?: string; + params?: { + missingProperty?: string; + allowedValues?: unknown[]; + additionalProperty?: string; + }; +}): string { + const instancePath = error.instancePath && error.instancePath.length > 0 + ? error.instancePath + : "/"; + if (error.keyword === "required" && error.params?.missingProperty) { + return `${instancePath} missing required property '${error.params.missingProperty}'`; + } + if ( + error.keyword === "additionalProperties" && + error.params?.additionalProperty + ) { + return `${instancePath} has unknown property '${error.params.additionalProperty}'`; + } + if (error.keyword === "enum" && error.params?.allowedValues) { + const allowed = error.params.allowedValues + .filter((v) => v !== null) + .map((v) => `'${v}'`) + .join(", "); + return `${instancePath} must be one of: ${allowed}`; + } + if (error.message) { + return `${instancePath} ${error.message}`; + } + return `${instancePath} validation error`; +} + +function formatYamlDiagnostics(parsed: { diagnostics?: Array<{ message?: string }> }): string[] { + const diagnostics = parsed?.diagnostics; + if (!Array.isArray(diagnostics) || diagnostics.length === 0) { + return []; + } + return diagnostics.map((d) => d?.message || "Invalid YAML document"); +} + +export async function runLint( + opts: LintOptions, + directory?: string, +): Promise { + const initialCwd = Deno.cwd(); + const explicitTargetDirectory = directory + ? path.resolve(initialCwd, directory) + : undefined; + + const { json: _json, ...syncOpts } = opts; + const mergedOpts = await mergeConfigWithConfigFile(syncOpts); + const targetDirectory = explicitTargetDirectory ?? Deno.cwd(); + + const stats = await Deno.stat(targetDirectory).catch(() => null); + if (!stats) { + throw new Error(`Directory not found: ${targetDirectory}`); + } + if (!stats.isDirectory) { + throw new Error(`Path is not a directory: ${targetDirectory}`); + } + + const ignore = await ignoreF(mergedOpts); + const root = await FSFSElement(targetDirectory, [], false); + const validator = new WindmillYamlValidator(); + + const warnings: LintWarning[] = []; + const issues: FileIssue[] = []; + let scannedFiles = 0; + let validatedFiles = 0; + let validFiles = 0; + let skippedUnsupportedFiles = 0; + + for await (const entry of readDirRecursiveWithIgnore(ignore, root)) { + if (entry.isDirectory || entry.ignored) { + continue; + } + scannedFiles += 1; + + const normalizedPath = normalizePath(entry.path); + if (!YAML_FILE_REGEX.test(normalizedPath)) { + continue; + } + + if (isUnsupportedTriggerPath(normalizedPath)) { + warnings.push({ + path: normalizedPath, + message: + "Unsupported trigger schema for linting (native triggers are skipped)", + }); + skippedUnsupportedFiles += 1; + continue; + } + + const target = getValidationTargetFromFilename(normalizedPath); + if (!target) { + continue; + } + + validatedFiles += 1; + const content = await entry.getContentText(); + const result = validator.validate(content, target); + + const fileErrors = [ + ...formatYamlDiagnostics(result.parsed), + ...result.errors.map((error) => formatValidationError(error)), + ]; + if (fileErrors.length > 0) { + issues.push({ + path: normalizedPath, + target: formatTarget(target), + errors: fileErrors, + }); + } else { + validFiles += 1; + } + } + + const invalidFiles = issues.length; + const shouldFail = invalidFiles > 0 || + (!!opts.failOnWarn && warnings.length > 0); + + return { + scannedFiles, + validatedFiles, + validFiles, + invalidFiles, + skippedUnsupportedFiles, + warnings, + issues, + success: !shouldFail, + exitCode: shouldFail ? 1 : 0, + }; +} + +export function printReport(report: LintReport, jsonOutput: boolean) { + if (jsonOutput) { + console.log(JSON.stringify(report, null, 2)); + return; + } + + if (report.warnings.length > 0) { + log.info(colors.yellow(`\n⚠️ Warnings (${report.warnings.length}):`)); + for (const warning of report.warnings) { + log.info(colors.yellow(` - ${warning.path}: ${warning.message}`)); + } + } + + if (report.issues.length > 0) { + log.info(colors.red(`\n❌ Invalid files (${report.issues.length}):`)); + for (const issue of report.issues) { + log.info(colors.red(` - ${issue.path} (${issue.target})`)); + for (const error of issue.errors) { + log.info(colors.red(` • ${error}`)); + } + } + } + + if (report.success) { + log.info( + colors.green( + `\n✅ Lint passed (${report.validatedFiles} file(s) validated, ${report.validFiles} valid)\n`, + ), + ); + } else { + log.info( + colors.red( + `\n❌ Lint failed (${report.invalidFiles} invalid file(s), ${report.warnings.length} warning(s))\n`, + ), + ); + } +} + +async function lint(opts: LintOptions, directory?: string) { + try { + const report = await runLint(opts, directory); + printReport(report, !!opts.json); + if (report.exitCode !== 0) { + Deno.exit(report.exitCode); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (opts.json) { + console.log( + JSON.stringify( + { + success: false, + exitCode: 1, + error: message, + }, + null, + 2, + ), + ); + } else { + log.error(colors.red(`❌ ${message}`)); + } + Deno.exit(1); + } +} + +const command = new Command() + .description( + "Validate Windmill flow, schedule, and trigger YAML files in a directory", + ) + .arguments("[directory:string]") + .option("--json", "Output results in JSON format") + .option("--fail-on-warn", "Exit with code 1 when warnings are emitted") + .action(lint as any); + +export default command; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 0af6b725d0..10b4b856d8 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -25,6 +25,7 @@ import { extractNativeTriggerInfo, } from "../../types.ts"; import { downloadZip } from "./pull.ts"; +import { runLint, printReport } from "../lint/lint.ts"; import { exts, @@ -2209,6 +2210,16 @@ export async function push( // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(originalCliOpts, effectiveOpts); + if (opts.lint) { + log.info("Running lint validation before push..."); + const lintReport = await runLint(opts); + printReport(lintReport, !!opts.jsonOutput); + if (!lintReport.success) { + log.error(colors.red("Push aborted due to lint failures.")); + Deno.exit(1); + } + } + const codebases = await listSyncCodebases(opts); if (opts.raw) { log.info("--raw is now the default, you can remove it as a flag"); @@ -3055,6 +3066,7 @@ const command = new Command() "--branch ", "Override the current git branch (works even outside a git repository)", ) + .option("--lint", "Run lint validation before pushing") // deno-lint-ignore no-explicit-any .action(push as any); diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 5fc9c1b452..dd38adbd0d 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -96,6 +96,7 @@ export interface SyncOptions { }; }; promotion?: string; + lint?: boolean; } export interface Codebase { diff --git a/cli/src/main.ts b/cli/src/main.ts index f159044ca3..7a81e9b288 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -30,6 +30,7 @@ import sync from "./commands/sync/sync.ts"; import gitsyncSettings from "./commands/gitsync-settings/gitsync-settings.ts"; import instance from "./commands/instance/instance.ts"; import workerGroups from "./commands/worker-groups/worker-groups.ts"; +import lint from "./commands/lint/lint.ts"; import dev from "./commands/dev/dev.ts"; import { GlobalOptions } from "./types.ts"; @@ -61,6 +62,7 @@ export { schedule, trigger, sync, + lint, gitsyncSettings, instance, dev, @@ -131,6 +133,7 @@ const command = new Command() .command("trigger", trigger) .command("dev", dev) .command("sync", sync) + .command("lint", lint) .command("gitsync-settings", gitsyncSettings) .command("instance", instance) .command("worker-groups", workerGroups) diff --git a/cli/test/lint_command.test.ts b/cli/test/lint_command.test.ts new file mode 100644 index 0000000000..f7871c7034 --- /dev/null +++ b/cli/test/lint_command.test.ts @@ -0,0 +1,354 @@ +import { + assert, + assertEquals, + assertStringIncludes, +} from "https://deno.land/std@0.224.0/assert/mod.ts"; +import { + formatValidationError, + runLint, +} from "../src/commands/lint/lint.ts"; + +async function withTempDir( + fn: (tempDir: string) => Promise, +): Promise { + const tempDir = await Deno.makeTempDir({ prefix: "wmill_lint_test_" }); + const originalCwd = Deno.cwd(); + try { + Deno.chdir(tempDir); + await fn(tempDir); + } finally { + Deno.chdir(originalCwd); + await Deno.remove(tempDir, { recursive: true }); + } +} + +Deno.test("lint: validates flow, schedule, and trigger yaml files", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_flow.flow`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/flow.yaml`, + `summary: My flow +value: + modules: [] +`, + ); + + await Deno.mkdir(`${tempDir}/f/jobs`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/jobs/daily.schedule.yaml`, + `schedule: "0 0 12 * * *" +timezone: "UTC" +enabled: true +script_path: "f/jobs/daily_sync" +is_flow: false +`, + ); + + await Deno.mkdir(`${tempDir}/f/triggers`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/triggers/hook.http_trigger.yaml`, + `script_path: "f/triggers/http_handler" +is_flow: false +route_path: "api/webhook" +request_type: "sync" +authentication_method: "none" +http_method: "post" +is_static_website: false +workspaced_route: false +wrap_body: false +raw_string: false +`, + ); + + await Deno.writeTextFile( + `${tempDir}/f/triggers/inbox.email_trigger.yaml`, + `script_path: "f/triggers/email_handler" +is_flow: false +local_part: "inbox" +`, + ); + + const report = await runLint({} as any, tempDir); + + assertEquals(report.exitCode, 0); + assertEquals(report.validatedFiles, 4); + assertEquals(report.validFiles, 4); + assertEquals(report.invalidFiles, 0); + assertEquals(report.warnings.length, 0); + }); +}); + +Deno.test("lint: returns errors for invalid schedule documents", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/jobs`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/jobs/broken.schedule.yaml`, + `timezone: "UTC" +enabled: true +script_path: "f/jobs/broken" +is_flow: false +`, + ); + + const report = await runLint({} as any, tempDir); + + assertEquals(report.exitCode, 1); + assertEquals(report.validatedFiles, 1); + assertEquals(report.invalidFiles, 1); + assertEquals(report.issues[0].path, "f/jobs/broken.schedule.yaml"); + assert( + report.issues[0].errors.some((message) => + message.includes("missing required property 'schedule'") + ), + ); + }); +}); + +Deno.test("lint: warns and skips unsupported native trigger schemas", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/triggers`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/triggers/webhook.script.123.nextcloud_native_trigger.yaml`, + `path: "f/triggers/native" +`, + ); + + const report = await runLint({} as any, tempDir); + + assertEquals(report.exitCode, 0); + assertEquals(report.validatedFiles, 0); + assertEquals(report.skippedUnsupportedFiles, 1); + assertEquals(report.warnings.length, 1); + assertStringIncludes( + report.warnings[0].message, + "Unsupported trigger schema", + ); + + const failOnWarnReport = await runLint( + { failOnWarn: true } as any, + tempDir, + ); + assertEquals(failOnWarnReport.exitCode, 1); + }); +}); + +Deno.test("lint: uses wmill.yaml include filters for file discovery", async () => { + await withTempDir(async (tempDir) => { + await Deno.writeTextFile( + `${tempDir}/wmill.yaml`, + `defaultTs: bun +includes: + - "f/allowed/**" +excludes: [] +`, + ); + + await Deno.mkdir(`${tempDir}/f/allowed`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/allowed/ok.schedule.yaml`, + `schedule: "0 0 12 * * *" +timezone: "UTC" +enabled: true +script_path: "f/jobs/ok" +is_flow: false +`, + ); + + await Deno.mkdir(`${tempDir}/f/blocked`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/blocked/bad.schedule.yaml`, + `timezone: "UTC" +enabled: true +script_path: "f/jobs/bad" +is_flow: false +`, + ); + + const report = await runLint({} as any, tempDir); + + assertEquals(report.exitCode, 0); + assertEquals(report.validatedFiles, 1); + assertEquals(report.validFiles, 1); + assertEquals(report.invalidFiles, 0); + assertEquals(report.issues.length, 0); + }); +}); + +// --- formatValidationError unit tests --- + +Deno.test("formatValidationError: required keyword", () => { + assertEquals( + formatValidationError({ + instancePath: "/value", + keyword: "required", + message: "must have required property 'modules'", + params: { missingProperty: "modules" }, + }), + "/value missing required property 'modules'", + ); +}); + +Deno.test("formatValidationError: additionalProperties keyword", () => { + assertEquals( + formatValidationError({ + instancePath: "/value", + keyword: "additionalProperties", + message: "must NOT have additional properties", + params: { additionalProperty: "typo_field" }, + }), + "/value has unknown property 'typo_field'", + ); +}); + +Deno.test("formatValidationError: enum keyword filters null values", () => { + assertEquals( + formatValidationError({ + instancePath: "/http_method", + keyword: "enum", + message: "must be equal to one of the allowed values", + params: { allowedValues: [null, "get", "post", "put"] }, + }), + "/http_method must be one of: 'get', 'post', 'put'", + ); +}); + +Deno.test("formatValidationError: falls back to message", () => { + assertEquals( + formatValidationError({ + instancePath: "/timeout", + keyword: "type", + message: "must be integer", + }), + "/timeout must be integer", + ); +}); + +Deno.test("formatValidationError: uses / for empty instancePath", () => { + assertEquals( + formatValidationError({ + instancePath: "", + keyword: "required", + message: "must have required property 'summary'", + params: { missingProperty: "summary" }, + }), + "/ missing required property 'summary'", + ); +}); + +Deno.test("formatValidationError: generic fallback when no message", () => { + assertEquals( + formatValidationError({ instancePath: "/field", keyword: "custom" }), + "/field validation error", + ); +}); + +// --- runLint integration tests --- + +Deno.test("lint: throws for non-existent directory", async () => { + let threw = false; + try { + await runLint({} as any, "/tmp/wmill_lint_nonexistent_" + Date.now()); + } catch (e) { + threw = true; + assertStringIncludes((e as Error).message, "Directory not found"); + } + assert(threw, "Expected runLint to throw for non-existent directory"); +}); + +Deno.test("lint: json-shaped report contains all fields", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/jobs`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/jobs/ok.schedule.yaml`, + `schedule: "0 0 * * *" +timezone: "UTC" +enabled: true +script_path: "f/jobs/ok" +is_flow: false +`, + ); + + const report = await runLint({ json: true } as any, tempDir); + + // Verify the report object has the shape expected by --json output + assertEquals(typeof report.scannedFiles, "number"); + assertEquals(typeof report.validatedFiles, "number"); + assertEquals(typeof report.validFiles, "number"); + assertEquals(typeof report.invalidFiles, "number"); + assertEquals(typeof report.skippedUnsupportedFiles, "number"); + assert(Array.isArray(report.warnings)); + assert(Array.isArray(report.issues)); + assertEquals(typeof report.success, "boolean"); + assertEquals(typeof report.exitCode, "number"); + + // JSON.stringify should round-trip cleanly + const json = JSON.parse(JSON.stringify(report)); + assertEquals(json.success, true); + assertEquals(json.exitCode, 0); + }); +}); + +Deno.test("lint: --fail-on-warn with mixed valid and warning files", async () => { + await withTempDir(async (tempDir) => { + // A valid schedule + await Deno.mkdir(`${tempDir}/f/jobs`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/jobs/ok.schedule.yaml`, + `schedule: "0 0 * * *" +timezone: "UTC" +enabled: true +script_path: "f/jobs/ok" +is_flow: false +`, + ); + + // An unsupported native trigger that produces a warning + await Deno.mkdir(`${tempDir}/f/triggers`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/triggers/webhook.script.123.nextcloud_native_trigger.yaml`, + `path: "f/triggers/native" +`, + ); + + // Without --fail-on-warn: passes + const normalReport = await runLint({} as any, tempDir); + assertEquals(normalReport.exitCode, 0); + assertEquals(normalReport.success, true); + assertEquals(normalReport.validFiles, 1); + assertEquals(normalReport.warnings.length, 1); + + // With --fail-on-warn: fails due to warning + const strictReport = await runLint({ failOnWarn: true } as any, tempDir); + assertEquals(strictReport.exitCode, 1); + assertEquals(strictReport.success, false); + assertEquals(strictReport.validFiles, 1); + assertEquals(strictReport.warnings.length, 1); + }); +}); + +Deno.test("lint: reports enum errors with allowed values for invalid trigger", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/triggers`, { recursive: true }); + await Deno.writeTextFile( + `${tempDir}/f/triggers/hook.http_trigger.yaml`, + `script_path: "f/triggers/http_handler" +is_flow: false +route_path: "api/webhook" +authentication_method: "none" +http_method: "invalid_method" +is_static_website: false +workspaced_route: false +wrap_body: false +raw_string: false +`, + ); + + const report = await runLint({} as any, tempDir); + + assertEquals(report.invalidFiles, 1); + assert( + report.issues[0].errors.some((msg) => msg.includes("must be one of:")), + `Expected 'must be one of' error but got: ${report.issues[0].errors}`, + ); + }); +}); diff --git a/frontend/src/lib/components/flows/scheduleUtils.ts b/frontend/src/lib/components/flows/scheduleUtils.ts index ad35e5f61f..d158756f22 100644 --- a/frontend/src/lib/components/flows/scheduleUtils.ts +++ b/frontend/src/lib/components/flows/scheduleUtils.ts @@ -23,7 +23,7 @@ export async function loadSchedule(path: string, workspace: string): Promise - import { HttpTriggerService, type EditHttpTrigger, type NewHttpTrigger } from '$lib/gen' + import { HttpTriggerService, type EditHttpTrigger, type HttpTrigger, type NewHttpTrigger } from '$lib/gen' import { Pen, Save } from 'lucide-svelte' import Button from '../../common/button/Button.svelte' import ToggleButton from '../../common/toggleButton-v2/ToggleButton.svelte' @@ -42,7 +42,7 @@ let isCreating = $state(false) let acceptedFileTypes: string[] = ['.json', '.yaml'] - let callback: ((cfg: NewHttpTrigger | EditHttpTrigger) => void) | undefined = $state(undefined) + let callback: ((cfg: HttpTrigger | EditHttpTrigger) => void) | undefined = $state(undefined) let lang: 'yaml' | 'json' = $derived.by(() => { if (code.trimStart().startsWith('{')) { diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 0b7734b8d5..4aca94c4b5 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -288,19 +288,19 @@ schedule = s?.schedule ?? '0 0 12 * *' initialSchedule = schedule timezone = s?.timezone ?? Intl.DateTimeFormat().resolvedOptions().timeZone - paused_until = s?.paused_until + paused_until = s?.paused_until ?? undefined showPauseUntil = paused_until !== undefined summary = s?.summary ?? '' description = s?.description ?? '' script_path = s?.script_path ?? initialScriptPath args = s?.args ?? {} - tag = s?.tag + tag = s?.tag ?? undefined await loadScript(script_path) no_flow_overlap = s?.no_flow_overlap ?? false wsErrorHandlerMuted = s?.ws_error_handler_muted ?? false - retry = s?.retry + retry = s?.retry ?? undefined await setScheduleHandler(s) } finally { diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 42ab22ab4a..e9f2328b71 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -105,7 +105,7 @@ export async function loadScriptSchedule( }) return { - summary: schedule.summary, + summary: schedule.summary ?? undefined, enabled: schedule.enabled, cron: schedule.schedule, timezone: schedule.timezone, diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index cd67ddab30..ab7a80051b 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -153,6 +153,7 @@ export async function getTriggersDeployData(kind: TriggerKind, path: string, wor const data: GcpTriggerData = { ...gcpTrigger, + delivery_config: gcpTrigger.delivery_config ?? undefined, base_endpoint: gcpTrigger.delivery_type === 'push' ? `${window.location.origin}${base}` : undefined } @@ -540,7 +541,7 @@ export async function getTriggerDependency(kind: TriggerKind, path: string, work result = retrieveKindsValues({ script_path, is_flow, - resource_path: authentication_resource_path + resource_path: authentication_resource_path ?? undefined }) } else if (kind === 'schedules') { const { script_path, is_flow } = await ScheduleService.getSchedule({ diff --git a/windmill-yaml-validator/README.md b/windmill-yaml-validator/README.md index 0921482b36..6a693c8200 100644 --- a/windmill-yaml-validator/README.md +++ b/windmill-yaml-validator/README.md @@ -1,14 +1,19 @@ # Windmill YAML Validator -A TypeScript-based YAML validator for Windmill flow files. This package validates flow.yaml files against the OpenFlow JSON schema to ensure they conform to the Windmill flow specification. +A TypeScript-based YAML validator for Windmill flow, schedule, and trigger files. ## Overview -The windmill-yaml-validator provides runtime validation for Windmill flow YAML files. It is currently used in the **Windmill VSCode extension** to show syntax errors and validation issues on `flow.yaml` files in real-time as developers edit them. +The windmill-yaml-validator provides runtime validation for Windmill YAML files. It is used by editor integrations to show validation errors while editing: + +- `flow.yaml` / `flow.yml` +- `*.schedule.yaml` / `*.schedule.yml` +- `*.{http|websocket|kafka|nats|postgres|mqtt|sqs|gcp}_trigger.yaml` (or `.yml`) ## Features -- **Schema-based validation**: Validates against the official OpenFlow JSON schema +- **Unified validation API**: One validator class for flow/schedule/trigger files +- **Schema-based validation**: Uses OpenFlow and backend OpenAPI-derived schemas - **Detailed error reporting**: Returns comprehensive error information with specific paths to invalid fields ## Installation @@ -22,29 +27,70 @@ npm install windmill-yaml-validator ### Basic Validation ```typescript -import { FlowValidator } from "windmill-yaml-validator"; +import { WindmillYamlValidator } from "windmill-yaml-validator"; -const validator = new FlowValidator(); +const validator = new WindmillYamlValidator(); -const yamlContent = ` +const flowYaml = ` summary: Test Flow value: modules: [] `; -const result = validator.validateFlow(yamlContent); +const flowResult = validator.validate(flowYaml, { type: "flow" }); -if (result.errors.length === 0) { - console.log("Flow is valid!"); -} else { - console.log("Validation errors:", result.errors); +const scheduleYaml = ` +schedule: "0 0 12 * * *" +timezone: "UTC" +enabled: true +script_path: "f/jobs/daily_sync" +is_flow: false +`; + +const scheduleResult = validator.validate(scheduleYaml, { type: "schedule" }); + +const triggerYaml = ` +script_path: "f/triggers/http_handler" +is_flow: false +route_path: "api/webhook" +request_type: "sync" +authentication_method: "none" +http_method: "post" +is_static_website: false +workspaced_route: false +wrap_body: false +raw_string: false +`; + +const triggerResult = validator.validate(triggerYaml, { + type: "trigger", + triggerKind: "http", +}); + +console.log(flowResult.errors, scheduleResult.errors, triggerResult.errors); +``` + +### Target Inference by Filename + +```typescript +import { + WindmillYamlValidator, + getValidationTargetFromFilename, +} from "windmill-yaml-validator"; + +const validator = new WindmillYamlValidator(); +const target = getValidationTargetFromFilename( + "f/webhooks/order_created.http_trigger.yaml" +); + +if (target) { + const result = validator.validate(fileContents, target); + console.log(result.errors); } ``` ### Error Handling -The validator returns detailed error information for invalid flows: - ```typescript const invalidYaml = ` summary: 123 # Should be a string @@ -56,7 +102,7 @@ value: language: invalid_language # Invalid enum value `; -const result = validator.validateFlow(invalidYaml); +const result = validator.validate(invalidYaml, { type: "flow" }); result.errors.forEach((error) => { console.log(`Error at ${error.instancePath}: ${error.message}`); @@ -68,27 +114,31 @@ result.errors.forEach((error) => { ## API -### `FlowValidator` +### `WindmillYamlValidator` -Main validator class that validates Windmill flow YAML files. +Main validator class for Windmill YAML validation. #### Constructor ```typescript -new FlowValidator(); +new WindmillYamlValidator(); ``` -Creates a new validator instance. The constructor initializes the AJV validator with the OpenFlow schema. +Initializes AJV validators for flow, schedule, and trigger schemas. #### Methods -##### `validateFlow(doc: string)` +##### `validate(doc: string, target: ValidationTarget)` -Validates a flow document against the OpenFlow schema. +Validates a YAML document against the selected target schema. **Parameters:** -- `doc` (string): The YAML flow document as a string +- `doc` (string): YAML document string +- `target` (`ValidationTarget`): + - `{ type: "flow" }` + - `{ type: "schedule" }` + - `{ type: "trigger", triggerKind: "http" | "websocket" | "kafka" | "nats" | "postgres" | "mqtt" | "sqs" | "gcp" | "email" }` **Returns:** @@ -103,6 +153,10 @@ Validates a flow document against the OpenFlow schema. - Error if `doc` is not a string +### `getValidationTargetFromFilename(path: string)` + +Infers validation target from file naming conventions. Returns `null` for unsupported files. + ## Development ### Building @@ -113,7 +167,10 @@ npm run build The build process: -1. Runs `gen_openflow_schema.sh` to generate the OpenFlow JSON schema from `openflow.openapi.yaml` +1. Runs `gen_openflow_schema.sh` to generate: + - `src/gen/openflow.json` + - `src/gen/schedule.json` + - `src/gen/triggers/*.json` 2. Removes discriminator mappings (not supported by AJV) 3. Compiles TypeScript to JavaScript @@ -129,6 +186,44 @@ Run tests in watch mode: npm test:watch ``` +### Testing locally with the CLI + +The Windmill CLI (`cli/`) is Deno-based and imports this package via `npm:windmill-yaml-validator@1.1.0`. Since Deno's `npm:` specifier always resolves from the npm registry, local testing requires a compatibility script that makes the TypeScript sources directly importable by Deno. + +The `deno-compat.sh` script handles two Deno requirements: +- Adding `.ts` extensions to relative imports +- Adding `with { type: "json" }` assertions to JSON imports + +**Steps:** + +1. Apply Deno compatibility: + +```bash +./deno-compat.sh +``` + +2. Add the following entries to `cli/deno.json` imports: + +```json +"npm:windmill-yaml-validator@1.1.0": "../windmill-yaml-validator/src/index.ts", +"ajv": "npm:ajv@^8.17.1", +"@stoplight/yaml": "npm:@stoplight/yaml@^4.3.0" +``` + +3. Run the CLI directly with Deno: + +```bash +cd ../cli +deno run -A src/main.ts lint +``` + +4. When done, restore everything: + +```bash +./deno-compat.sh -r # restore original imports +# Remove the 3 import map lines from cli/deno.json +``` + ### Schema Generation The validator uses a JSON schema generated from the OpenAPI specification: @@ -139,10 +234,10 @@ The validator uses a JSON schema generated from the OpenAPI specification: This script: -- Bundles `openflow.openapi.yaml` into a single JSON schema +- Converts `openflow.openapi.yaml` and `backend/windmill-api/openapi.yaml` into JSON - Removes discriminator mappings for AJV compatibility - Removes the `ToolValue` discriminator entirely (see below) -- Outputs to `src/gen/openflow.json` +- Generates standalone schedule/trigger schemas for CLI file shape #### Why Remove Discriminators? @@ -157,3 +252,7 @@ The OpenFlow schema uses OpenAPI discriminators for efficient type resolution in - Is slightly slower but still performant for our use case - Provides the same validation correctness - Works correctly with complex schema compositions like `allOf` + +## Breaking Change + +`FlowValidator` and `validateFlow()` were replaced by `WindmillYamlValidator` and `validate(doc, target)`. diff --git a/windmill-yaml-validator/deno-compat.sh b/windmill-yaml-validator/deno-compat.sh new file mode 100755 index 0000000000..154fe72d0b --- /dev/null +++ b/windmill-yaml-validator/deno-compat.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Makes windmill-yaml-validator source files Deno-compatible by: +# 1. Adding .ts extensions to relative imports +# 2. Adding `with { type: "json" }` to JSON imports +# Use -r to restore (undo changes). + +set -e +script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RESTORE_MODE=false +while [[ $# -gt 0 ]]; do + case $1 in + -r) + RESTORE_MODE=true + shift + ;; + *) + echo "Unknown option: $1" + echo "Usage: $0 [-r]" + echo " -r Restore original imports" + exit 1 + ;; + esac +done + +if [[ "$OSTYPE" == "darwin"* ]]; then + SED=gsed + if ! command -v gsed &> /dev/null; then + echo "Error: gsed not found. Run: brew install gnu-sed" + exit 1 + fi +else + SED=sed +fi + +if [[ "$RESTORE_MODE" == true ]]; then + echo "Restoring original imports..." + find "$script_dirpath"/src -name "*.ts" -type f ! -path '*__tests__*' | while read -r file; do + # Remove .ts from relative imports: from "./foo.ts" -> from "./foo" + $SED -E -i 's|(from "\.\.?/[^"]*)\.ts(")|\1\2|g' "$file" + # Remove ` with { type: "json" }` from JSON imports + $SED -E -i 's/ with \{ type: "json" \}//' "$file" + done + echo "✓ Restored original imports" +else + echo "Making sources Deno-compatible..." + find "$script_dirpath"/src -name "*.ts" -type f ! -path '*__tests__*' | while read -r file; do + # Add .ts to relative imports that don't already end in .ts or .json + $SED -E -i '/\.json"/! { /\.ts"/! s|(from "(\.\.?/[^"]*[^/]))"(;?)$|\1.ts"\3|; }' "$file" + # Add `with { type: "json" }` to .json imports that don't already have it + $SED -E -i '/with \{ type: "json" \}/! s/(from "[^"]*\.json")(;?)$/\1 with { type: "json" }\2/' "$file" + done + echo "✓ Sources are now Deno-compatible" +fi diff --git a/windmill-yaml-validator/gen_openflow_schema.sh b/windmill-yaml-validator/gen_openflow_schema.sh index f2c9ecf2a2..eb387b8d95 100755 --- a/windmill-yaml-validator/gen_openflow_schema.sh +++ b/windmill-yaml-validator/gen_openflow_schema.sh @@ -1,10 +1,34 @@ #!/usr/bin/env bash +set -euo pipefail + script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" output_dirpath="${script_dirpath}/src/gen" +tmp_dirpath="$(mktemp -d)" + +cleanup() { + rm -rf "${tmp_dirpath}" +} +trap cleanup EXIT mkdir -p "${output_dirpath}" -npx @redocly/openapi-cli@latest bundle "${script_dirpath}/../openflow.openapi.yaml" --ext json > "${output_dirpath}/openflow.json" +mkdir -p "${output_dirpath}/triggers" + +node -e " +const fs = require('fs'); +const yaml = require('js-yaml'); + +const openflowPath = '${script_dirpath}/../openflow.openapi.yaml'; +const backendPath = '${script_dirpath}/../backend/windmill-api/openapi.yaml'; +const openflowOutputPath = '${output_dirpath}/openflow.json'; +const backendOutputPath = '${tmp_dirpath}/backend-openapi.json'; + +const openflowData = yaml.load(fs.readFileSync(openflowPath, 'utf8')); +const backendData = yaml.load(fs.readFileSync(backendPath, 'utf8')); + +fs.writeFileSync(openflowOutputPath, JSON.stringify(openflowData, null, 2) + '\\n'); +fs.writeFileSync(backendOutputPath, JSON.stringify(backendData, null, 2) + '\\n'); +" # Remove discriminator mapping from openflow.json as it's not supported by ajv node -e " @@ -34,4 +58,39 @@ try { } catch (e) { console.error('Error removing discriminator mappings:', e); } -" \ No newline at end of file +" + +node "${script_dirpath}/scripts/generate-resource-schemas.js" \ + "${tmp_dirpath}/backend-openapi.json" \ + "${output_dirpath}/openflow.json" \ + "${output_dirpath}" + +# AJV does not handle OpenAPI 3.0 `nullable: true` combined with `enum` — null must +# be explicitly listed in the enum for validation to accept null values. +# We post-process all generated JSON schemas to add null to such enums. +node -e " +const fs = require('fs'); +const path = require('path'); + +function addNullToNullableEnums(obj) { + if (!obj || typeof obj !== 'object') return; + if (Array.isArray(obj)) { obj.forEach(addNullToNullableEnums); return; } + if (obj.nullable === true && Array.isArray(obj.enum) && !obj.enum.includes(null)) { + obj.enum.push(null); + } + for (const v of Object.values(obj)) addNullToNullableEnums(v); +} + +const files = [ + '${output_dirpath}/openflow.json', + '${output_dirpath}/schedule.json', + ...fs.readdirSync('${output_dirpath}/triggers').map(f => path.join('${output_dirpath}/triggers', f)) +].filter(f => f.endsWith('.json')); + +for (const file of files) { + const schema = JSON.parse(fs.readFileSync(file, 'utf8')); + addNullToNullableEnums(schema); + fs.writeFileSync(file, JSON.stringify(schema, null, 2) + '\n'); +} +console.log('Added null to nullable enums in generated schemas'); +" diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index f211310a12..e63399e721 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.0.4", + "version": "1.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.0.4", + "version": "1.1.1", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", @@ -16,6 +16,7 @@ "@types/jest": "^29.5.0", "@types/node": "^24.1.0", "jest": "^29.5.0", + "js-yaml": "^3.14.1", "ts-jest": "^29.1.0", "typescript": "^5.0.0" } diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index 2f16ef4a27..a43291c048 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,11 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.0.4", - "description": "YAML validator for Windmill", + "version": "1.1.1", + "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", "scripts": { "build": "./gen_openflow_schema.sh && tsc", + "pretest": "./gen_openflow_schema.sh", "test": "jest", "test:watch": "jest --watch", "prepublishOnly": "npm run build" @@ -18,6 +19,7 @@ "devDependencies": { "@types/jest": "^29.5.0", "@types/node": "^24.1.0", + "js-yaml": "^3.14.1", "jest": "^29.5.0", "ts-jest": "^29.1.0", "typescript": "^5.0.0" @@ -29,4 +31,4 @@ "@stoplight/yaml": "^4.3.0", "ajv": "^8.17.1" } -} +} \ No newline at end of file diff --git a/windmill-yaml-validator/scripts/generate-resource-schemas.js b/windmill-yaml-validator/scripts/generate-resource-schemas.js new file mode 100644 index 0000000000..5d25b8d005 --- /dev/null +++ b/windmill-yaml-validator/scripts/generate-resource-schemas.js @@ -0,0 +1,234 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const path = require("path"); + +const CLI_EXCLUDED_FIELDS = new Set([ + "workspace_id", + "path", + "name", + "versions", + "id", + "created_at", + "updated_at", + "created_by", + "updated_by", + "edited_at", + "edited_by", + "archived", + "has_draft", + "error", + "last_server_ping", + "server_id", + "extra_perms", + "email", + "mode", +]); + +const TARGET_SCHEMAS = { + schedule: "Schedule", + triggers: { + http: "HttpTrigger", + websocket: "WebsocketTrigger", + kafka: "KafkaTrigger", + nats: "NatsTrigger", + postgres: "PostgresTrigger", + mqtt: "MqttTrigger", + sqs: "SqsTrigger", + gcp: "GcpTrigger", + email: "EmailTrigger", + }, +}; + +function deepClone(value) { + return JSON.parse(JSON.stringify(value)); +} + +function loadJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function mergeSchemas(target, source) { + if (!source || typeof source !== "object") { + return; + } + + if (source.type && !target.type) { + target.type = source.type; + } + + if (source.properties && typeof source.properties === "object") { + target.properties = target.properties || {}; + Object.assign(target.properties, deepClone(source.properties)); + } + + if (Array.isArray(source.required)) { + target.required = target.required || []; + target.required.push(...source.required); + } +} + +function getRefName(refPath) { + const marker = "#/components/schemas/"; + const markerIndex = refPath.indexOf(marker); + if (markerIndex === -1) { + return null; + } + return refPath.slice(markerIndex + marker.length); +} + +function resolveRef(refPath, backendSchemas, openflowSchemas) { + const refName = getRefName(refPath); + if (!refName) { + return null; + } + + if (refPath.startsWith("#/components/schemas/")) { + if (backendSchemas[refName]) { + return deepClone(backendSchemas[refName]); + } + if (openflowSchemas[refName]) { + return deepClone(openflowSchemas[refName]); + } + return null; + } + + if (refPath.includes("openflow.openapi.yaml#/components/schemas/")) { + if (openflowSchemas[refName]) { + return deepClone(openflowSchemas[refName]); + } + return null; + } + + return null; +} + +function extractCliSchema(schema, allSchemas, openflowSchemas) { + if (!schema || typeof schema !== "object") { + return {}; + } + + const result = { type: "object", properties: {}, required: [] }; + + if (Array.isArray(schema.allOf)) { + for (const item of schema.allOf) { + if (!item || typeof item !== "object") { + continue; + } + if (item.$ref) { + const refSchema = resolveRef(item.$ref, allSchemas, openflowSchemas); + const transformed = extractCliSchema(refSchema, allSchemas, openflowSchemas); + mergeSchemas(result, transformed); + } else { + const transformed = extractCliSchema(item, allSchemas, openflowSchemas); + mergeSchemas(result, transformed); + } + } + } + + if (schema.properties && typeof schema.properties === "object") { + for (const [key, value] of Object.entries(schema.properties)) { + if (CLI_EXCLUDED_FIELDS.has(key)) { + continue; + } + result.properties[key] = deepClone(value); + } + } + + if (Array.isArray(schema.required)) { + for (const field of schema.required) { + if (!CLI_EXCLUDED_FIELDS.has(field)) { + result.required.push(field); + } + } + } + + const dedupRequired = Array.from(new Set(result.required)); + result.required = dedupRequired.filter((key) => key in result.properties); + + return result; +} + +function resolveSchemaRefs(value, backendSchemas, openflowSchemas, stack = new Set()) { + if (Array.isArray(value)) { + return value.map((item) => resolveSchemaRefs(item, backendSchemas, openflowSchemas, stack)); + } + + if (!value || typeof value !== "object") { + return value; + } + + if (typeof value.$ref === "string") { + const refName = getRefName(value.$ref); + if (!refName) { + return value; + } + + if (stack.has(refName)) { + return {}; + } + + const resolved = resolveRef(value.$ref, backendSchemas, openflowSchemas); + if (!resolved) { + return value; + } + + const merged = { ...resolved, ...value }; + delete merged.$ref; + const nextStack = new Set(stack); + nextStack.add(refName); + return resolveSchemaRefs(merged, backendSchemas, openflowSchemas, nextStack); + } + + const result = {}; + for (const [key, nested] of Object.entries(value)) { + result[key] = resolveSchemaRefs(nested, backendSchemas, openflowSchemas, stack); + } + return result; +} + +function writeJson(filePath, value) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(value, null, 2) + "\n", "utf8"); +} + +function generateSchemas(backendPath, openflowPath, outputDir) { + const backendOpenapi = loadJson(backendPath); + const openflowOpenapi = loadJson(openflowPath); + + const backendSchemas = backendOpenapi.components?.schemas || {}; + const openflowSchemas = openflowOpenapi.components?.schemas || {}; + + const scheduleSchema = extractCliSchema( + backendSchemas[TARGET_SCHEMAS.schedule], + backendSchemas, + openflowSchemas + ); + const resolvedScheduleSchema = resolveSchemaRefs(scheduleSchema, backendSchemas, openflowSchemas); + writeJson(path.join(outputDir, "schedule.json"), resolvedScheduleSchema); + + for (const [triggerKind, schemaName] of Object.entries(TARGET_SCHEMAS.triggers)) { + const triggerSchema = extractCliSchema( + backendSchemas[schemaName], + backendSchemas, + openflowSchemas + ); + const resolvedTriggerSchema = resolveSchemaRefs(triggerSchema, backendSchemas, openflowSchemas); + writeJson(path.join(outputDir, "triggers", `${triggerKind}.json`), resolvedTriggerSchema); + } +} + +function main() { + const [backendPath, openflowPath, outputDir] = process.argv.slice(2); + if (!backendPath || !openflowPath || !outputDir) { + console.error( + "Usage: node scripts/generate-resource-schemas.js " + ); + process.exit(1); + } + + generateSchemas(backendPath, openflowPath, outputDir); + console.log("Generated schedule and trigger schemas"); +} + +main(); diff --git a/windmill-yaml-validator/src/index.ts b/windmill-yaml-validator/src/index.ts index cf880449e3..fed12a1ca8 100644 --- a/windmill-yaml-validator/src/index.ts +++ b/windmill-yaml-validator/src/index.ts @@ -1 +1 @@ -export * from "./validation"; \ No newline at end of file +export * from "./validation/index"; \ No newline at end of file diff --git a/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts b/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts index 0efee8d189..392567f795 100644 --- a/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts +++ b/windmill-yaml-validator/src/validation/__tests__/flow-validator.test.ts @@ -1,9 +1,9 @@ -import { FlowValidator } from "../flow-validator"; +import { WindmillYamlValidator } from "../yaml-validator"; import * as fs from "fs"; import * as path from "path"; -describe("FlowValidator", () => { - let validator: FlowValidator; +describe("WindmillYamlValidator", () => { + let validator: WindmillYamlValidator; const samplesDir = path.join(__dirname, "test-samples"); const readSample = (filename: string): string => { @@ -11,31 +11,31 @@ describe("FlowValidator", () => { }; beforeEach(() => { - validator = new FlowValidator(); + validator = new WindmillYamlValidator(); }); describe("constructor", () => { it("should create a validator instance", () => { - expect(validator).toBeInstanceOf(FlowValidator); + expect(validator).toBeInstanceOf(WindmillYamlValidator); }); it("should initialize without throwing", () => { - expect(() => new FlowValidator()).not.toThrow(); + expect(() => new WindmillYamlValidator()).not.toThrow(); }); }); - describe("validateFlow", () => { + describe("validate (flow target)", () => { it("should throw error for non-string input", () => { - expect(() => validator.validateFlow(null as any)).toThrow( + expect(() => validator.validate(null as any, { type: "flow" })).toThrow( "Document must be a string" ); - expect(() => validator.validateFlow(123 as any)).toThrow( + expect(() => validator.validate(123 as any, { type: "flow" })).toThrow( "Document must be a string" ); - expect(() => validator.validateFlow({} as any)).toThrow( + expect(() => validator.validate({} as any, { type: "flow" })).toThrow( "Document must be a string" ); - expect(() => validator.validateFlow([] as any)).toThrow( + expect(() => validator.validate([] as any, { type: "flow" })).toThrow( "Document must be a string" ); }); @@ -44,7 +44,7 @@ describe("FlowValidator", () => { it("should validate a valid minimal flow from sample file", () => { const validFlow = readSample("valid-minimal.yaml"); - const result = validator.validateFlow(validFlow); + const result = validator.validate(validFlow, { type: "flow" }); expect(result.errors).toHaveLength(0); expect(result.parsed).toBeDefined(); @@ -59,7 +59,7 @@ describe("FlowValidator", () => { it("should validate a script flow from sample file", () => { const validFlow = readSample("valid-script-flow.yaml"); - const result = validator.validateFlow(validFlow); + const result = validator.validate(validFlow, { type: "flow" }); expect(result.errors).toHaveLength(0); expect(result.parsed.data).toMatchObject({ @@ -94,7 +94,7 @@ describe("FlowValidator", () => { it("should return errors for missing summary from sample file", () => { const invalidFlow = readSample("invalid-missing-summary.yaml"); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); expect( @@ -110,7 +110,7 @@ describe("FlowValidator", () => { it("should return errors for invalid types from sample file", () => { const invalidFlow = readSample("invalid-wrong-types.yaml"); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); expect( @@ -124,7 +124,7 @@ describe("FlowValidator", () => { it("should return errors for invalid language from sample file", () => { const invalidFlow = readSample("invalid-language.yaml"); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); expect( @@ -139,7 +139,7 @@ describe("FlowValidator", () => { it("should handle empty file from sample", () => { const emptyFlow = readSample("empty.yaml"); - const result = validator.validateFlow(emptyFlow); + const result = validator.validate(emptyFlow, { type: "flow" }); expect(result.parsed).toBeDefined(); expect(result.errors.length).toBeGreaterThan(0); @@ -148,7 +148,7 @@ describe("FlowValidator", () => { it("should handle complex invalid flow with comprehensive error detection", () => { const complexInvalidFlow = readSample("invalid-complex-flow.yaml"); - const result = validator.validateFlow(complexInvalidFlow); + const result = validator.validate(complexInvalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(20); // Should have many errors @@ -202,7 +202,7 @@ describe("FlowValidator", () => { it("should handle deeply nested invalid structures with detailed error reporting", () => { const nestedInvalidFlow = readSample("invalid-nested-structures.yaml"); - const result = validator.validateFlow(nestedInvalidFlow); + const result = validator.validate(nestedInvalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(10); // Should have many nested errors @@ -244,7 +244,7 @@ describe("FlowValidator", () => { it("should provide specific error locations for complex validation failures", () => { const complexInvalidFlow = readSample("invalid-complex-flow.yaml"); - const result = validator.validateFlow(complexInvalidFlow); + const result = validator.validate(complexInvalidFlow, { type: "flow" }); // Verify that errors have meaningful instance paths const errorsWithPaths = result.errors.filter( @@ -268,7 +268,7 @@ describe("FlowValidator", () => { it("should handle all major flow control structures with validation errors", () => { const complexInvalidFlow = readSample("invalid-complex-flow.yaml"); - const result = validator.validateFlow(complexInvalidFlow); + const result = validator.validate(complexInvalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(20); @@ -317,7 +317,7 @@ describe("FlowValidator", () => { it("should validate a basic AI agent flow with FlowModule tools", () => { const validFlow = readSample("valid-aiagent-basic.yaml"); - const result = validator.validateFlow(validFlow); + const result = validator.validate(validFlow, { type: "flow" }); expect(result.errors).toHaveLength(0); }); @@ -325,7 +325,7 @@ describe("FlowValidator", () => { it("should validate an AI agent flow with MCP tools", () => { const validFlow = readSample("valid-aiagent-mcp.yaml"); - const result = validator.validateFlow(validFlow); + const result = validator.validate(validFlow, { type: "flow" }); expect(result.errors).toHaveLength(0); }); @@ -333,7 +333,7 @@ describe("FlowValidator", () => { it("should validate an AI agent flow with mixed FlowModule and MCP tools", () => { const validFlow = readSample("valid-aiagent-mixed.yaml"); - const result = validator.validateFlow(validFlow); + const result = validator.validate(validFlow, { type: "flow" }); expect(result.errors).toHaveLength(0); }); @@ -341,7 +341,7 @@ describe("FlowValidator", () => { it("should validate an AI agent flow with parallel execution enabled", () => { const validFlow = readSample("valid-aiagent-parallel.yaml"); - const result = validator.validateFlow(validFlow); + const result = validator.validate(validFlow, { type: "flow" }); expect(result.errors).toHaveLength(0); // Verify tools array has expected structure @@ -354,7 +354,7 @@ describe("FlowValidator", () => { it("should return errors for AI agent missing required tools field", () => { const invalidFlow = readSample("invalid-aiagent-missing-tools.yaml"); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); expect( @@ -369,7 +369,7 @@ describe("FlowValidator", () => { it("should return errors for AI agent missing type field", () => { const invalidFlow = readSample("invalid-aiagent-missing-type.yaml"); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); // Should fail discriminator validation since type is missing @@ -387,7 +387,7 @@ describe("FlowValidator", () => { "invalid-aiagent-invalid-tool-type.yaml" ); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); // Should fail discriminator validation for invalid tool_type @@ -404,7 +404,7 @@ describe("FlowValidator", () => { "invalid-aiagent-mcp-missing-resource.yaml" ); - const result = validator.validateFlow(invalidFlow); + const result = validator.validate(invalidFlow, { type: "flow" }); expect(result.errors.length).toBeGreaterThan(0); expect( diff --git a/windmill-yaml-validator/src/validation/__tests__/resource-validator.test.ts b/windmill-yaml-validator/src/validation/__tests__/resource-validator.test.ts new file mode 100644 index 0000000000..c7c7cfaf39 --- /dev/null +++ b/windmill-yaml-validator/src/validation/__tests__/resource-validator.test.ts @@ -0,0 +1,643 @@ +import { + getValidationTargetFromFilename, + TriggerKind, + WindmillYamlValidator, +} from "../yaml-validator"; + +describe("WindmillYamlValidator resource validation", () => { + let validator: WindmillYamlValidator; + + beforeEach(() => { + validator = new WindmillYamlValidator(); + }); + + // ── Schedules ────────────────────────────────────────────────────────── + + describe("schedules", () => { + it("validates a minimal schedule", () => { + const result = validator.validate( + JSON.stringify({ + schedule: "0 0 12 * * *", + timezone: "UTC", + enabled: true, + script_path: "f/jobs/daily_sync", + is_flow: false, + }), + { type: "schedule" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("validates a fully-configured schedule with handlers and retry", () => { + const result = validator.validate( + JSON.stringify({ + schedule: "0 */5 * * * *", + timezone: "Europe/Paris", + enabled: true, + script_path: "f/jobs/full_sync", + is_flow: true, + args: { batch_size: 100, dry_run: false }, + on_failure: "f/handlers/on_fail", + on_failure_times: 3, + on_failure_exact: true, + on_failure_extra_args: { notify: true }, + on_recovery: "f/handlers/on_recover", + on_recovery_times: 2, + on_recovery_extra_args: { channel: "#ops" }, + on_success: "f/handlers/on_success", + on_success_extra_args: { log: true }, + ws_error_handler_muted: true, + retry: { + constant: { attempts: 3, seconds: 10 }, + exponential: { + attempts: 5, + multiplier: 2, + seconds: 1, + random_factor: 50, + }, + retry_if: { expr: "error.message.includes('timeout')" }, + }, + summary: "Full sync every 5 minutes", + description: "Runs the full synchronization pipeline", + no_flow_overlap: true, + tag: "heavy", + paused_until: "2026-03-01T00:00:00Z", + cron_version: "v2", + dynamic_skip: "f/helpers/should_skip", + }), + { type: "schedule" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("accepts null for all nullable optional fields", () => { + const result = validator.validate( + JSON.stringify({ + schedule: "0 0 12 * * *", + timezone: "UTC", + enabled: true, + script_path: "f/jobs/daily_sync", + is_flow: false, + args: null, + on_failure: null, + tag: null, + retry: null, + paused_until: null, + summary: null, + description: null, + cron_version: null, + dynamic_skip: null, + }), + { type: "schedule" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("rejects a schedule missing required fields", () => { + const result = validator.validate( + JSON.stringify({ timezone: "UTC" }), + { type: "schedule" } + ); + expect(result.errors.length).toBeGreaterThan(0); + }); + + it("rejects wrong types in schedule fields", () => { + const result = validator.validate( + JSON.stringify({ + schedule: 12345, + timezone: "UTC", + enabled: "yes", + script_path: "f/jobs/daily_sync", + is_flow: "true", + }), + { type: "schedule" } + ); + // Should catch schedule (not string), enabled (not boolean), is_flow (not boolean) + expect(result.errors.length).toBeGreaterThanOrEqual(3); + }); + + it("rejects invalid retry constraints", () => { + const result = validator.validate( + JSON.stringify({ + schedule: "0 0 12 * * *", + timezone: "UTC", + enabled: true, + script_path: "f/jobs/daily_sync", + is_flow: false, + retry: { + exponential: { attempts: 3, seconds: 0, random_factor: 150 }, + retry_if: {}, + }, + }), + { type: "schedule" } + ); + // seconds < 1, random_factor > 100, retry_if missing expr + expect(result.errors.length).toBeGreaterThanOrEqual(3); + }); + }); + + // ── Triggers — valid minimal + valid with missing required ───────────── + + describe("trigger schemas", () => { + const validTriggers: Record> = { + http: { + script_path: "f/triggers/http_handler", + is_flow: false, + route_path: "api/webhook", + request_type: "sync", + authentication_method: "none", + http_method: "post", + is_static_website: false, + workspaced_route: false, + wrap_body: false, + raw_string: false, + }, + websocket: { + script_path: "f/triggers/ws_handler", + is_flow: false, + url: "wss://example.com/socket", + filters: [], + can_return_message: false, + can_return_error_result: true, + }, + kafka: { + script_path: "f/triggers/kafka_handler", + is_flow: false, + kafka_resource_path: "f/resources/kafka", + group_id: "group-a", + topics: ["topic-a"], + filters: [], + }, + nats: { + script_path: "f/triggers/nats_handler", + is_flow: false, + nats_resource_path: "f/resources/nats", + use_jetstream: false, + subjects: ["events.>"], + }, + postgres: { + script_path: "f/triggers/postgres_handler", + is_flow: false, + postgres_resource_path: "f/resources/postgres", + publication_name: "pub_main", + replication_slot_name: "slot_main", + }, + mqtt: { + script_path: "f/triggers/mqtt_handler", + is_flow: false, + mqtt_resource_path: "f/resources/mqtt", + subscribe_topics: [], + }, + sqs: { + script_path: "f/triggers/sqs_handler", + is_flow: false, + queue_url: "https://sqs.us-east-1.amazonaws.com/12345/my-queue", + aws_resource_path: "f/resources/aws", + aws_auth_resource_type: "credentials", + }, + gcp: { + script_path: "f/triggers/gcp_handler", + is_flow: false, + gcp_resource_path: "f/resources/gcp", + topic_id: "topic-a", + subscription_id: "sub-a", + delivery_type: "pull", + subscription_mode: "existing", + }, + email: { + script_path: "f/triggers/email_handler", + is_flow: false, + local_part: "inbox", + }, + }; + + const missingRequiredField: Record = { + http: "request_type", + websocket: "url", + kafka: "topics", + nats: "subjects", + postgres: "publication_name", + mqtt: "mqtt_resource_path", + sqs: "queue_url", + gcp: "topic_id", + email: "local_part", + }; + + for (const [kind, validDocument] of Object.entries(validTriggers) as [ + TriggerKind, + Record, + ][]) { + it(`validates a valid ${kind} trigger file`, () => { + const result = validator.validate(JSON.stringify(validDocument), { + type: "trigger", + triggerKind: kind, + }); + + expect(result.errors).toHaveLength(0); + }); + + it(`returns required-field errors for invalid ${kind} trigger file`, () => { + const missingField = missingRequiredField[kind]; + const invalidDocument = { ...validDocument }; + delete invalidDocument[missingField]; + + const result = validator.validate(JSON.stringify(invalidDocument), { + type: "trigger", + triggerKind: kind, + }); + + expect(result.errors.length).toBeGreaterThan(0); + expect( + result.errors.some( + (error) => + error.keyword === "required" && + (error.params as { missingProperty?: string })?.missingProperty === + missingField + ) + ).toBe(true); + }); + } + }); + + it("throws for unsupported trigger kinds", () => { + expect(() => + validator.validate("{}", { + type: "trigger", + triggerKind: "foobar" as any, + }) + ).toThrow("Unsupported trigger kind: foobar"); + }); + + // ── Triggers — realistic full documents with all optional fields ─────── + + describe("fully-configured triggers", () => { + it("validates http trigger with static assets, auth, error handling and retry", () => { + const result = validator.validate( + JSON.stringify({ + script_path: "f/triggers/http_handler", + is_flow: false, + route_path: "api/webhook", + request_type: "sync_sse", + authentication_method: "api_key", + http_method: "put", + is_static_website: true, + workspaced_route: true, + wrap_body: true, + raw_string: false, + summary: "Incoming webhook", + description: "Handles external webhook deliveries", + authentication_resource_path: "f/resources/api_key_config", + static_asset_config: { s3: "my-bucket/assets", filename: "index.html" }, + error_handler_path: "f/handlers/on_error", + error_handler_args: { notify: true }, + retry: { + constant: { attempts: 2, seconds: 5 }, + retry_if: { expr: "error.status === 429" }, + }, + }), + { type: "trigger", triggerKind: "http" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("validates websocket trigger with initial messages and filters", () => { + const result = validator.validate( + JSON.stringify({ + script_path: "f/triggers/ws_handler", + is_flow: true, + url: "wss://stream.example.com/v1", + filters: [{ key: "event", value: "trade" }], + can_return_message: true, + can_return_error_result: false, + initial_messages: [ + { raw_message: '{"action":"subscribe","channel":"trades"}' }, + { + runnable_result: { + path: "f/helpers/ws_auth", + args: { token: "abc" }, + is_flow: false, + }, + }, + ], + url_runnable_args: { env: "production" }, + error_handler_path: "f/handlers/on_error", + retry: { exponential: { attempts: 5, multiplier: 2, seconds: 1, random_factor: 25 } }, + }), + { type: "trigger", triggerKind: "websocket" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("validates mqtt trigger with v5 config and subscribe topics", () => { + const result = validator.validate( + JSON.stringify({ + script_path: "f/triggers/mqtt_handler", + is_flow: false, + mqtt_resource_path: "f/resources/mqtt", + subscribe_topics: [ + { topic: "sensor/+/data", qos: "qos1" }, + { topic: "alerts/#", qos: "qos2" }, + ], + client_version: "v5", + client_id: "windmill-consumer-1", + v5_config: { + clean_start: true, + topic_alias_maximum: 10, + session_expiry_interval: 300, + }, + error_handler_path: "f/handlers/on_error", + retry: { constant: { attempts: 3, seconds: 10 } }, + }), + { type: "trigger", triggerKind: "mqtt" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("validates nats trigger with jetstream config", () => { + const result = validator.validate( + JSON.stringify({ + script_path: "f/triggers/nats_handler", + is_flow: true, + nats_resource_path: "f/resources/nats", + use_jetstream: true, + subjects: ["orders.>"], + stream_name: "ORDERS", + consumer_name: "windmill-consumer", + error_handler_path: "f/handlers/on_error", + error_handler_args: {}, + retry: { constant: { attempts: 5, seconds: 30 } }, + }), + { type: "trigger", triggerKind: "nats" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("validates sqs trigger with message attributes and oidc auth", () => { + const result = validator.validate( + JSON.stringify({ + script_path: "f/triggers/sqs_handler", + is_flow: false, + queue_url: "https://sqs.us-east-1.amazonaws.com/12345/my-queue", + aws_resource_path: "f/resources/aws", + aws_auth_resource_type: "oidc", + message_attributes: ["traceId", "source"], + error_handler_path: "f/handlers/on_error", + }), + { type: "trigger", triggerKind: "sqs" } + ); + expect(result.errors).toHaveLength(0); + }); + + it("validates gcp trigger with push delivery config", () => { + const result = validator.validate( + JSON.stringify({ + script_path: "f/triggers/gcp_handler", + is_flow: false, + gcp_resource_path: "f/resources/gcp", + topic_id: "topic-a", + subscription_id: "sub-push", + delivery_type: "push", + subscription_mode: "create_update", + delivery_config: { + authenticate: true, + base_endpoint: "https://app.example.com/webhook", + audience: "https://app.example.com", + }, + error_handler_path: "f/handlers/on_error", + retry: { retry_if: { expr: "error.message.includes('quota')" } }, + }), + { type: "trigger", triggerKind: "gcp" } + ); + expect(result.errors).toHaveLength(0); + }); + }); + + // ── Realistic user mistakes ──────────────────────────────────────────── + + describe("realistic user mistakes", () => { + it("catches invalid enum values across trigger types", () => { + // User typos an http_method as uppercase + const httpResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/http_handler", + is_flow: false, + route_path: "api/webhook", + request_type: "sync", + authentication_method: "none", + http_method: "POST", + is_static_website: false, + workspaced_route: false, + wrap_body: false, + raw_string: false, + }), + { type: "trigger", triggerKind: "http" } + ); + expect(httpResult.errors.length).toBeGreaterThan(0); + + // User writes "iam_role" instead of "oidc" or "credentials" + const sqsResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/sqs_handler", + is_flow: false, + queue_url: "https://sqs.us-east-1.amazonaws.com/12345/q", + aws_resource_path: "f/resources/aws", + aws_auth_resource_type: "iam_role", + }), + { type: "trigger", triggerKind: "sqs" } + ); + expect(sqsResult.errors.length).toBeGreaterThan(0); + + // User writes "stream" instead of "push" or "pull" + const gcpResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/gcp_handler", + is_flow: false, + gcp_resource_path: "f/resources/gcp", + topic_id: "t", + subscription_id: "s", + delivery_type: "stream", + subscription_mode: "existing", + }), + { type: "trigger", triggerKind: "gcp" } + ); + expect(gcpResult.errors.length).toBeGreaterThan(0); + + // User writes "v4" instead of "v3" or "v5" + const mqttResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/mqtt_handler", + is_flow: false, + mqtt_resource_path: "f/resources/mqtt", + subscribe_topics: [{ topic: "t", qos: "qos1" }], + client_version: "v4", + }), + { type: "trigger", triggerKind: "mqtt" } + ); + expect(mqttResult.errors.length).toBeGreaterThan(0); + }); + + it("catches malformed nested structures", () => { + // MQTT topic entry missing qos + const mqttResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/mqtt_handler", + is_flow: false, + mqtt_resource_path: "f/resources/mqtt", + subscribe_topics: [{ topic: "test/topic" }], + }), + { type: "trigger", triggerKind: "mqtt" } + ); + expect(mqttResult.errors.length).toBeGreaterThan(0); + + // HTTP static_asset_config without required s3 field + const httpResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/http_handler", + is_flow: false, + route_path: "api/assets", + request_type: "sync", + authentication_method: "none", + http_method: "get", + is_static_website: true, + workspaced_route: false, + wrap_body: false, + raw_string: false, + static_asset_config: { filename: "index.html" }, + }), + { type: "trigger", triggerKind: "http" } + ); + expect(httpResult.errors.length).toBeGreaterThan(0); + + // GCP delivery_config without required authenticate / base_endpoint + const gcpResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/gcp_handler", + is_flow: false, + gcp_resource_path: "f/resources/gcp", + topic_id: "t", + subscription_id: "s", + delivery_type: "push", + subscription_mode: "create_update", + delivery_config: { audience: "test" }, + }), + { type: "trigger", triggerKind: "gcp" } + ); + expect(gcpResult.errors.length).toBeGreaterThan(0); + + // Websocket filter missing key + const wsResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/ws_handler", + is_flow: false, + url: "wss://example.com/socket", + filters: [{ value: "test" }], + can_return_message: false, + can_return_error_result: true, + }), + { type: "trigger", triggerKind: "websocket" } + ); + expect(wsResult.errors.length).toBeGreaterThan(0); + }); + + it("catches wrong types in YAML (string-for-boolean, object-for-array)", () => { + // is_flow: "false" — YAML without quotes would parse to boolean, + // but a quoted "false" parses as string + const natsResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/nats_handler", + is_flow: "false", + nats_resource_path: "f/resources/nats", + use_jetstream: "yes", + subjects: ["events.>"], + }), + { type: "trigger", triggerKind: "nats" } + ); + expect(natsResult.errors.length).toBeGreaterThanOrEqual(2); + + // topics as {} instead of [] + const kafkaResult = validator.validate( + JSON.stringify({ + script_path: "f/triggers/kafka_handler", + is_flow: false, + kafka_resource_path: "f/resources/kafka", + group_id: "g", + topics: {}, + filters: "not-an-array", + }), + { type: "trigger", triggerKind: "kafka" } + ); + expect(kafkaResult.errors.length).toBeGreaterThanOrEqual(2); + }); + }); +}); + +describe("getValidationTargetFromFilename", () => { + it("detects flow files", () => { + expect(getValidationTargetFromFilename("f/my.flow/flow.yaml")).toEqual({ + type: "flow", + }); + expect(getValidationTargetFromFilename("flow.yml")).toEqual({ + type: "flow", + }); + }); + + it("detects schedule files", () => { + expect( + getValidationTargetFromFilename("f/folder/daily.schedule.yaml") + ).toEqual({ + type: "schedule", + }); + }); + + it("detects all 9 trigger kinds", () => { + const kinds = [ + "http", + "websocket", + "kafka", + "nats", + "postgres", + "mqtt", + "sqs", + "gcp", + "email", + ] as const; + + for (const kind of kinds) { + expect( + getValidationTargetFromFilename( + `f/triggers/handler.${kind}_trigger.yaml` + ) + ).toEqual({ + type: "trigger", + triggerKind: kind, + }); + } + }); + + it("is case-insensitive for extensions", () => { + expect(getValidationTargetFromFilename("f/my.flow/flow.YAML")).toEqual({ + type: "flow", + }); + expect( + getValidationTargetFromFilename("f/folder/daily.schedule.YML") + ).toEqual({ + type: "schedule", + }); + }); + + it("handles dots in directory names", () => { + expect( + getValidationTargetFromFilename("f/my.app.flow/flow.yaml") + ).toEqual({ + type: "flow", + }); + }); + + it("returns null for unsupported trigger kinds and non-windmill files", () => { + expect(getValidationTargetFromFilename("README.md")).toBeNull(); + expect(getValidationTargetFromFilename("f/folder/script.py")).toBeNull(); + expect( + getValidationTargetFromFilename("f/folder/resource.yaml") + ).toBeNull(); + }); +}); diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml index 83758cbb48..16109dbc16 100644 --- a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-basic.yaml @@ -12,9 +12,18 @@ value: value: type: aiagent input_transforms: - prompt: + provider: + type: static + value: + kind: openai + resource: "$res:u/admin/openai" + model: gpt-4o-mini + user_message: type: static value: "Please analyze the data" + output_type: + type: static + value: text tools: - id: tool_1 summary: Data fetcher diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml index c8d9fceb55..d619f25da0 100644 --- a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mcp.yaml @@ -6,9 +6,18 @@ value: value: type: aiagent input_transforms: - task: + provider: + type: static + value: + kind: openai + resource: "$res:u/admin/openai" + model: gpt-4o-mini + user_message: type: static value: "Search for information" + output_type: + type: static + value: text tools: - id: mcp_search summary: Search tool from MCP diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml index 902aef8dd7..c9820d0b6e 100644 --- a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-mixed.yaml @@ -11,9 +11,18 @@ value: value: type: aiagent input_transforms: - query: + provider: + type: static + value: + kind: openai + resource: "$res:u/admin/openai" + model: gpt-4o-mini + user_message: type: javascript expr: "flow_input.user_query" + output_type: + type: static + value: text tools: - id: custom_script value: diff --git a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml index 5053b4f9ee..4219d9cc7f 100644 --- a/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml +++ b/windmill-yaml-validator/src/validation/__tests__/test-samples/valid-aiagent-parallel.yaml @@ -7,12 +7,18 @@ value: type: aiagent parallel: true input_transforms: - instruction: + provider: + type: static + value: + kind: openai + resource: "$res:u/admin/openai" + model: gpt-4o-mini + user_message: type: static value: "Process data in parallel" - model: - type: javascript - expr: "'gpt-4'" + output_type: + type: static + value: text tools: - id: parallel_tool_1 summary: First parallel tool diff --git a/windmill-yaml-validator/src/validation/flow-validator.ts b/windmill-yaml-validator/src/validation/flow-validator.ts deleted file mode 100644 index 7969c8fce8..0000000000 --- a/windmill-yaml-validator/src/validation/flow-validator.ts +++ /dev/null @@ -1,50 +0,0 @@ -import Ajv, { AnySchema, ErrorObject, ValidateFunction } from 'ajv'; -import { parseWithPointers, YamlParserResult } from '@stoplight/yaml'; -import openFlowSchema from '../gen/openflow.json'; - -/** - * Flow validator class that initializes AJV once and reuses it for validation. - */ -export class FlowValidator { - private readonly validate: ValidateFunction; - - constructor() { - const ajv = new Ajv({ strict: false, allErrors: true, discriminator: true }); - - for (const [n, s] of Object.entries(openFlowSchema.components.schemas)) { - ajv.addSchema(s as AnySchema, `#/components/schemas/${n}`); - } - - this.validate = ajv.getSchema('#/components/schemas/OpenFlow')!; - } - - /** - * Validates a flow document against the OpenFlow schema. - * @param doc - The YAML flow document as a string - * @returns Object containing the parsed document and any validation errors - */ - validateFlow(doc: string): { - parsed: YamlParserResult; - errors: ErrorObject[]; - } { - if (typeof doc !== 'string') { - throw new Error('Document must be a string'); - } - - const parsed = parseWithPointers(doc); - const { data } = parsed; - const ok = this.validate(data); - - if (ok) { - return { - parsed, - errors: [], - }; - } - - return { - parsed, - errors: this.validate.errors!, - }; - } -} \ No newline at end of file diff --git a/windmill-yaml-validator/src/validation/index.ts b/windmill-yaml-validator/src/validation/index.ts index 3957d14305..1e832c5737 100644 --- a/windmill-yaml-validator/src/validation/index.ts +++ b/windmill-yaml-validator/src/validation/index.ts @@ -1 +1 @@ -export * from './flow-validator'; \ No newline at end of file +export * from "./yaml-validator"; diff --git a/windmill-yaml-validator/src/validation/yaml-validator.ts b/windmill-yaml-validator/src/validation/yaml-validator.ts new file mode 100644 index 0000000000..ace394e7ad --- /dev/null +++ b/windmill-yaml-validator/src/validation/yaml-validator.ts @@ -0,0 +1,141 @@ +import Ajv, { AnySchema, ErrorObject, ValidateFunction } from "ajv"; +import { parseWithPointers, YamlParserResult } from "@stoplight/yaml"; +import openFlowSchema from "../gen/openflow.json"; +import scheduleSchema from "../gen/schedule.json"; +import gcpTriggerSchema from "../gen/triggers/gcp.json"; +import httpTriggerSchema from "../gen/triggers/http.json"; +import kafkaTriggerSchema from "../gen/triggers/kafka.json"; +import mqttTriggerSchema from "../gen/triggers/mqtt.json"; +import natsTriggerSchema from "../gen/triggers/nats.json"; +import postgresTriggerSchema from "../gen/triggers/postgres.json"; +import sqsTriggerSchema from "../gen/triggers/sqs.json"; +import websocketTriggerSchema from "../gen/triggers/websocket.json"; +import emailTriggerSchema from "../gen/triggers/email.json"; + +export const SUPPORTED_TRIGGER_KINDS = [ + "http", + "websocket", + "kafka", + "nats", + "postgres", + "mqtt", + "sqs", + "gcp", + "email", +] as const; + +export type TriggerKind = (typeof SUPPORTED_TRIGGER_KINDS)[number]; + +export type ValidationTarget = + | { type: "flow" } + | { type: "schedule" } + | { type: "trigger"; triggerKind: TriggerKind }; + +const TRIGGER_SCHEMAS: Record = { + http: httpTriggerSchema as AnySchema, + websocket: websocketTriggerSchema as AnySchema, + kafka: kafkaTriggerSchema as AnySchema, + nats: natsTriggerSchema as AnySchema, + postgres: postgresTriggerSchema as AnySchema, + mqtt: mqttTriggerSchema as AnySchema, + sqs: sqsTriggerSchema as AnySchema, + gcp: gcpTriggerSchema as AnySchema, + email: emailTriggerSchema as AnySchema, +}; + +/** + * Infers validation target from file name conventions used by Windmill sync. + */ +export function getValidationTargetFromFilename( + filePath: string +): ValidationTarget | null { + const path = filePath.toLowerCase(); + + if (/[/\\]flow\.ya?ml$/.test(path) || /^flow\.ya?ml$/.test(path)) { + return { type: "flow" }; + } + + if (/\.schedule\.ya?ml$/.test(path)) { + return { type: "schedule" }; + } + + const triggerMatch = path.match( + /\.(http|websocket|kafka|nats|postgres|mqtt|sqs|gcp|email)_trigger\.ya?ml$/ + ); + if (triggerMatch) { + return { + type: "trigger", + triggerKind: triggerMatch[1] as TriggerKind, + }; + } + + return null; +} + +/** + * Unified YAML validator for Windmill flow, schedule, and trigger files. + */ +export class WindmillYamlValidator { + private readonly validateFlow: ValidateFunction; + private readonly validateSchedule: ValidateFunction; + private readonly validateTrigger: Record; + + constructor() { + const ajv = new Ajv({ + strict: false, + allErrors: true, + discriminator: true, + validateFormats: false, + }); + + for (const [name, schema] of Object.entries(openFlowSchema.components.schemas)) { + ajv.addSchema(schema as AnySchema, `#/components/schemas/${name}`); + } + + this.validateFlow = ajv.getSchema("#/components/schemas/OpenFlow")!; + this.validateSchedule = ajv.compile(scheduleSchema as AnySchema); + + this.validateTrigger = Object.fromEntries( + SUPPORTED_TRIGGER_KINDS.map((kind) => [kind, ajv.compile(TRIGGER_SCHEMAS[kind])]) + ) as Record; + } + + /** + * Validates a Windmill YAML document based on the selected target. + * @param doc - The YAML document as string + * @param target - Which Windmill schema to validate against + */ + validate( + doc: string, + target: ValidationTarget + ): { parsed: YamlParserResult; errors: ErrorObject[] } { + if (typeof doc !== "string") { + throw new Error("Document must be a string"); + } + + const parsed = parseWithPointers(doc); + const { data } = parsed; + + let validator: ValidateFunction; + if (target.type === "flow") { + validator = this.validateFlow; + } else if (target.type === "schedule") { + validator = this.validateSchedule; + } else { + validator = this.validateTrigger[target.triggerKind]; + if (!validator) { + throw new Error(`Unsupported trigger kind: ${target.triggerKind}`); + } + } + + const ok = validator(data); + if (ok) { + return { parsed, errors: [] }; + } + + return { + parsed, + errors: validator.errors || [], + }; + } +}