diff --git a/.agents/skills/commit/SKILL.md b/.agents/skills/commit/SKILL.md new file mode 100644 index 0000000000..d610fa3f65 --- /dev/null +++ b/.agents/skills/commit/SKILL.md @@ -0,0 +1,59 @@ +--- +name: commit +description: Create a git commit with conventional commit format. MUST use anytime you want to commit changes. +--- + +# Git Commit Skill + +Create a focused, single-line commit following conventional commit conventions. + +## Instructions + +1. **Analyze changes**: Run `git status` and `git diff` to understand what was modified +2. **Stage only modified files**: Add files individually by name. NEVER use `git add -A` or `git add .` +3. **Write commit message**: Follow the conventional commit format as a single line + +## Conventional Commit Format + +``` +: +``` + +### Types +- `feat`: New feature or capability +- `fix`: Bug fix +- `refactor`: Code change that neither fixes a bug nor adds a feature +- `docs`: Documentation only changes +- `style`: Formatting, missing semicolons, etc (no code change) +- `test`: Adding or correcting tests +- `chore`: Maintenance tasks, dependency updates, etc +- `perf`: Performance improvement + +### Rules +- Message MUST be a single line (no multi-line messages) +- Description should be lowercase, imperative mood ("add" not "added") +- No period at the end +- Keep under 72 characters total + +### Examples +``` +feat: add token usage tracking for AI providers +fix: resolve null pointer in job executor +refactor: extract common validation logic +docs: update API endpoint documentation +chore: upgrade sqlx to 0.7 +``` + +## Execution Steps + +1. Run `git status` to see all changes +2. Run `git diff` to understand the changes in detail +3. Run `git log --oneline -5` to see recent commit style +4. Stage ONLY the modified/relevant files: `git add ...` +5. Create the commit with conventional format: + ```bash + git commit -m ": + + Co-Authored-By: Claude Opus 4.5 " + ``` +6. Run `git status` to verify the commit succeeded diff --git a/.agents/skills/local-review/SKILL.md b/.agents/skills/local-review/SKILL.md new file mode 100644 index 0000000000..ad701ac367 --- /dev/null +++ b/.agents/skills/local-review/SKILL.md @@ -0,0 +1,97 @@ +--- +name: local-review +description: Code review a pull request for bugs and CLAUDE.md compliance. MUST use when asked to review code. +--- + +# Local Code Review Skill + +Review a pull request for real bugs and CLAUDE.md compliance violations. This review targets HIGH SIGNAL issues only. + +## Review Philosophy + +- **Only flag issues you are certain about.** If you are not sure an issue is real, do not flag it. False positives erode trust and waste reviewer time. +- Think like a senior engineer doing a final review — flag things that would cause incidents, not things that are merely imperfect. + +## What to Flag + +- Code that won't compile or parse (syntax errors, type errors, missing imports) +- Code that will definitely produce wrong results regardless of inputs +- Clear, unambiguous CLAUDE.md violations (quote the exact rule being violated) +- Security issues in introduced code (injection, auth bypass, data exposure) +- Incorrect logic that will fail in production + +## What NOT to Flag + +- Code style or quality concerns +- Potential issues that depend on specific inputs or runtime state +- Subjective suggestions or improvements +- Pre-existing issues not introduced by this PR +- Pedantic nitpicks a senior engineer wouldn't flag +- Issues a linter or type checker will catch +- General quality concerns unless explicitly prohibited in CLAUDE.md +- Issues silenced via lint ignore comments + +## Execution Steps + +1. **Determine the PR scope**: + - If an argument is provided, use it as the PR number or branch + - Otherwise, detect from the current branch vs main + - Run `gh pr view` if a PR exists, or use `git diff main...HEAD` + +2. **Find relevant CLAUDE.md files**: + - Read the root `CLAUDE.md` + - Check for CLAUDE.md files in directories containing changed files + +3. **Get the diff and metadata**: + - `gh pr diff` or `git diff main...HEAD` for the full diff + - `gh pr view` or `git log main..HEAD --oneline` for context + +4. **Read changed files** where the diff alone is insufficient to understand context + +5. **Review for**: + - CLAUDE.md compliance — check each rule against the changed code + - Bugs and logic errors — will this code work correctly? + - Security issues — injection, auth, data exposure in new code + +6. **Self-validate each finding**: Before reporting, ask yourself: + - "Is this definitely a real issue, not a false positive?" + - "Would a senior engineer flag this in review?" + - If the answer to either is no, discard the finding + +7. **Output findings** to the terminal (default) or post as PR comments (with `--comment` flag) + +## Output Format + +``` +## Code review + +Found N issues: + +1. () + + +2. () + +``` + +If no issues are found: + +``` +## Code review + +No issues found. Checked for bugs and CLAUDE.md compliance. +``` + +## Posting Comments (--comment flag) + +If the user passes `--comment`, post findings as inline PR comments using: + +```bash +gh pr review --comment --body "" +``` + +Or for inline comments on specific lines: + +```bash +gh api repos/{owner}/{repo}/pulls/{pr}/reviews -f body="" -f event="COMMENT" -f comments="[...]" +``` diff --git a/.agents/skills/native-trigger/SKILL.md b/.agents/skills/native-trigger/SKILL.md new file mode 100644 index 0000000000..026c1900bf --- /dev/null +++ b/.agents/skills/native-trigger/SKILL.md @@ -0,0 +1,777 @@ +# Skill: Adding Native Trigger Services + +This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications. + +## Architecture Overview + +The native trigger system consists of: + +1. **Database Layer** - PostgreSQL tables and enum types +2. **Backend Rust Implementation** - Core trait, handlers, and service modules in the `windmill-native-triggers` crate +3. **Frontend Svelte Components** - Configuration forms and UI components + +### Key Files + +| Component | Path | +|-----------|------| +| Core module with `External` trait | `backend/windmill-native-triggers/src/lib.rs` | +| Generic CRUD handlers | `backend/windmill-native-triggers/src/handler.rs` | +| Background sync logic | `backend/windmill-native-triggers/src/sync.rs` | +| OAuth/workspace integration | `backend/windmill-native-triggers/src/workspace_integrations.rs` | +| Re-export shim (windmill-api) | `backend/windmill-api/src/native_triggers/mod.rs` | +| TriggerKind enum | `backend/windmill-common/src/triggers.rs` | +| JobTriggerKind enum | `backend/windmill-common/src/jobs.rs` | +| Frontend service registry | `frontend/src/lib/components/triggers/native/utils.ts` | +| Frontend trigger utilities | `frontend/src/lib/components/triggers/utils.ts` | +| Trigger badges (icons + counts) | `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte` | +| Workspace integrations UI | `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte` | +| OAuth config form component | `frontend/src/lib/components/workspaceSettings/OAuthClientConfig.svelte` | +| OpenAPI spec | `backend/windmill-api/openapi.yaml` | +| Reference: Nextcloud module | `backend/windmill-native-triggers/src/nextcloud/` | +| Reference: Google module | `backend/windmill-native-triggers/src/google/` | + +### Crate Structure + +The native trigger code lives in the `windmill-native-triggers` crate (`backend/windmill-native-triggers/`). The `windmill-api` crate re-exports everything via a shim: + +```rust +// backend/windmill-api/src/native_triggers/mod.rs +pub use windmill_native_triggers::*; +``` + +All new service modules go in `backend/windmill-native-triggers/src/`. + +--- + +## Core Concepts + +### The `External` Trait + +Every native trigger service implements the `External` trait defined in `lib.rs`: + +```rust +#[async_trait] +pub trait External: Send + Sync + 'static { + // Associated types: + type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + type TriggerData: Debug + Serialize + Send + Sync; + type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; + type CreateResponse: DeserializeOwned + Send + Sync; + + // Constants: + const SUPPORT_WEBHOOK: bool; + const SERVICE_NAME: ServiceName; + const DISPLAY_NAME: &'static str; + const TOKEN_ENDPOINT: &'static str; + const REFRESH_ENDPOINT: &'static str; + const AUTH_ENDPOINT: &'static str; + + // Required methods: + async fn create(&self, w_id, oauth_data, webhook_token, data, db, tx) -> Result; + async fn update(&self, w_id, oauth_data, external_id, webhook_token, data, db, tx) -> Result; + async fn get(&self, w_id, oauth_data, external_id, db, tx) -> Result; + async fn delete(&self, w_id, oauth_data, external_id, db, tx) -> Result<()>; + async fn exists(&self, w_id, oauth_data, external_id, db, tx) -> Result; + async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors); + fn external_id_and_metadata_from_response(&self, resp) -> (String, Option); + + // Methods with defaults: + async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result; + fn service_config_from_create_response(&self, data, resp) -> Option; + fn additional_routes(&self) -> axum::Router; + async fn http_client_request(&self, url, method, workspace_id, tx, db, headers, body) -> Result; +} +``` + +Key design points: +- **`update()` returns `serde_json::Value`** - the resolved service_config to store. Each service is responsible for building the final config. +- **`maintain_triggers()`** - periodic background maintenance. Each service implements its own strategy (Nextcloud: reconcile with external state; Google: renew expiring channels). +- **No `list_all()` in the trait** - services that need it (Nextcloud) implement it privately; services that don't (Google) use different maintenance strategies. +- **No `get_external_id_from_trigger_data()` or `extract_service_config_from_trigger_data()`** - removed in favor of the `maintain_triggers` pattern. + +### Create Lifecycle: Two Paths + +The `create_native_trigger` handler in `handler.rs` supports two creation flows, controlled by `service_config_from_create_response()`: + +**Path A: Short (Google pattern)** - `service_config_from_create_response()` returns `Some(config)`: +1. `create()` registers on external service +2. `external_id_and_metadata_from_response()` extracts the ID +3. `service_config_from_create_response()` builds the config directly from input data + response metadata +4. Stores trigger in DB -- done, no extra round-trip + +Use this when the external_id is known before the create call (e.g., Google generates the channel_id as a UUID upfront and includes it in the webhook URL). + +**Path B: Long (Nextcloud pattern)** - `service_config_from_create_response()` returns `None` (default): +1. `create()` registers on external service (webhook URL has no external_id yet) +2. `external_id_and_metadata_from_response()` extracts the ID +3. `update()` is called to fix the webhook URL with the now-known external_id +4. `update()` returns the resolved service_config +5. Stores trigger in DB + +Use this when the external_id is assigned by the remote service and the webhook URL needs to be corrected after creation. + +### OAuth Token Storage (Three-Table Pattern) + +OAuth tokens are stored across three tables, NOT in `workspace_integrations.oauth_data` directly: + +| Table | What's Stored | +|-------|---------------| +| `workspace_integrations` | `oauth_data` JSON with `base_url`, `client_id`, `client_secret`, `instance_shared` flag; `resource_path` pointing to the variable | +| `variable` | Encrypted `access_token` (at the path stored in `resource_path`), linked to `account` via `account` column | +| `account` | `refresh_token`, keyed by `workspace_id` + `client` (service name) + `is_workspace_integration = true` | + +The `decrypt_oauth_data()` function in `lib.rs` assembles these into a unified struct: +```rust +pub struct OAuthConfig { + pub base_url: String, + pub access_token: String, // decrypted from variable + pub refresh_token: Option, // from account table + pub client_id: String, // from oauth_data or instance settings + pub client_secret: String, // from oauth_data or instance settings +} +``` + +Instance-level sharing: when `oauth_data.instance_shared == true`, `client_id` and `client_secret` are read from global settings instead of workspace_integrations. + +### URL Resolution + +The `resolve_endpoint()` helper handles both absolute and relative OAuth URLs: + +```rust +pub fn resolve_endpoint(base_url: &str, endpoint: &str) -> String { + if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() // Google: absolute URLs + } else { + format!("{}{}", base_url, endpoint) // Nextcloud: relative paths + } +} +``` + +### ServiceName Methods + +`ServiceName` is the central registry enum. Each variant must implement these match arms: + +| Method | Purpose | +|--------|---------| +| `as_str()` | Lowercase identifier (e.g., `"google"`) | +| `as_trigger_kind()` | Maps to `TriggerKind` enum | +| `as_job_trigger_kind()` | Maps to `JobTriggerKind` enum | +| `token_endpoint()` | OAuth token endpoint (relative or absolute) | +| `auth_endpoint()` | OAuth authorization endpoint | +| `oauth_scopes()` | Space-separated OAuth scopes | +| `resource_type()` | Resource type for token storage (e.g., `"gworkspace"`) | +| `extra_auth_params()` | Extra OAuth params (e.g., Google needs `access_type=offline`, `prompt=consent`) | +| `integration_service()` | Maps to the workspace integration service (usually `*self`) | +| `TryFrom` | Parse from string | +| `Display` | Delegates to `as_str()` | + +--- + +## Step-by-Step Implementation Guide + +### Step 1: Database Migration + +Create a new migration file: `backend/migrations/YYYYMMDDHHMMSS_newservice_trigger.up.sql` + +```sql +-- Add the service to the native_trigger_service enum +ALTER TYPE native_trigger_service ADD VALUE IF NOT EXISTS 'newservice'; + +-- Add to TRIGGER_KIND enum (used for trigger tracking) +ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'newservice'; + +-- Add to job_trigger_kind enum (used for job tracking) +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'newservice'; +``` + +Also create the corresponding down migration. + +### Step 2: Update windmill-common Enums + +#### `backend/windmill-common/src/triggers.rs` + +Add variant to `TriggerKind` enum, and update `to_key()` and `fmt()` implementations. + +#### `backend/windmill-common/src/jobs.rs` + +Add variant to `JobTriggerKind` enum and update the `Display` implementation. + +### Step 3: Backend Service Module + +Create a new directory: `backend/windmill-native-triggers/src/newservice/` + +#### `mod.rs` - Type Definitions + +```rust +use serde::{Deserialize, Serialize}; + +pub mod external; +// pub mod routes; // Only if you need additional service-specific routes + +/// OAuth data deserialized from the three-table pattern. +/// The actual structure is built by decrypt_oauth_data() from variable + account + workspace_integrations. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct NewServiceOAuthData { + pub base_url: String, // from workspace_integrations.oauth_data + pub access_token: String, // decrypted from variable table + pub refresh_token: Option, // from account table + // Note: client_id and client_secret are in OAuthConfig, not here + // unless the service needs them at runtime for API calls +} + +/// Configuration provided by user when creating/updating a trigger. +/// Stored as JSON in native_trigger.service_config. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewServiceConfig { + // Service-specific configuration fields + pub folder_path: String, + pub file_filter: Option, +} + +/// Data retrieved from the external service about a trigger. +/// Returned by the get() method and shown in the UI. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewServiceTriggerData { + pub folder_path: String, + pub file_filter: Option, + // Fields that shouldn't affect service_config comparison should use #[serde(skip_serializing)] +} + +/// Response from external service when creating a trigger/webhook. +#[derive(Debug, Deserialize)] +pub struct CreateTriggerResponse { + pub id: String, +} + +/// Handler struct (stateless, used for routing) +#[derive(Copy, Clone)] +pub struct NewService; +``` + +#### `external.rs` - External Trait Implementation + +```rust +use async_trait::async_trait; +use reqwest::Method; +use sqlx::PgConnection; +use std::collections::HashMap; +use windmill_common::{ + error::{Error, Result}, + BASE_URL, DB, +}; + +use crate::{ + generate_webhook_service_url, External, NativeTrigger, NativeTriggerData, ServiceName, + sync::{SyncError, TriggerSyncInfo}, +}; +use super::{NewService, NewServiceConfig, NewServiceOAuthData, NewServiceTriggerData, CreateTriggerResponse}; + +#[async_trait] +impl External for NewService { + type ServiceConfig = NewServiceConfig; + type TriggerData = NewServiceTriggerData; + type OAuthData = NewServiceOAuthData; + type CreateResponse = CreateTriggerResponse; + + const SERVICE_NAME: ServiceName = ServiceName::NewService; + const DISPLAY_NAME: &'static str = "New Service"; + const SUPPORT_WEBHOOK: bool = true; + const TOKEN_ENDPOINT: &'static str = "/oauth/token"; + const REFRESH_ENDPOINT: &'static str = "/oauth/token"; + const AUTH_ENDPOINT: &'static str = "/oauth/authorize"; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let base_url = &*BASE_URL.read().await; + + // external_id is None during create (we get it from the response) + let webhook_url = generate_webhook_service_url( + base_url, w_id, &data.script_path, data.is_flow, + None, Self::SERVICE_NAME, webhook_token, + ); + + let url = format!("{}/api/webhooks/create", oauth_data.base_url); + let payload = serde_json::json!({ + "callback_url": webhook_url, + "folder_path": data.service_config.folder_path, + }); + + let response: CreateTriggerResponse = self + .http_client_request(&url, Method::POST, w_id, tx, db, None, Some(&payload)) + .await?; + + Ok(response) + } + + /// Update returns the resolved service_config as JSON. + /// For services using the update+get pattern, call self.get() and serialize. + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let base_url = &*BASE_URL.read().await; + + let webhook_url = generate_webhook_service_url( + base_url, w_id, &data.script_path, data.is_flow, + Some(external_id), Self::SERVICE_NAME, webhook_token, + ); + + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + let payload = serde_json::json!({ + "callback_url": webhook_url, + "folder_path": data.service_config.folder_path, + }); + + let _: serde_json::Value = self + .http_client_request(&url, Method::PUT, w_id, tx, db, None, Some(&payload)) + .await?; + + // Fetch back the updated state to get the resolved config + let trigger_data = self.get(w_id, oauth_data, external_id, db, tx).await?; + serde_json::to_value(&trigger_data) + .map_err(|e| Error::InternalErr(format!("Failed to serialize trigger data: {}", e))) + } + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + self.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, None, None).await + } + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()> { + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + let _: serde_json::Value = self + .http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, None, None) + .await + .or_else(|e| match &e { + Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null), + _ => Err(e), + })?; + Ok(()) + } + + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + match self.get(w_id, oauth_data, external_id, db, tx).await { + Ok(_) => Ok(true), + Err(Error::NotFound(_)) => Ok(false), + Err(e) => Err(e), + } + } + + /// Background maintenance. Choose the right pattern for your service: + /// - For services with queryable external state: use reconcile_with_external_state() + /// - For channel-based services with expiration: implement renewal logic + async fn maintain_triggers( + &self, + db: &DB, + workspace_id: &str, + triggers: &[NativeTrigger], + oauth_data: &Self::OAuthData, + synced: &mut Vec, + errors: &mut Vec, + ) { + // Option A: Reconcile with external state (Nextcloud pattern) + // Fetch all triggers from external service and compare with DB + let external_triggers = match self.list_all(workspace_id, oauth_data, db).await { + Ok(triggers) => triggers, + Err(e) => { + errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!("Failed to list triggers: {}", e), + error_type: "api_error".to_string(), + }); + return; + } + }; + + // Convert to (external_id, config_json) pairs + let external_pairs: Vec<(String, serde_json::Value)> = external_triggers + .into_iter() + .map(|t| (t.id.clone(), serde_json::to_value(&t).unwrap_or_default())) + .collect(); + + crate::sync::reconcile_with_external_state( + db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, + ).await; + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option) { + (resp.id.clone(), None) + } + + // service_config_from_create_response: NOT overridden (returns None). + // This means the handler uses the update+get pattern after create. + // Override and return Some(...) to skip the update+get cycle (Google pattern). +} + +impl NewService { + /// Private helper to list all triggers from the external service. + async fn list_all( + &self, + w_id: &str, + oauth_data: &::OAuthData, + db: &DB, + ) -> Result::TriggerData>> { + // Implementation depends on the external service's API + todo!() + } +} +``` + +### Step 4: Update lib.rs Registry + +In `backend/windmill-native-triggers/src/lib.rs`: + +```rust +// Service modules - add new services here: +#[cfg(feature = "native_trigger")] +pub mod newservice; // <-- Add this + +// ServiceName enum - add variant: +pub enum ServiceName { + Nextcloud, + Google, + NewService, // <-- Add this +} + +// Then add match arms in ALL ServiceName methods: +// as_str(), as_trigger_kind(), as_job_trigger_kind(), token_endpoint(), +// auth_endpoint(), oauth_scopes(), resource_type(), extra_auth_params(), +// integration_service(), TryFrom, Display +``` + +### Step 5: Update handler.rs Routes + +In `backend/windmill-native-triggers/src/handler.rs`: + +```rust +pub fn generate_native_trigger_routers() -> Router { + // ... + #[cfg(feature = "native_trigger")] + { + use crate::newservice::NewService; + return router + .nest("/nextcloud", service_routes(NextCloud)) + .nest("/google", service_routes(Google)) + .nest("/newservice", service_routes(NewService)); // <-- Add this + } + // ... +} +``` + +### Step 6: Update sync.rs + +In `backend/windmill-native-triggers/src/sync.rs`: + +```rust +pub async fn sync_all_triggers(db: &DB) -> Result { + // ... + #[cfg(feature = "native_trigger")] + { + use crate::newservice::NewService; + + // ... existing service syncs ... + + // New service sync + let (service_name, result) = sync_service_triggers(db, NewService).await; + total_synced += result.synced_triggers.len(); + total_errors += result.errors.len(); + service_results.insert(service_name, result); + } + // ... +} +``` + +### Step 7: Frontend Service Registry + +In `frontend/src/lib/components/triggers/native/utils.ts`: + +Add to `NATIVE_TRIGGER_SERVICES`, `getTriggerIconName()`, and `getServiceIcon()`. + +### Step 8: Frontend Trigger Form Component + +Create: `frontend/src/lib/components/triggers/native/services/newservice/NewServiceTriggerForm.svelte` + +### Step 9: Frontend Icon Component + +Create: `frontend/src/lib/components/icons/NewServiceIcon.svelte` + +### Step 10: Update NativeTriggerEditor + +Check `frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte` to ensure it dynamically loads form components based on service name. + +### Step 11: Workspace Integration UI + +Add your service to the `supportedServices` map in `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte`: + +```typescript +const supportedServices: Record = { + // ... existing services ... + newservice: { + name: 'newservice', + displayName: 'New Service', + description: 'Connect to New Service for triggers', + icon: NewServiceIcon, + docsUrl: 'https://www.windmill.dev/docs/integrations/newservice', + requiresBaseUrl: false, // false for cloud services, true for self-hosted + setupInstructions: [ + 'Step 1: Create an OAuth app on the service', + 'Step 2: Configure the redirect URI shown below', + 'Step 3: Enter the client credentials below' + ] + } +} +``` + +### Step 12: Update `frontend/src/lib/components/triggers/utils.ts` + +Update ALL of these maps/functions: +1. `triggerIconMap` - import and add icon +2. `triggerDisplayNamesMap` - add display name +3. `triggerTypeOrder` in `sortTriggers()` - add type +4. `getLightConfig()` - add case for your service +5. `getTriggerLabel()` - add case for your service +6. `jobTriggerKinds` - add to array +7. `countPropertyMap` - add count property +8. `triggerSaveFunctions` - add save function + +### Step 13: Update TriggersBadge Component + +In `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte`: + +1. Import the icon +2. Add to `baseConfig` with `countKey` (the dynamic `availableNativeServices` loop does NOT set `countKey`) +3. Add to the `allTypes` array + +### Step 14: Update TriggersWrapper.svelte + +In `frontend/src/lib/components/triggers/TriggersWrapper.svelte`: + +Add a `{:else if selectedTrigger.type === 'yourservice'}` case that renders `` with the same props pattern as the existing native trigger cases (e.g., `nextcloud`). + +### Step 15: Update AddTriggersButton.svelte + +In `frontend/src/lib/components/triggers/AddTriggersButton.svelte`: + +1. Add `yourserviceAvailable` state variable +2. Add `setYourserviceState()` async function using `isServiceAvailable('yourservice', $workspaceStore!)` +3. Call it at module level +4. Add a dropdown entry to `addTriggerItems` with `hidden: !yourserviceAvailable` + +### Step 16: Update TriggersEditor.svelte Delete Handling + +In `frontend/src/lib/components/triggers/TriggersEditor.svelte`: + +Add your service to the `nativeTriggerServices` map in `deleteDeployedTrigger()`. Native triggers use `NativeTriggerService.deleteNativeTrigger({ workspace, serviceName, externalId })` instead of the standard `path`-based delete. + +### Step 17: Update OpenAPI Spec and Regenerate Types + +Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then: + +```bash +cd frontend && npm run generate-backend-client +``` + +--- + +## Special Patterns + +### Unified Service with `trigger_type` (Google Pattern) + +When a single service handles multiple trigger types (e.g., Google Drive + Calendar share OAuth and API patterns), use a single `ServiceName` variant with a discriminator field: + +```rust +pub enum GoogleTriggerType { Drive, Calendar } + +pub struct GoogleServiceConfig { + pub trigger_type: GoogleTriggerType, + // Drive-specific fields (only used when trigger_type = Drive) + pub resource_id: Option, + pub resource_name: Option, + // Calendar-specific fields (only used when trigger_type = Calendar) + pub calendar_id: Option, + pub calendar_name: Option, + // Metadata set after creation + pub google_resource_id: Option, + pub expiration: Option, +} +``` + +Branch in trait methods based on `trigger_type`. Frontend uses a `ToggleButtonGroup` to switch between types. This keeps the codebase simpler (one service, one OAuth flow, one set of routes). + +See `backend/windmill-native-triggers/src/google/` for the reference implementation. + +### Skipping update+get After Create (Google Pattern) + +Override `service_config_from_create_response()` to return `Some(config)` when the external_id is known before the create call: + +```rust +fn service_config_from_create_response( + &self, + data: &NativeTriggerData, + resp: &Self::CreateResponse, +) -> Option { + // Clone input config, add metadata from response + let mut config = data.service_config.clone(); + config.google_resource_id = Some(resp.resource_id.clone()); + config.expiration = Some(resp.expiration.clone()); + Some(serde_json::to_value(&config).unwrap()) +} +``` + +### Services with Absolute OAuth Endpoints (Google) + +Unlike self-hosted services where OAuth endpoints are relative paths appended to `base_url`, services like Google have absolute URLs: + +```rust +// Nextcloud: relative paths +ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", +// Google: absolute URLs +ServiceName::Google => "https://oauth2.googleapis.com/token", +``` + +The `resolve_endpoint()` function handles both. For services with absolute endpoints: +- `base_url` can be empty +- `requiresBaseUrl: false` in the frontend workspace integration config +- Add `extra_auth_params()` if needed (Google requires `access_type=offline` and `prompt=consent`) + +### Channel-Based Push Notifications with Renewal (Google Pattern) + +For services using expiring watch channels instead of persistent webhooks: + +1. Store expiration in `service_config` (as part of `ServiceConfig`) +2. In `maintain_triggers()`, implement renewal logic instead of using `reconcile_with_external_state()`: + ```rust + async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors) { + for trigger in triggers { + if should_renew_channel(trigger) { + self.renew_channel(db, trigger, oauth_data).await; + } + } + } + ``` +3. Renewal: best-effort stop old channel, create new one with same external_id, update service_config with new expiration +4. Google example: Drive channels expire in 24h (renew when <1h left), Calendar channels expire in 7 days (renew when <1 day left) + +### reconcile_with_external_state (Nextcloud Pattern) + +The reusable function in `sync.rs` compares external triggers with DB state: +- Triggers missing externally: sets error "Trigger no longer exists on external service" +- Triggers present externally: clears errors, updates service_config if it differs + +Usage in `maintain_triggers()`: +```rust +let external_pairs: Vec<(String, serde_json::Value)> = /* fetch from external */; +crate::sync::reconcile_with_external_state( + db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, +).await; +``` + +### Webhook Payload Processing + +Override `prepare_webhook()` to parse service-specific payloads into script/flow args: + +```rust +async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result { + let mut args = HashMap::new(); + args.insert("event_type".to_string(), Box::new(headers.get("x-event-type").cloned()) as _); + args.insert("payload".to_string(), Box::new(serde_json::from_str::(&body)?) as _); + Ok(PushArgsOwned { extra: None, args }) +} +``` + +Then register in `prepare_native_trigger_args()` in `lib.rs`: +```rust +pub async fn prepare_native_trigger_args(service_name, db, w_id, headers, body) -> Result> { + match service_name { + ServiceName::Google => { /* ... */ Ok(Some(args)) } + ServiceName::NewService => { /* ... */ Ok(Some(args)) } + ServiceName::Nextcloud => Ok(None), // Uses default body parsing + } +} +``` + +### Instance-Level OAuth Credentials + +When `workspace_integrations.oauth_data.instance_shared == true`, `decrypt_oauth_data()` reads `client_id` and `client_secret` from instance-level global settings instead of workspace-level. This allows admins to share OAuth app credentials across workspaces. + +The frontend handles this via the `generate_instance_connect_url` endpoint in `workspace_integrations.rs`. + +--- + +## Testing Checklist + +- [ ] Database migration runs successfully +- [ ] `cargo check -p windmill-native-triggers --features native_trigger` passes +- [ ] `npx svelte-check --threshold error` passes (in frontend/) +- [ ] Service appears in workspace integrations list +- [ ] OAuth flow completes successfully +- [ ] Can create a new trigger +- [ ] Can view trigger details +- [ ] Can update trigger configuration +- [ ] Can delete trigger +- [ ] Webhook receives and processes payloads +- [ ] Background sync works correctly (reconciliation or channel renewal) +- [ ] Error handling works (expired tokens, service unavailable) + +--- + +## Reference Implementations + +### Nextcloud (Self-Hosted, Update+Get Pattern) + +| File | Purpose | +|------|---------| +| `nextcloud/mod.rs` | Types: NextCloudOAuthData, NextcloudServiceConfig, NextCloudTriggerData | +| `nextcloud/external.rs` | External trait: uses update+get pattern, reconcile_with_external_state for sync | +| `nextcloud/routes.rs` | Additional route: `GET /events` | + +Key patterns: relative OAuth endpoints, base_url required, list_all + reconcile for sync, update returns JSON from get(). + +### Google (Cloud, Unified Service, Short Create) + +| File | Purpose | +|------|---------| +| `google/mod.rs` | Types: GoogleServiceConfig with trigger_type discriminator, GoogleTriggerType enum | +| `google/external.rs` | External trait: overrides service_config_from_create_response, channel renewal for sync | +| `google/routes.rs` | Additional routes: `GET /calendars`, `GET /drive/files`, `GET /drive/shared_drives` | + +Key patterns: absolute OAuth endpoints, empty base_url, trigger_type for Drive/Calendar, expiring watch channels with renewal, service_config_from_create_response skips update+get, get() reconstructs data from stored service_config (no external "get channel" API). diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md new file mode 100644 index 0000000000..2efcc4e0a6 --- /dev/null +++ b/.agents/skills/pr/SKILL.md @@ -0,0 +1,109 @@ +--- +name: pr +description: Open a draft pull request on GitHub. MUST use when you want to create/open a PR. +--- + +# Pull Request Skill + +Create a draft pull request with a clear title and explicit description of changes. + +## Instructions + +1. **Analyze branch changes**: Understand all commits since diverging from main +2. **Push to remote**: Ensure all commits are pushed +3. **Create draft PR**: Always open as draft for review before merging + +## PR Title Format + +Follow conventional commit format for the PR title: +``` +: +``` + +### Types +- `feat`: New feature or capability +- `fix`: Bug fix +- `refactor`: Code restructuring +- `docs`: Documentation changes +- `chore`: Maintenance tasks +- `perf`: Performance improvements + +### Title Rules +- Keep under 70 characters +- Use lowercase, imperative mood +- No period at the end +- If `*_ee.rs` files were modified, prefix with `[ee]`: `[ee] : ` + +## PR Body Format + +The body MUST be explicit about what changed. Structure: + +```markdown +## Summary + + +## Changes +- +- +- + +## Test plan +- [ ] +- [ ] + +--- +Generated with [Claude Code](https://claude.com/claude-code) +``` + +## Execution Steps + +1. Run `git status` to check for uncommitted changes +2. Run `git log main..HEAD --oneline` to see all commits in this branch +3. Run `git diff main...HEAD` to see the full diff against main +4. Check if remote branch exists and is up to date: + ```bash + git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream" + ``` +5. Push to remote if needed: `git push -u origin HEAD` +6. Create draft PR using gh CLI: + ```bash + gh pr create --draft --title ": " --body "$(cat <<'EOF' + ## Summary + + + ## Changes + - + - + + ## Test plan + - [ ] + - [ ] + + --- + Generated with [Claude Code](https://claude.com/claude-code) + EOF + )" + ``` +7. Return the PR URL to the user + +## EE Companion PR (when `*_ee.rs` files were modified) + +The `*_ee.rs` files in the windmill repo are **symlinks** to `windmill-ee-private` — changes won't appear in `git diff` of the windmill repo. Instead, check the EE repo for uncommitted or unpushed changes. + +Follow the full EE PR workflow in `docs/enterprise.md`. The key PR-specific details: + +1. Find the EE repo/worktree: see "Finding the EE Repo" in `docs/enterprise.md` +2. Check for changes: `git -C status --short` + - If there are no changes in the EE repo, skip this entire section +3. Follow steps 1–5 from the "EE PR Workflow" in `docs/enterprise.md` +4. Create the companion PR (title does NOT get the `[ee]` prefix): + ```bash + gh pr create --draft --repo windmill-labs/windmill-ee-private --title ": " --body "$(cat <<'EOF' + Companion PR for windmill-labs/windmill# + + --- + Generated with [Claude Code](https://claude.com/claude-code) + EOF + )" + ``` +5. Commit `ee-repo-ref.txt` and push the updated windmill branch diff --git a/.agents/skills/refine/SKILL.md b/.agents/skills/refine/SKILL.md new file mode 100644 index 0000000000..b96e97e8a2 --- /dev/null +++ b/.agents/skills/refine/SKILL.md @@ -0,0 +1,38 @@ +--- +name: refine +description: End-of-session reflection. Reviews friction encountered during the session and proposes updates to docs/ to capture lessons learned. +--- + +# Refine Skill + +Reflect on the current session and update documentation with lessons learned. + +## Instructions + +1. **Identify friction**: Review what happened in this session: + - Run `git diff main...HEAD --stat` to see what files were touched + - Think about: what was slow, what failed, what required multiple attempts, what information was missing or hard to find + +2. **Read current docs**: Read the docs that were relevant to this session: + - `docs/validation.md` + - `docs/enterprise.md` + - `docs/autonomous-mode.md` + - Any skills that were invoked + +3. **Propose updates**: For each piece of friction, decide if it warrants a doc update: + - **Missing knowledge**: Information you had to discover that should be documented + - **Wrong guidance**: Instructions that led you astray + - **Missing validation rule**: A check that should be in the validation matrix + - **New pattern**: A codebase pattern worth capturing for next time + +4. **Apply updates**: Edit the relevant `docs/` files. Keep changes minimal and specific — add only what would have saved time this session. + +5. **Report**: Summarize what was added/changed and why. + +## Rules + +- Only add knowledge confirmed by this session — no speculative additions +- Keep docs concise — add a line or two, not a paragraph +- If a whole new doc is needed, create it in `docs/` and add a pointer in `CLAUDE.md` +- Don't update skills unless a coding pattern was genuinely wrong +- Don't add things Claude already knows — only Windmill-specific knowledge diff --git a/.agents/skills/rust-backend/SKILL.md b/.agents/skills/rust-backend/SKILL.md new file mode 100644 index 0000000000..f0c52002bc --- /dev/null +++ b/.agents/skills/rust-backend/SKILL.md @@ -0,0 +1,107 @@ +--- +name: rust-backend +description: Rust coding guidelines for the Windmill backend. MUST use when writing or modifying Rust code in the backend directory. +--- + +# Windmill Rust Patterns + +Apply these Windmill-specific patterns when writing Rust code in `backend/`. + +## Error Handling + +Use `Error` from `windmill_common::error`. Return `Result` or `JsonResult`: + +```rust +use windmill_common::error::{Error, Result}; + +pub async fn get_job(db: &DB, id: Uuid) -> Result { + sqlx::query_as!(Job, "SELECT id, workspace_id FROM v2_job WHERE id = $1", id) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound("job not found".to_string()))?; +} +``` + +Never panic in library code. Reserve `.unwrap()` for compile-time guarantees. + +## SQLx Patterns + +**Never use `SELECT *`** — always list columns explicitly. Critical for backwards compatibility when workers lag behind API version: + +```rust +// Correct +sqlx::query_as!(Job, "SELECT id, workspace_id, path FROM v2_job WHERE id = $1", id) + +// Wrong — breaks when columns are added +sqlx::query_as!(Job, "SELECT * FROM v2_job WHERE id = $1", id) +``` + +Use batch operations to avoid N+1: + +```rust +// Preferred — single query with IN clause +sqlx::query!("SELECT ... WHERE id = ANY($1)", &ids[..]).fetch_all(db).await? +``` + +Use transactions for multi-step operations. Parameterize all queries. + +## JSON Handling + +Prefer `Box` over `serde_json::Value` when storing/passing JSON without inspection: + +```rust +pub struct Job { + pub args: Option>, +} +``` + +Only use `serde_json::Value` when you need to inspect or modify the JSON. + +## Serde Optimizations + +```rust +#[derive(Serialize, Deserialize)] +pub struct Job { + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tags: Vec, + #[serde(default)] + pub priority: i32, +} +``` + +## Async & Concurrency + +Never block the async runtime. Use `spawn_blocking` for CPU-intensive work: + +```rust +let result = tokio::task::spawn_blocking(move || expensive_computation(&data)).await?; +``` + +**Mutex selection**: Prefer `std::sync::Mutex` (or `parking_lot::Mutex`) for data protection. Only use `tokio::sync::Mutex` when holding locks across `.await` points. + +Use `tokio::sync::mpsc` (bounded) for channels. Avoid `std::thread::sleep` in async contexts. + +## Module Structure & Visibility + +- Use `pub(crate)` instead of `pub` when possible +- Place new code in the appropriate crate based on functionality +- API endpoints go in `windmill-api/src/` organized by domain +- Shared functionality goes in `windmill-common/src/` + +## Code Navigation + +Always use rust-analyzer LSP for go-to-definition, find-references, and type info. Do not guess at module paths. + +## Axum Handlers + +Destructure extractors directly in function signatures: + +```rust +async fn process_job( + Extension(db): Extension, + Path((workspace, job_id)): Path<(String, Uuid)>, + Query(pagination): Query, +) -> Result> { ... } +``` diff --git a/.agents/skills/svelte-frontend/SKILL.md b/.agents/skills/svelte-frontend/SKILL.md new file mode 100644 index 0000000000..57cac70302 --- /dev/null +++ b/.agents/skills/svelte-frontend/SKILL.md @@ -0,0 +1,80 @@ +--- +name: svelte-frontend +description: Svelte coding guidelines for the Windmill frontend. MUST use when writing or modifying code in the frontend directory. +--- + +# Windmill Svelte Patterns + +Apply these Windmill-specific patterns when writing Svelte code in `frontend/`. For general Svelte 5 syntax (runes, snippets, event handling), use the Svelte MCP server. + +## Windmill UI Components (MUST use) + +Always use Windmill's design-system components. Never use raw HTML elements. + +### Buttons — ` +remove - {/each} + + {/each} - {/snippet} + {/snippet} {:else}
- {#each new Array(6) as _} + {#each new Array(6) as _, i (i)} {/each}
diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 1c69ac8a80..a429920e89 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -23,6 +23,7 @@ import SmtpSettings from './instanceSettings/SmtpSettings.svelte' import SecretBackendConfig from './instanceSettings/SecretBackendConfig.svelte' import GhesAppSettings from './instanceSettings/GhesAppSettings.svelte' + import WsConnectivityTest from './instanceSettings/WsConnectivityTest.svelte' import IndexerMemorySettings from './instanceSettings/IndexerMemorySettings.svelte' import IndexerJobIndexSettings from './instanceSettings/IndexerJobIndexSettings.svelte' import IndexerLogIndexSettings from './instanceSettings/IndexerLogIndexSettings.svelte' @@ -285,12 +286,16 @@ {:else} @@ -719,6 +724,8 @@ {:else if setting.fieldType == 'github_enterprise_app'} + {:else if setting.fieldType == 'ws_connectivity'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 5f5ac925f4..18514b3836 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -20,6 +20,7 @@ import Toggle from './Toggle.svelte' import SettingsFooter from './workspaceSettings/SettingsFooter.svelte' import SettingsPageHeader from './settings/SettingsPageHeader.svelte' + import WorkspaceRegistries from './instanceSettings/WorkspaceRegistries.svelte' interface Props { tab?: string @@ -267,12 +268,13 @@ async function downloadStats() { try { downloadingStats = true - const encryptedData = await SettingService.getStats() - const blob = new Blob([encryptedData], { type: 'application/octet-stream' }) + const result = await SettingService.getStats() + const blob = new Blob([result.data ?? ''], { type: 'application/json' }) const url = URL.createObjectURL(blob) const a = document.createElement('a') a.href = url - a.download = `windmill-telemetry-${new Date().toISOString().split('T')[0]}.enc` + const date = new Date().toISOString().split('T')[0] + a.download = `windmill-telemetry-${date}-${result.signature}.json` document.body.appendChild(a) a.click() document.body.removeChild(a) @@ -401,6 +403,9 @@ obj['require_preexisting_user_for_oauth'] = reqPreexisting } } + if (category === 'Registries') { + obj['workspace_registries'] = vals['workspace_registries'] ?? null + } return YAML.stringify(obj) } @@ -466,6 +471,11 @@ const v = initialValues[s.key] $values[s.key] = v !== undefined ? JSON.parse(JSON.stringify(v)) : undefined } + if (category === 'Registries') { + const v = initialValues['workspace_registries'] + $values['workspace_registries'] = + v !== undefined ? JSON.parse(JSON.stringify(v)) : undefined + } } } @@ -569,6 +579,19 @@ } } + // Handle workspace_registries (saved separately like oauths) + if (category === 'Registries') { + if (!deepEqual(initialValues['workspace_registries'], $values['workspace_registries'])) { + await SettingService.setGlobal({ + key: 'workspace_registries', + requestBody: { value: $values['workspace_registries'] ?? null } + }) + initialValues['workspace_registries'] = $values['workspace_registries'] + ? JSON.parse(JSON.stringify($values['workspace_registries'])) + : undefined + } + } + if (licenseKeySet) setLicense() if (shouldReloadPage) { @@ -595,7 +618,8 @@ .filter((s) => s.fieldType === 'password' || s.fieldType === 'license_key') .map((s) => s.key), 'ducklake_user_pg_pwd', - 'jwt_secret' + 'jwt_secret', + 'workspace_registries' ]) // Settings that should never appear in YAML export/import @@ -954,6 +978,10 @@
When minimal telemetry is disabled, the following is also collected:
  • job usage (language, total duration, count)
    • +
  • git sync repo count (sync vs promotion mode)
    • +
  • AI chat usage (provider, model, mode, session count, message count — last 30 days)

For air-gapped instances, you can download the telemetry data and send it manually. @@ -989,6 +1017,9 @@
  • worker usage (worker, worker instance, vCPUs, memory)
  • user usage (author count, operator count)
  • development instance status
    • +
  • AI chat usage (provider, model, mode, session count, message count — last 30 days)
    • {/if} @@ -1104,6 +1135,10 @@ {/if} + {#if category === 'Registries'} + + {/if} + {#if !loading && !quickSetup && !hideTabs} | null ): Promise { return abstractRun( () => @@ -310,7 +311,8 @@ tag, lock, script_hash: hash, - flow_path: flowPath + flow_path: flowPath, + modules: modules ?? undefined } }), callbacks diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index b9c0de018d..1e375cb53d 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -341,6 +341,13 @@ btnClasses="mt-2 w-full" on:click={() => { if (saml) { + if (rd) { + try { + localStorage.setItem('rd', rd) + } catch (e) { + console.error('Could not persist redirection to local storage', e) + } + } window.location.href = saml } else { sendUserToast('No SAML login available', true) diff --git a/frontend/src/lib/components/NoMainFuncBadge.svelte b/frontend/src/lib/components/NoMainFuncBadge.svelte index 3e2a2d28bb..c72e05583c 100644 --- a/frontend/src/lib/components/NoMainFuncBadge.svelte +++ b/frontend/src/lib/components/NoMainFuncBadge.svelte @@ -5,7 +5,7 @@ {#snippet text()} - The script has no main function exported + Library script (no exported main function) {/snippet} - No main + Library diff --git a/frontend/src/lib/components/OnBehalfOfSelector.svelte b/frontend/src/lib/components/OnBehalfOfSelector.svelte index 9921972a39..55ea1a99f4 100644 --- a/frontend/src/lib/components/OnBehalfOfSelector.svelte +++ b/frontend/src/lib/components/OnBehalfOfSelector.svelte @@ -1,16 +1,25 @@ @@ -24,31 +33,39 @@ interface Props { targetWorkspace: string - targetEmail: string | undefined + /** The target value: email for flows/scripts, permissioned_as (u/username) for triggers */ + targetValue: string | undefined selected: OnBehalfOfChoice - onSelect: (choice: OnBehalfOfChoice, email?: string, username?: string) => void + /** + * Called when the user picks a choice. + * For 'custom', `details` contains both `email` and `permissionedAs` (u/{username}). + * Callers pick whichever format they need. + */ + onSelect: (choice: OnBehalfOfChoice, details?: OnBehalfOfDetails) => void kind: string canPreserve: boolean - /** The email of the custom-selected user (for display) */ - customEmail?: string | undefined + /** The value of the custom-selected user (for display) */ + customValue?: string | undefined /** When false, labels say "current" instead of "target" and modal text refers to "this workspace" */ isDeployment?: boolean } let { targetWorkspace, - targetEmail, + targetValue, selected, onSelect, kind, canPreserve, - customEmail, + customValue, isDeployment = true }: Props = $props() + const isTrigger = $derived(kind === 'trigger') + let label = $derived( - kind === 'trigger' - ? 'Set the user this will be recorded as edited by:' + isTrigger + ? 'Set the user this will be permissioned as:' : 'Set the user this will be run on behalf of:' ) @@ -70,13 +87,17 @@ // Fetch users eagerly so we can resolve usernames for display loadUsers() - function resolveUsername(email: string | undefined): string | undefined { - if (!email) return undefined - return users.find((u) => u.email === email)?.username ?? email + /** Resolve a value to a display name, always showing u/username format */ + function resolveDisplayName(value: string | undefined): string | undefined { + if (!value) return undefined + if (value.startsWith('u/') || value.startsWith('g/')) return value + const username = users.find((u) => u.email === value)?.username + return username ? `u/${username}` : value } - let targetUsername = $derived(resolveUsername(targetEmail)) - let customUsername = $derived(resolveUsername(customEmail)) + let targetDisplayName = $derived(resolveDisplayName(targetValue)) + let customDisplayName = $derived(resolveDisplayName(customValue)) + let myDisplayName = $derived($userStore?.username ? `u/${$userStore.username}` : undefined) let activeUsers = $derived(users.filter((u) => !u.disabled)) let filteredUsers = $derived( @@ -94,7 +115,7 @@ // Preselect "target" when available and user has permission to preserve $effect(() => { - if (selected === undefined && targetEmail && canPreserve) { + if (selected === undefined && targetValue && canPreserve) { onSelect('target') } }) @@ -106,34 +127,32 @@ } function selectUser(user: User) { - onSelect('custom', user.email, user.username) + onSelect('custom', { email: user.email, permissionedAs: `u/${user.username}` }) modalOpen = false } let selectedDisplayName = $derived.by(() => { - if (selected === 'target') return targetUsername - if (selected === 'me') return $userStore?.username - if (selected === 'custom') return customUsername + if (selected === 'target') return targetDisplayName + if (selected === 'me') return myDisplayName + if (selected === 'custom') return customDisplayName return undefined }) e.detail && loadUsers()}> {#snippet trigger()} - - - - {#if selectedDisplayName} - {selectedDisplayName} - {/if} - - + + + {#if selectedDisplayName} + {selectedDisplayName} + {/if} + {/snippet} {#snippet content({ close: closePopover })} -
    +
    {label}
    - {#if targetEmail} + {#if targetValue} {/if} @@ -152,7 +171,7 @@ onclick={() => onSelect('me')} > - {$userStore?.username} + {myDisplayName} (me) @@ -166,9 +185,9 @@ openModal() }} > - {#if selected === 'custom' && customUsername} + {#if selected === 'custom' && customDisplayName} - {customUsername} + {customDisplayName} (custom) {:else} @@ -184,11 +203,15 @@
    - {#if kind === 'trigger'} - Choose the user this trigger will be recorded as edited by {isDeployment ? 'in the target workspace' : 'in this workspace'}. + {#if isTrigger} + Choose the user this trigger will be permissioned as {isDeployment + ? 'in the target workspace' + : 'in this workspace'}. The selected user's permissions will be used when the trigger + fires. {:else} - Choose the user this {kind} will run on behalf of {isDeployment ? 'in the target workspace' : 'in this workspace'}. The selected - user's permissions will be used when executing. + Choose the user this {kind} will run on behalf of {isDeployment + ? 'in the target workspace' + : 'in this workspace'}. The selected user's permissions will be used when executing. {/if} selectUser(user)} >
    - {user.username} + u/{user.username} {user.email}
    - {#if customEmail === user.email && selected === 'custom'} + {#if selected === 'custom' && (customValue === `u/${user.username}` || customValue === user.email)} {/if} diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 74b85bd327..9968c23180 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -592,10 +592,18 @@
    {/if} - {#if fileListLoading === false && displayedFileKeys.length === 0} -
    - No files in the workspace S3 bucket at that prefix -
    + {#if displayedFileKeys.length === 0} + {#if fileListLoading} +
    +
    + Loading content +
    +
    + {:else} +
    + No files in the workspace S3 bucket at that prefix +
    + {/if} {:else}
    shouldHideNoInputs?: boolean compact?: boolean - linkedSecret?: string | undefined + linkedSecrets?: string[] linkedSecretCandidates?: string[] | undefined noVariablePicker?: boolean flexWrap?: boolean @@ -86,7 +90,7 @@ defaultValues = {}, shouldHideNoInputs = false, compact = false, - linkedSecret = $bindable(undefined), + linkedSecrets = $bindable([]), linkedSecretCandidates = undefined, noVariablePicker = false, flexWrap = false, @@ -295,7 +299,9 @@ class={twMerge( typeof diff[argName] === 'object' && diff[argName].diff !== 'same' && - 'bg-red-300 dark:bg-red-800 rounded-md' + 'bg-red-300 dark:bg-red-800 rounded-md', + item[SHADOW_ITEM_MARKER_PROPERTY_NAME] && + '!visible border-2 border-dashed border-blue-300 dark:border-blue-600 bg-blue-50 dark:bg-blue-900/20 rounded-md [&>*]:invisible' )} innerClass="w-full" > @@ -333,7 +339,7 @@ {variableEditor} {itemPicker} {pickForField} - password={linkedSecret == argName} + password={linkedSecrets.includes(argName)} extra={formerProperty} {showSchemaExplorer} simpleTooltip={schemaFieldTooltip[argName]} @@ -398,22 +404,24 @@ customErrorMessage={prop?.customErrorMessage} bind:properties={ () => prop?.properties, - (v) => { if (prop) prop.properties = v } + (v) => { + if (prop) prop.properties = v + } } bind:order={ () => prop?.order, - (v) => { if (prop) prop.order = v } + (v) => { + if (prop) prop.order = v + } } nestedRequired={prop?.required} itemsType={prop?.items} - disabled={disabledArgs.includes(argName) || - disabled || - prop?.disabled} + disabled={disabledArgs.includes(argName) || disabled || prop?.disabled} {compact} {variableEditor} {itemPicker} bind:pickForField - password={linkedSecret == argName} + password={linkedSecrets.includes(argName)} extra={prop} {showSchemaExplorer} simpleTooltip={schemaFieldTooltip[argName]} @@ -440,12 +448,14 @@ {#if linkedSecretCandidates?.includes(argName)}
    { if (e.detail === 'secret') { - linkedSecret = argName - } else if (linkedSecret == argName) { - linkedSecret = undefined + if (!linkedSecrets.includes(argName)) { + linkedSecrets = [...linkedSecrets, argName] + } + } else { + linkedSecrets = linkedSecrets.filter((s) => s !== argName) } }} > diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 7de0926f0a..024fe0e907 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -107,6 +107,25 @@ import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' import { buildForkEditUrl } from '$lib/utils/editInFork' import OnBehalfOfSelector, { type OnBehalfOfChoice } from './OnBehalfOfSelector.svelte' + import WacExportDrawer from './scripts/WacExportDrawer.svelte' + import Modal from './common/modal/Modal.svelte' + + const WAC_ALPHA_ACK_KEY = 'windmill_wac_alpha_ack' + let wacAlphaModalOpen = $state(false) + + function showWacAlphaModalIfNeeded() { + if ( + typeof sessionStorage !== 'undefined' && + sessionStorage.getItem(WAC_ALPHA_ACK_KEY) !== 'true' + ) { + wacAlphaModalOpen = true + } + } + + function acknowledgeWacAlpha() { + sessionStorage.setItem(WAC_ALPHA_ACK_KEY, 'true') + wacAlphaModalOpen = false + } let { script = $bindable(), @@ -182,6 +201,7 @@ let editor: Editor | undefined = $state(undefined) let scriptEditor: ScriptEditor | undefined = $state(undefined) let captureTable: CaptureTable | undefined = $state(undefined) + let wacExportDrawer: WacExportDrawer | undefined = $state(undefined) // Draft triggers confirmation modal let draftTriggersModalOpen = $state(false) @@ -362,6 +382,23 @@ } if (script.content == '') { + if (template === 'wac_python') { + script.modules = { + 'helper.py': { + content: 'def main(a: str) -> str:\n return f"hello {a}"\n', + language: 'python3' + } + } + showWacAlphaModalIfNeeded() + } else if (template === 'wac_typescript') { + script.modules = { + 'helper.ts': { + content: 'export function main(a: string): string {\n return `hello ${a}`\n}\n', + language: 'bun' + } + } + showWacAlphaModalIfNeeded() + } initContent(script.language, script.kind, template) } @@ -388,7 +425,16 @@ async function initContent( language: SupportedLanguage, kind: Script['kind'] | undefined, - template: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' + template: + | 'pgsql' + | 'mysql' + | 'script' + | 'docker' + | 'powershell' + | 'bunnative' + | 'claudesandbox' + | 'wac_python' + | 'wac_typescript' ) { scriptEditor?.disableCollaboration() const templateScript = await isTemplateScript() @@ -403,6 +449,7 @@ } async function handleEditScript(stay: boolean, deployMsg?: string): Promise { + scriptEditor?.flushModuleState() // Fetch latest version and fetch entire script after if needed let actual_parent_hash: string | undefined = undefined @@ -510,10 +557,10 @@ script.kind === 'preprocessor' ? 'preprocessor' : undefined ) if (script.kind === 'preprocessor') { - script.no_main_func = undefined + script.auto_kind = undefined script.has_preprocessor = undefined } else { - script.no_main_func = result?.no_main_func || undefined + script.auto_kind = result?.auto_kind || undefined script.has_preprocessor = result?.has_preprocessor || undefined } } catch (error) { @@ -554,12 +601,13 @@ timeout: script.timeout, concurrency_key: emptyString(script.concurrency_key) ? undefined : script.concurrency_key, visible_to_runner_only: script.visible_to_runner_only, - no_main_func: script.no_main_func, + auto_kind: script.auto_kind, has_preprocessor: script.has_preprocessor, deployment_message: deploymentMsg || undefined, on_behalf_of_email: script.on_behalf_of_email, preserve_on_behalf_of: preserveOnBehalfOf || undefined, - assets: script.assets + assets: script.assets, + modules: script.modules } }) @@ -592,7 +640,12 @@ if (!disableHistoryChange) { history.replaceState(history.state, '', `/scripts/edit/${script.path}`) } - if (stay || (script.no_main_func && script.kind !== 'preprocessor' && !isWorkflowAsCode(script.content, script.language))) { + if ( + stay || + (script.auto_kind === 'lib' && + script.kind !== 'preprocessor' && + !isWorkflowAsCode(script.content, script.language)) + ) { script.parent_hash = newHash sendUserToast('Deployed') } else { @@ -606,6 +659,7 @@ } async function saveDraft(forceSave = false): Promise { + scriptEditor?.flushModuleState() if (initialPath != '' && !savedScript) { return } @@ -643,10 +697,10 @@ script.kind === 'preprocessor' ? 'preprocessor' : undefined ) if (script.kind === 'preprocessor') { - script.no_main_func = undefined + script.auto_kind = undefined script.has_preprocessor = undefined } else { - script.no_main_func = result?.no_main_func || undefined + script.auto_kind = result?.auto_kind || undefined script.has_preprocessor = result?.has_preprocessor || undefined } } catch (error) { @@ -707,10 +761,11 @@ ? undefined : script.concurrency_key, visible_to_runner_only: script.visible_to_runner_only, - no_main_func: script.no_main_func, + auto_kind: script.auto_kind, has_preprocessor: script.has_preprocessor, on_behalf_of_email: script.on_behalf_of_email, - assets: script.assets + assets: script.assets, + modules: script.modules } }) } @@ -816,7 +871,7 @@ } ] : []), - ...(!script.draft_only && script.kind === 'script' && !script.no_main_func + ...(!script.draft_only && script.kind === 'script' && !script.auto_kind ? [ { label: 'Exit & See details', @@ -825,10 +880,31 @@ } } ] + : []), + ...(isWorkflowAsCode(script.content, script.language) + ? [ + { + label: 'Export as YAML/JSON', + onClick: () => { + wacExportDrawer?.open(script) + } + } + ] : []) ] : [] + if (dropdownItems.length === 0 && isWorkflowAsCode(script.content, script.language)) { + dropdownItems = [ + { + label: 'Export as YAML/JSON', + onClick: () => { + wacExportDrawer?.open(script) + } + } + ] + } + return dropdownItems.length > 0 ? dropdownItems : undefined } @@ -1165,7 +1241,10 @@
    {#each langs as [label, lang] (lang)} {@const isPicked = - (lang == script.language && template != 'bunnative' && template != 'docker' && template != 'claudesandbox') || + (lang == script.language && + template != 'bunnative' && + template != 'docker' && + template != 'claudesandbox') || (template == 'bunnative' && lang == 'bunnative') || (template == 'docker' && lang == 'docker') || (template == 'claudesandbox' && lang == 'bun')} @@ -1201,7 +1280,7 @@
    {/if} -
    +
    Template + + + + + +
    {#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
    @@ -1653,9 +1783,9 @@ {#if script.on_behalf_of_email && canPreserve} → { + onSelect={(choice, details) => { onBehalfOfChoice = choice if (choice === 'me') { script.on_behalf_of_email = $userStore?.email @@ -1665,15 +1795,15 @@ script.on_behalf_of_email = originalOnBehalfOfEmail customOnBehalfOfEmail = '' preserveOnBehalfOf = true - } else if (choice === 'custom' && email) { - script.on_behalf_of_email = email - customOnBehalfOfEmail = email + } else if (choice === 'custom' && details) { + script.on_behalf_of_email = details.email + customOnBehalfOfEmail = details.email preserveOnBehalfOf = true } }} kind="script" {canPreserve} - customEmail={customOnBehalfOfEmail} + customValue={customOnBehalfOfEmail} isDeployment={false} /> {:else if script.on_behalf_of_email && !canPreserve} @@ -1948,9 +2078,35 @@ bind:hasPreprocessor bind:captureTable bind:assets={script.assets} + bind:modules={script.modules} enablePreprocessorSnippet />
    {:else} Script Builder not available to operators {/if} + + + + +
    +

    + Workflow-as-Code is in alpha — use in production at your own risk. It is an + alternative to the Flow editor for advanced users. Feedback welcome on + GitHub + or + Discord. +

    +
    + +
    +
    + diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index f310010c18..6a4fd088e0 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -1,8 +1,14 @@ {#if flow_status}
    -
    -
    {min ? displayDate(new Date(min), true) : ''}
    {#if max && min} - {/if}
    {max ? displayDate(new Date(max), true) : ''}{#if !max && min}{#if now} +
    +
    +
    +
    {min ? displayDate(new Date(min), true) : ''}
    + {#if max && min} + + {/if} +
    + {max ? displayDate(new Date(max), true) : ''} + {#if !max && min} + {#if now} {msToSec(now - min, 3)}s - {/if}{/if}
    + {/if} + + {/if} +
    +
    @@ -61,61 +142,220 @@
    Waiting for executor
    -
    Execution
    - {#each Object.entries(flow_status) as [k, v] (k)} -
    -
    - {v.name ?? k} { + const ta = new Date(a.started_at ?? a.scheduled_for ?? 0).getTime() + const tb = new Date(b.started_at ?? b.scheduled_for ?? 0).getTime() + return ta - tb + }) as [k, v] (k)} + {@const isInlineStep = isStep(k)} + {@const isSleep = (v as any).sleep_duration_s != undefined} + {@const isApproval = (v as any).approval === true} + {@const isRunning = !flowDone && v.duration_ms == undefined && v.started_at != undefined} + {@const isDone = v.duration_ms != undefined || flowDone} + {@const isExpanded = expandedRows[k] ?? false} +
    + {#if isSleep} +
    +
    + + sleep ({(v as any).sleep_duration_s}s) +
    + {:else if isApproval} +
    +
    + + + {v.name ?? stepKey(k)} + + {#if !isDone} + + + waiting + + {:else} + {msToSec(v.duration_ms ?? 0)}s + {/if} +
    + {:else} + + + {#if isExpanded} +
    + {#if isInlineStep} + + {@const result = stepResults[stepKey(k)]} + {#if isDone && result !== undefined} +
    +
    Result
    +
    + +
    +
    + {:else} +
    Step completed (no result)
    + {/if} + {:else if loadingJobs[k] && !childJobs[k]} +
    + + Loading... +
    + {:else if childJobs[k]} + {@const job = childJobs[k]} + + {#if job.logs || isRunning} +
    +
    Logs
    + +
    + {/if} + + + {#if isDone && job.result !== undefined} +
    +
    Result
    +
    + +
    +
    + {/if} + {:else} +
    No data available
    + {/if} +
    + {/if} + {/if}
    {/each} + {#if flowDone && result !== undefined} +
    + + {#if resultExpanded} +
    +
    + +
    +
    + {/if} +
    + {/if}
    {:else} diff --git a/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte b/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte index ec0b342f56..7bc0351c82 100644 --- a/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte +++ b/frontend/src/lib/components/WorkspaceDependenciesEditor.svelte @@ -40,6 +40,19 @@ return deps.name || `Default (${deps.language})` } + function getEditorLang(language: ScriptLang): string { + switch (language) { + case 'bun': + case 'php': + case 'powershell': + return 'json' + case 'python3': + return 'plaintext' + default: + return 'markdown' + } + } + export function getFileExtension(language: ScriptLang): string | null { switch (language) { case 'python3': @@ -50,6 +63,8 @@ // return 'go.mod' case 'php': return 'composer.json' + case 'powershell': + return 'modules.json' default: return null } @@ -113,7 +128,8 @@ { value: 'python3', label: 'Python' }, { value: 'bun', label: 'TypeScript (Bun/Bunnative)' }, // { value: 'go', label: 'Go' }, - { value: 'php', label: 'PHP' } + { value: 'php', label: 'PHP' }, + { value: 'powershell', label: 'PowerShell' } ] // Default templates for each language @@ -157,6 +173,13 @@ numpy>=1.24.0 "vlucas/phpdotenv": "^5.6", "symfony/console": "^6.4" } +}`, + + powershell: `{ + "modules": { + "PSWriteColor": "*", + "ImportExcel": "7.8.6" + } }` } @@ -497,7 +520,7 @@ numpy>=1.24.0 handleEditorChange(e.detail)} fixedOverflowWidgets={false} @@ -505,6 +528,15 @@ numpy>=1.24.0 /> {/await}
    + {#if workspaceDependencies.language === 'powershell'} +
    + JSON object with a "modules" key mapping module names to versions. Use + "*" + or null for latest version, or a specific version string to pin. These + modules are merged with script-level + Import-Module statements at runtime (workspace versions take precedence). +
    + {/if}
    diff --git a/frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte b/frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte index 5d8778b01e..0e30c6b421 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte +++ b/frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte @@ -62,8 +62,8 @@ onCancel: () => { sendUserToast('Error deleting row', true) }, - onError: () => { - sendUserToast('Error updating row', true) + onError: (e) => { + sendUserToast(`Error deleting row: ${e?.message ?? e}`, true) } } ) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte b/frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte index 6b11362f3b..fea5232c6b 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte +++ b/frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte @@ -59,8 +59,8 @@ onCancel: () => { sendUserToast('Error inserting row', true) }, - onError: () => { - sendUserToast('Error inserting row', true) + onError: (e) => { + sendUserToast(`Error inserting row: ${e?.message ?? e}`, true) } }) } diff --git a/frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte b/frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte index e90ea9d96b..c324d7e408 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte +++ b/frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte @@ -64,8 +64,8 @@ onCancel: () => { sendUserToast('Error updating value', true) }, - onError: () => { - sendUserToast('Error updating value', true) + onError: (e) => { + sendUserToast(`Error updating value: ${e?.message ?? e}`, true) } } ) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts index 4d970b2303..2fdb61522d 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts @@ -1,4 +1,4 @@ -import { JobService, ResourceService, type ScriptLang } from '$lib/gen' +import { JobService, ResourceService } from '$lib/gen' import { runScriptAndPollResult } from '$lib/components/jobs/utils' import type { DbInput } from '$lib/components/dbTypes' @@ -15,9 +15,18 @@ import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores' import { stringifyGraphqlSchema, stringifySchema } from '$lib/components/copilot/lib' import type { DbType } from '$lib/components/dbTypes' -import { getDatabaseArg } from '$lib/components/dbOps' +import { getDatabaseArg, getDbType } from '$lib/components/dbOps' import { sendUserToast } from '$lib/toast' +function makeMetadataMarker( + op: string, + payload: Record, + ducklake: string | undefined +): string { + if (ducklake) payload.ducklake = ducklake + return `-- WM_INTERNAL_DB_${op} ${JSON.stringify(payload)}` +} + export async function loadTableMetaData( input: DbInput, workspace: string | undefined, @@ -25,11 +34,26 @@ export async function loadTableMetaData( ): Promise { if (!input || !table || !workspace) return undefined - let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, table) + const dbType = getDbType(input) + const language = getLanguageByResourceType(dbType) + const ducklake = input.type === 'ducklake' ? input.ducklake : undefined + const dbArg = getDatabaseArg(input) + + // MySQL needs the database name for metadata queries + let databaseName: string | undefined + if (input.type === 'database' && input.resourceType === 'mysql') { + const resourceObj = (await ResourceService.getResourceValue({ + workspace, + path: input.resourcePath + })) as any + databaseName = resourceObj?.database + } + + const content = makeMetadataMarker('LOAD_TABLE_METADATA', { table, databaseName }, ducklake) const job = await JobService.runScriptPreview({ - workspace: workspace, - requestBody: { language, content: query, args: getDatabaseArg(input) } + workspace, + requestBody: { language, content, args: dbArg } }) const maxRetries = 8 @@ -39,7 +63,7 @@ export async function loadTableMetaData( await new Promise((resolve) => setTimeout(resolve, 1000 * (attempts || 0.6))) const testResult = (await JobService.getCompletedJob({ - workspace: workspace, + workspace, id: job })) as any @@ -78,10 +102,30 @@ export async function loadAllTablesMetaData( if (!input || !workspace) return undefined try { - let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, undefined) + const dbType = getDbType(input) + const dbArg = getDatabaseArg(input) + const ducklake = input.type === 'ducklake' ? input.ducklake : undefined + + // MySQL needs the database name for metadata queries + let databaseName: string | undefined + if (input.type === 'database' && input.resourceType === 'mysql') { + const resourceObj = (await ResourceService.getResourceValue({ + workspace, + path: input.resourcePath + })) as any + databaseName = resourceObj?.database + } + + const language = getLanguageByResourceType(dbType) + const content = makeMetadataMarker( + 'LOAD_TABLE_METADATA', + { table: undefined, databaseName }, + ducklake + ) + let result = (await runScriptAndPollResult({ - workspace: workspace, - requestBody: { language, content: query, args: getDatabaseArg(input) } + workspace, + requestBody: { language, content, args: dbArg } })) as ({ table_name: string; schema_name?: string } & object)[] const map: Record = {} @@ -101,241 +145,6 @@ export async function loadAllTablesMetaData( } } -async function makeLoadTableMetaDataQuery( - input: DbInput, - workspace: string, - table: string | undefined -): Promise<{ query: string; language: ScriptLang }> { - if (input.type === 'ducklake') { - const query = `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__; - SELECT - COLUMN_NAME as field, - DATA_TYPE as DataType, - COLUMN_DEFAULT as DefaultValue, - false as IsPrimaryKey, - false as IsIdentity, - IS_NULLABLE as IsNullable, - false as IsEnum, - TABLE_NAME as table_name - FROM information_schema.columns c - WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()` - return { query, language: 'duckdb' } - } else if (input.resourceType === 'mysql') { - const resourceObj = (await ResourceService.getResourceValue({ - workspace, - path: input.resourcePath - })) as any - const query = ` - SELECT - COLUMN_NAME as field, - COLUMN_TYPE as DataType, - COLUMN_DEFAULT as DefaultValue, - CASE WHEN COLUMN_KEY = 'PRI' THEN 1 ELSE 0 END as IsPrimaryKey, - CASE WHEN EXTRA like '%auto_increment%' THEN 'YES' ELSE 'NO' END as IsIdentity, - CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable, - CASE WHEN DATA_TYPE = 'enum' THEN true ELSE false END as IsEnum${ - table - ? '' - : `, - TABLE_NAME as table_name` - } - FROM - INFORMATION_SCHEMA.COLUMNS${ - table - ? ` - WHERE - TABLE_NAME = '${table.split('.').reverse()[0]}' AND TABLE_SCHEMA = '${ - table.split('.').reverse()[1] ?? resourceObj?.database ?? '' - }'` - : ` - WHERE - TABLE_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys')` - } - ORDER BY - TABLE_NAME, - ORDINAL_POSITION; - ` - return { query, language: 'mysql' } - } else if (input.resourceType === 'postgresql') { - const query = ` - SELECT - a.attname as field, - pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType, - (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128) - FROM pg_catalog.pg_attrdef d - WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as DefaultValue, - (SELECT CASE WHEN i.indisprimary THEN true ELSE 'NO' END - FROM pg_catalog.pg_class tbl, pg_catalog.pg_class idx, pg_catalog.pg_index i, pg_catalog.pg_attribute att - WHERE tbl.oid = a.attrelid AND idx.oid = i.indexrelid AND att.attrelid = tbl.oid - AND i.indrelid = tbl.oid AND att.attnum = any(i.indkey) AND att.attname = a.attname LIMIT 1) as IsPrimaryKey, - CASE a.attidentity - WHEN 'd' THEN 'By Default' - WHEN 'a' THEN 'Always' - ELSE 'No' - END as IsIdentity, - CASE a.attnotnull - WHEN false THEN 'YES' - ELSE 'NO' - END as IsNullable, - (SELECT true - FROM pg_catalog.pg_enum e - WHERE e.enumtypid = a.atttypid FETCH FIRST ROW ONLY) as IsEnum${ - table - ? '' - : `, - ns.nspname AS schema_name, - c.relname AS table_name` - } - FROM pg_catalog.pg_attribute a${ - table - ? ` - WHERE a.attrelid = (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid WHERE relname = '${ - table.split('.').reverse()[0] - }' AND ns.nspname = '${table.split('.').reverse()[1] ?? 'public'}') - AND a.attnum > 0 AND NOT a.attisdropped - ` - : ` - JOIN pg_catalog.pg_class c ON a.attrelid = c.oid - JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid - WHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped - AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'` - } - ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'}; - - ` - return { query, language: 'postgresql' } - } else if (input.resourceType === 'ms_sql_server') { - const query = ` - SELECT - c.COLUMN_NAME as field, - c.DATA_TYPE as DataType, - c.COLUMN_DEFAULT as DefaultValue, - CASE WHEN COLUMNPROPERTY(OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME), c.COLUMN_NAME, 'IsIdentity') = 1 THEN 'By Default' ELSE 'No' END as IsIdentity, - CASE WHEN pk.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END as IsPrimaryKey, - CASE WHEN c.IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable, - CASE WHEN c.DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum, - dc.name as default_constraint_name${ - table - ? '' - : `, - c.TABLE_NAME as table_name` - } -FROM - INFORMATION_SCHEMA.COLUMNS c - LEFT JOIN ( - SELECT - ku.TABLE_SCHEMA, - ku.TABLE_NAME, - ku.COLUMN_NAME - FROM - INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc - INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE ku - ON tc.CONSTRAINT_TYPE = 'PRIMARY KEY' - AND tc.CONSTRAINT_NAME = ku.CONSTRAINT_NAME - AND tc.TABLE_SCHEMA = ku.TABLE_SCHEMA - AND tc.TABLE_NAME = ku.TABLE_NAME - ) pk ON c.TABLE_SCHEMA = pk.TABLE_SCHEMA - AND c.TABLE_NAME = pk.TABLE_NAME - AND c.COLUMN_NAME = pk.COLUMN_NAME - LEFT JOIN sys.default_constraints dc - ON dc.parent_object_id = OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME) - AND dc.parent_column_id = COLUMNPROPERTY(OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME), c.COLUMN_NAME, 'ColumnId')${ - table - ? ` -WHERE - c.TABLE_NAME = '${table}'` - : '' - } -ORDER BY - c.ORDINAL_POSITION; - ` - return { query, language: 'mssql' } - } else if ( - input.resourceType === 'snowflake' || - (input.resourceType as any) === 'snowflake_oauth' - ) { - const query = ` - select COLUMN_NAME as field, - DATA_TYPE as DataType, - COLUMN_DEFAULT as DefaultValue, - CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 'By Default' ELSE 'No' END as IsIdentity, - 0 as IsPrimaryKey, -- a one-query solution is not trivial, we will use SHOW PRIMARY KEYS separately - CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable, - CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${ - table - ? '' - : `, - table_name as table_name, - table_schema as schema_name` - } - from information_schema.columns${ - table - ? ` - where table_name = '${table.split('.').reverse()[0]}' and table_schema = '${ - table.split('.').reverse()[1] ?? 'PUBLIC' - }'` - : "\nwhere table_schema <> 'INFORMATION_SCHEMA'\n" - } - order by ORDINAL_POSITION; - ` - return { query, language: 'snowflake' } - } else if (input.resourceType === 'bigquery') { - if (table) { - const query = `SELECT - c.COLUMN_NAME as field, - DATA_TYPE as DataType, - CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue, - CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey, - 'No' as IsIdentity, - IS_NULLABLE as IsNullable, - false as IsEnum -FROM - ${table.split('.')[0]}.INFORMATION_SCHEMA.COLUMNS c - LEFT JOIN - ${table.split('.')[0]}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p - on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME -WHERE - c.TABLE_NAME = '${table.split('.')[1]}' -order by c.ORDINAL_POSITION;` - return { query, language: 'bigquery' } - } else { - const query = `import { BigQuery } from '@google-cloud/bigquery@7.5.0'; -export async function main(database: bigquery) { -const bq = new BigQuery({ - credentials: database -}) -const [datasets] = await bq.getDatasets(); -if (!datasets) return {} -const schema = {} as any -let queries = datasets.map(dataset => \` - (SELECT - c.COLUMN_NAME as field, - '\${dataset.id}' as schema_name, - c.TABLE_NAME as table_name, - DATA_TYPE as DataType, - CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue, - CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey, - 'No' as IsIdentity, - IS_NULLABLE as IsNullable, - false as IsEnum -FROM - \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.COLUMNS c - LEFT JOIN - \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p - on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME -ORDER BY c.ORDINAL_POSITION)\` -) -let query = queries.join('\\nUNION ALL \\n') -const [rows] = await bq.query(query) -return rows -}` - return { query, language: 'bun' } - } - } else { - throw new Error('Unsupported database type:' + input.resourceType) - } -} - type SnowflakeShowPrimaryKeysResult = { column_name: string database_name: string @@ -379,12 +188,15 @@ async function fetchSnowflakePrimaryKeys( dbArg: any, tableKey?: string ): Promise { + const payload: Record = {} + if (tableKey) payload.table = tableKey + const content = makeMetadataMarker('SNOWFLAKE_PRIMARY_KEYS', payload, undefined) return (await JobService.runScriptPreviewAndWaitResult({ workspace, requestBody: { language: 'snowflake', args: dbArg, - content: tableKey ? `SHOW PRIMARY KEYS IN TABLE ${tableKey}` : 'SHOW PRIMARY KEYS IN ACCOUNT' + content } })) as SnowflakeShowPrimaryKeysResult[] } diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts index 3a7bf66a62..dc422f48e6 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts @@ -1,3 +1,12 @@ +/** + * LEGACY: These query builders generate full SQL on the frontend. + * They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps + * whose policies were generated with expanded SQL digests. + * + * New code (Database Manager) should use WM_INTERNAL_DB markers instead, + * which are expanded server-side by the Rust query_builders module. + * See: dbOps.ts → dbTableOpsWithPreviewScripts() + */ import type { AppInput, RunnableByName } from '$lib/components/apps/inputType' import { wrapDucklakeQuery } from '../../../../../ducklake' import type { DbType, DbInput } from '$lib/components/dbTypes' diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts index a6ded0f727..3c6419634b 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts @@ -1,3 +1,12 @@ +/** + * LEGACY: These query builders generate full SQL on the frontend. + * They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps + * whose policies were generated with expanded SQL digests. + * + * New code (Database Manager) should use WM_INTERNAL_DB markers instead, + * which are expanded server-side by the Rust query_builders module. + * See: dbOps.ts → dbTableOpsWithPreviewScripts() + */ import type { AppInput, RunnableByName } from '$lib/components/apps/inputType' import type { DbType, DbInput } from '$lib/components/dbTypes' import { wrapDucklakeQuery } from '../../../../../ducklake' diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts index 9983b7c8e8..7a79a3cbad 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts @@ -1,3 +1,12 @@ +/** + * LEGACY: These query builders generate full SQL on the frontend. + * They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps + * whose policies were generated with expanded SQL digests. + * + * New code (Database Manager) should use WM_INTERNAL_DB markers instead, + * which are expanded server-side by the Rust query_builders module. + * See: dbOps.ts → dbTableOpsWithPreviewScripts() + */ import type { AppInput } from '$lib/components/apps/inputType' import { wrapDucklakeQuery } from '../../../../../ducklake' import type { DbType, DbInput } from '$lib/components/dbTypes' diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/relationalKeys.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/relationalKeys.ts index 1bda320bd6..ad42728d44 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/relationalKeys.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/relationalKeys.ts @@ -275,7 +275,7 @@ function makeSnowflakeForeignKeysQuery(tableName: string, schemaName: string): s * pk_database_name, pk_schema_name, pk_table_name, pk_column_name, key_sequence, * update_rule, delete_rule, fk_name, pk_name, deferrability */ -function transformSnowflakeForeignKeys(snowflakeResults: any[]): RawForeignKey[] { +export function transformSnowflakeForeignKeys(snowflakeResults: any[]): RawForeignKey[] { if (!snowflakeResults || !Array.isArray(snowflakeResults)) { return [] } diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts index 47b548c9d6..737dfdc329 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts @@ -1,3 +1,12 @@ +/** + * LEGACY: These query builders generate full SQL on the frontend. + * They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps + * whose policies were generated with expanded SQL digests. + * + * New code (Database Manager) should use WM_INTERNAL_DB markers instead, + * which are expanded server-side by the Rust query_builders module. + * See: dbOps.ts → dbTableOpsWithPreviewScripts() + */ import type { AppInput, RunnableByName } from '$lib/components/apps/inputType' import { wrapDucklakeQuery } from '../../../../../ducklake' import type { DbType, DbInput } from '$lib/components/dbTypes' diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts index 53aad833d4..b57fa4989b 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts @@ -1,3 +1,12 @@ +/** + * LEGACY: These query builders generate full SQL on the frontend. + * They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps + * whose policies were generated with expanded SQL digests. + * + * New code (Database Manager) should use WM_INTERNAL_DB markers instead, + * which are expanded server-side by the Rust query_builders module. + * See: dbOps.ts → dbTableOpsWithPreviewScripts() + */ import type { AppInput, RunnableByName } from '$lib/components/apps/inputType' import { wrapDucklakeQuery } from '../../../../../ducklake' import type { DbInput, DbType } from '$lib/components/dbTypes' diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index 5d5484a544..649e67ef0d 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -54,10 +54,9 @@ } = $props() let isDeployer = $derived($userStore?.groups?.includes(WM_DEPLOYERS_GROUP) ?? false) - let canPreserve = $derived( - !!$userStore?.is_admin || !!$userStore?.is_super_admin || isDeployer - ) + let canPreserve = $derived(!!$userStore?.is_admin || !!$userStore?.is_super_admin || isDeployer) let savedOnBehalfOfEmail = $derived(savedApp?.policy?.on_behalf_of_email) + let savedOnBehalfOf = $derived(savedApp?.policy?.on_behalf_of) let onBehalfOfChoice: OnBehalfOfChoice = $state(undefined) let customOnBehalfOfEmail: string = $state('') let dirtyCustomPath = $state(false) @@ -197,12 +196,13 @@ {#if canPreserve}
    - Because you are either an admin or part of the {WM_DEPLOYERS_GROUP} group, you can select another user to run this app on behalf of. Once deployed the app will be run on behalf of + Because you are either an admin or part of the {WM_DEPLOYERS_GROUP} group, you can select another + user to run this app on behalf of. Once deployed the app will be run on behalf of { + onSelect={(choice, details) => { onBehalfOfChoice = choice if (choice === 'me') { policy.on_behalf_of_email = $userStore?.email @@ -211,25 +211,25 @@ preserveOnBehalfOf = false } else if (choice === 'target') { policy.on_behalf_of_email = savedOnBehalfOfEmail + policy.on_behalf_of = savedOnBehalfOf customOnBehalfOfEmail = '' preserveOnBehalfOf = true - } else if (choice === 'custom' && email) { - policy.on_behalf_of_email = email - policy.on_behalf_of = username ? `u/${username}` : undefined - customOnBehalfOfEmail = email + } else if (choice === 'custom' && details) { + policy.on_behalf_of_email = details.email + policy.on_behalf_of = details.permissionedAs + customOnBehalfOfEmail = details.email preserveOnBehalfOf = true } }} kind="app" {canPreserve} - customEmail={customOnBehalfOfEmail} + customValue={customOnBehalfOfEmail} isDeployment={false} />
    {/if} -
    {#if !hideSecretUrl} diff --git a/frontend/src/lib/components/apps/editor/appUtilsS3.ts b/frontend/src/lib/components/apps/editor/appUtilsS3.ts index 5e35f1a4bd..f9811edd76 100644 --- a/frontend/src/lib/components/apps/editor/appUtilsS3.ts +++ b/frontend/src/lib/components/apps/editor/appUtilsS3.ts @@ -153,8 +153,8 @@ export function computeS3FileViewerPolicy(config: RichConfigurations) { } else if ( config.source.type === 'static' && typeof config.source.value === 'string' && - ((config.sourceKind.type === 'static' && - config.sourceKind.value === 's3 (workspace storage)') || + ((config.sourceKind?.type === 'static' && + config.sourceKind?.value === 's3 (workspace storage)') || config.source.value.startsWith('s3://')) ) { return { diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanel.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanel.svelte index 9acb196b46..0e77fd8b8e 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanel.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptsPanel.svelte @@ -1,6 +1,6 @@ @@ -58,7 +54,7 @@ > {#if type === 'textarea'}