Compare commits

...

63 Commits

Author SHA1 Message Date
Guilhem
fa22ec69d8 fix: only show email validation error after dropdown closes
Add onClose callback to AutocompleteSelect and use it in AddUser
to defer validation until the user finishes interacting with the
dropdown, avoiding visual noise between the dropdown and error.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-17 09:33:22 +00:00
Guilhem
da460fa6b4 nit 2026-02-17 09:27:57 +00:00
Guilhem
f03bd04cac fix: use AutocompleteSelect loading prop instead of separate branch
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-17 08:59:01 +00:00
Guilhem
e11a333998 fix: address PR review comments
- Add LIMIT 1000 to list_instance_emails SQL query
- Add loading state to prevent UI flash in AddUser
- Show email validation error only after blur
- Guard loadInstanceEmails with workspaceStore check

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-17 08:20:14 +00:00
Guilhem
2ff72e34dd Merge remote-tracking branch 'origin/main' into glm/select-users 2026-02-16 18:38:22 +00:00
Ruben Fiszel
0940d70a2b chore(main): release 1.636.0 (#7962)
* chore(main): release 1.636.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2026-02-16 17:09:46 +00:00
centdix
e7b0b00f56 fix: mark base_url as unsaved when using browser fallback (#7964)
* fix: mark base_url as unsaved when using browser fallback in instance settings

When base_url is not set in the database, the frontend silently fills in
window.location.origin but also snapshots it as the initial value. This
makes the dirty-check see no change, so the Save button stays disabled
and the user cannot persist the auto-detected value.

Fix by snapshotting initialValues before applying the fallback, and show
a yellow warning indicating the value is auto-detected and unsaved.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: render base_url fallback warning inside the setting card

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 17:01:18 +00:00
Ruben Fiszel
4587cd5f3e fix sqlx 2026-02-16 17:00:35 +00:00
Ruben Fiszel
7a5719217b fix sqlx 2026-02-16 16:49:42 +00:00
Ruben Fiszel
ec17b29ae2 fix private check issue 2026-02-16 16:39:59 +00:00
Guilhem
732aff1d5c Merge branch 'main' into claude/issue-7902-20260211-1113 2026-02-16 16:32:49 +00:00
Ruben Fiszel
713c39040b fix private check issue 2026-02-16 16:12:02 +00:00
centdix
5db6a405ad fix review comments (#7963) 2026-02-16 15:53:52 +00:00
hugocasa
6f24f1939d feat: google native triggers (#7837)
* feat: google native triggers

* nit skill

* better native trigger abstraction

* use resources for workspace integrations

* better and better

* better tests

* update native trigger skill

* sqlx

* less tx and google update fix

* refactor a bit the external logic

* nits

* fix

* fix google native trigger update

* fix oauth

* review fixes

* sqlx fix

* nit

* chore: update ee-repo-ref to a10eda4251610cceee67fbe05463b8be82ffa9e0

This commit updates the EE repository reference after PR #416 was merged in windmill-ee-private.

Previous ee-repo-ref: bf3696d5f2a39a3cb84dbbee81e092155f2a8c75

New ee-repo-ref: a10eda4251610cceee67fbe05463b8be82ffa9e0

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
2026-02-16 15:52:02 +00:00
Guilhem
13ff7f418e Merge branch 'main' into claude/issue-7902-20260211-1113 2026-02-16 15:50:33 +00:00
windmill-internal-app[bot]
d89d36728b chore: update ee-repo-ref to c927593b0b49867284fc64d50c59daba6c70da84
This commit updates the EE repository reference after PR #415 was merged in windmill-ee-private.

Previous ee-repo-ref: 8c214ec5039be5353f5fea920e27b0c6af61e1fd

New ee-repo-ref: c927593b0b49867284fc64d50c59daba6c70da84

Automated by sync-ee-ref workflow.
2026-02-16 15:48:08 +00:00
centdix
5b8ec502fe feat: allow adding workspace scripts and flows as AI chat context (#7882)
* feat: allow adding workspace scripts and flows as AI chat context

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* cleaning

* cleaning

* cleaning

* better

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-16 15:46:17 +00:00
Guilhem
47c027716b chore: remove test_dev select page
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 15:39:18 +00:00
Guilhem
9347a04ed5 style: format test_dev select page
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 15:38:41 +00:00
Guilhem
360b43caba nit 2026-02-16 15:38:22 +00:00
Guilhem
1dd2b0a1b2 refactor: extract AutocompleteSelect from Select component
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 15:31:22 +00:00
Guilhem
19f30ee6d5 Merge remote-tracking branch 'origin/main' into glm/select-users 2026-02-16 15:01:41 +00:00
Ruben Fiszel
b2128a0109 add dynamic tls support for email 2026-02-16 14:57:28 +00:00
Guilhem
90139c7754 Merge remote-tracking branch 'origin/main' into glm/select-users 2026-02-16 14:51:51 +00:00
Guilhem
4aa1f01210 refactor: replace Select placeholder hack with autocomplete model
Replace the bind:value getter/setter and placeholder-as-display hack with
a standard controlled input pattern (value={inputValue} + oninput handler).

- Remove showPlaceholderOnOpen prop (replaced by default behavior)
- Add allowUserInput prop: typed text syncs to value in real-time
- Separate rawLabel (for filtering) from displayText (for transformed display)
- Hide dropdown when no items match in allowUserInput mode
- Simplify AddUser to use allowUserInput instead of createText/onCreateItem

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 14:51:06 +00:00
Guilhem
bfb65fc8b4 fix: allow editing custom email in Select component
When a custom value (not in the items list) is selected, reopening
the dropdown now pre-fills the filter with the current value so users
can directly edit it. Also hide the "Use custom email" create option
when the filter matches the current value, and show the regular
placeholder instead of the value when editing a custom entry.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 11:52:54 +00:00
Guilhem
b54bea7192 feat: add email validation and use TextInput in AddUser
Add client-side email format validation with error message display
and disable the Add button when email is invalid. Replace plain
<input> elements with TextInput component for consistent styling.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 11:32:31 +00:00
Guilhem
793bb7b405 fix: restrict list_instance_emails to workspace admins
Move the endpoint from global to workspace-scoped route
(/w/{workspace}/users/list_instance_emails) and add require_admin
check so only workspace admins can list instance emails.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 10:54:06 +00:00
Guilhem
fb0f0031d9 fix: filter out existing workspace members from instance emails select
Also fixes "Add new userss" typo.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-16 10:25:15 +00:00
Ruben Fiszel
ba80570357 chore(main): release 1.635.1 (#7958)
* chore(main): release 1.635.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2026-02-15 17:14:04 +00:00
Ruben Fiszel
714f713a04 only update license key via IaC when new expiry is posterior (#7959)
When using infrastructure-as-code to update settings, if the desired
license key matches the current one (same client ID and signature) but
differs only in the expiration date, only apply the update if the new
key has a later expiry. This prevents accidental downgrades when an
older license key is present in the IaC configuration.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 17:13:37 +00:00
Ruben Fiszel
68f766e1ae fix: pin strum_macros to 0.27 to match strum version (#7957)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 16:56:39 +00:00
Ruben Fiszel
d9a5cb64b8 chore(main): release 1.635.0 (#7945)
* chore(main): release 2.0.0

* Apply automatic changes

* chore(main): release 1.635.0

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 16:48:38 +00:00
Ruben Fiszel
2ca3c8e409 nit 2026-02-15 16:44:43 +00:00
Ruben Fiszel
0cc4e2650c make api for setting instance config more consistent 2026-02-15 16:38:07 +00:00
Ruben Fiszel
bdffba53ed expose 'Set via API' in worker group config drawer 2026-02-15 16:32:38 +00:00
Ruben Fiszel
64532a1d12 list to record for worker config 2026-02-15 16:21:16 +00:00
Ruben Fiszel
8b8e33e2dc feat: IaC hints, YAML editor for worker configs (#7956)
* feat: add CRD configuration hints to instance settings and workers pages

Add subtle text notes with doc links to help K8s users discover that
instance settings and worker group configs can be managed via the
WindmillInstance CRD.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: IaC hints, YAML editor for worker configs, and EE badges

- Move IaC hint from form view to YAML mode in Instance Settings
- Add YAML editor drawer for worker group configs with diff review
- Add EE-only badges to gated fields in worker config drawer
- Make init_bash available in CE (frontend + backend gate)
- Promote Init/Periodic scripts to top-level sections
- Add "YAML editor" button to worker config drawer action bar
- Strip cache_clear and empty fields from YAML output
- Sort default/native groups first in YAML editor

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: address PR review feedback on worker YAML editor

- Extract serializeWorkerGroupsAsYaml() shared helper (dedup)
- Fix name:undefined leak using proper destructuring
- Protect default/native groups from accidental deletion
- Add loading indicator during YAML save
- Add restart warning for periodic script changes
- Use consistent on:click event syntax in YAML drawer actions

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* refactor: use onClick prop instead of on:click for Button components

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 15:53:03 +00:00
Ruben Fiszel
c5d870f480 make K8s operator private and add registry EE warning (#7955)
Move K8s operator source code (crd, db_sync, reconciler, resolve) to
windmill-ee-private and gate behind feature = "private". OSS stubs
provide error messages when the feature is disabled.

Add an info Alert banner in the Registries settings section when no
enterprise license is active.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 14:32:15 +00:00
Ruben Fiszel
6cf3f5f4a3 feat: open-source worker group configuration UI (#7954)
* feat: gate private registry configuration behind enterprise feature

Add read_ee_registry<T> helper that returns None and emits a job log
in CE builds when a private registry is configured. Applied to all
language executors: pip, npm/bun, deno, cargo, go, maven, nuget,
powershell, and ruby. Maven settings.xml file write in monitor.rs
is also gated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: open-source worker group configuration UI

Allow CE users to create worker group configs and edit worker_tags.
EE-only features (dedicated workers, custom env vars, init/periodic
scripts, autoscaling, priority tags, etc.) remain gated behind the
enterprise license in the UI. Backend now filters CE config updates
to only allow worker_tags and cache_clear fields instead of rejecting
the entire request.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 14:05:23 +00:00
Ruben Fiszel
423e07376b feat: gate private registry configuration behind enterprise feature (#7953)
Add read_ee_registry<T> helper that returns None and emits a job log
in CE builds when a private registry is configured. Applied to all
language executors: pip, npm/bun, deno, cargo, go, maven, nuget,
powershell, and ruby. Maven settings.xml file write in monitor.rs
is also gated.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 14:02:14 +00:00
Ruben Fiszel
dbe576406e add type annotations to opaque CRD schema fields (#7952)
* fix: add type annotations to opaque CRD schema fields

The WindmillInstance CRD failed K8s structural schema validation because
four `Option<serde_json::Value>` fields (`object_store_cache_config`,
`secret_backend`, `slack`, `teams`) generated schemas without a `type`
property. Kubernetes requires all specified object fields to declare a
type.

Add a `schemars(schema_with)` annotation that emits
`{"type": "object", "nullable": true, "x-kubernetes-preserve-unknown-fields": true}`
for these opaque settings fields.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: enable operator in CE builds

The operator feature was only included in `ee_core`, meaning CE images
couldn't run `windmill operator` as a proper k8s controller — the
subcommand silently fell through to standalone mode.

Add `operator` to `ce_core` so CE builds include the k8s controller
that watches and reconciles WindmillInstance CRDs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 12:26:21 +00:00
Ruben Fiszel
e4a34d031b feat: handle $var: and $res: in arrays for transform_json_value (#7949)
* feat: handle $var: and $res: in arrays for transform_json_value

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: resolve $var/$res arrays inside objects in transform_json_value

Replace top_level: bool with depth: u8 to fix the main bug where arrays
nested inside objects were never resolved. The Object arm recursed with
top_level: false, which caused the Array arm guard to reject the value.

With depth <= 2, arrays inside objects (depth 1-2) are now processed.
Also: transform all array elements unconditionally (handles $encrypted:
in worker, reserved $vars, nested structures), and warn on >1000 items.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 08:35:06 +00:00
Ruben Fiszel
43218c6285 fix: strip slack_oauth_client_secret from get_settings for non-admins (#7950)
The GET /api/w/{workspace}/workspaces/get_settings endpoint returned
slack_oauth_client_secret in plaintext to any authenticated workspace
member. Non-admin users now receive null for this field. The dedicated
get_slack_oauth_config endpoint (admin-only, masked) is unaffected.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-15 08:22:37 +00:00
Zhiyuan Zheng
8410b59a8f fix(go): preserve proxy envs for go mod tidy/download (#7946) 2026-02-15 07:46:15 +00:00
Ruben Fiszel
985d7fd3d6 iac nits (#7948) 2026-02-15 07:45:53 +00:00
Ruben Fiszel
82e5f6de48 feat: add Kubernetes operator and instance settings YAML editor (#7836)
* Add windmill-operator crate for Kubernetes CRD-based instance config

Introduces a new `windmill-operator` crate that enables declarative
management of Windmill instance configuration via a Kubernetes
`WindmillInstance` CRD. The operator watches CRD resources and performs
full declarative sync of global_settings and worker configs to the
database, supporting GitOps workflows for instance-level configuration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Add tests for windmill-operator CRD and db_sync

- 9 unit tests for CRD serialization, deserialization, metadata, and
  status field behavior
- 15 integration tests for db_sync using #[sqlx::test] with full
  declarative sync coverage: upsert, delete, protected keys,
  idempotency, worker config prefix handling, and end-to-end sync

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Replace untyped BTreeMap CRD fields with typed structs for schema validation

GlobalSettings, SmtpSettings, IndexerSettings, and WorkerGroupConfig now
have explicit typed fields with serde(flatten) catch-all for forward
compatibility. The generated CRD YAML includes a full OpenAPI v3 schema
that Kubernetes validates on kubectl apply.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Type opaque serde_json::Value CRD fields with real structs

Replace most remaining serde_json::Value fields in WindmillInstance CRD
with properly typed structs derived from the codebase:
- oauths: BTreeMap<String, OAuthClient>
- otel: OtelSettings
- otel_tracing_proxy: OtelTracingProxySettings with ScriptLang enum
- critical_error_channels: Vec<CriticalErrorChannel> (untagged enum)
- critical_alerts_on_db_oversize: DbOversizeAlert
- ducklake_settings: DucklakeSettings with nested catalog/storage types
- custom_instance_pg_databases: CustomInstancePgDatabases
- autoscaling (worker config): AutoscalingConfig with integration struct
- custom_tags, default_tags_workspaces: Vec<String>
- default_tags_per_workspace: bool

Still opaque (serde_json::Value): object_store_cache_config (kube-core
can't generate schemas for internally-tagged enums), secret_backend
(EE-private), slack, teams (no clear struct definitions).

Regenerated CRD YAML with full OpenAPI schema (352→703 lines).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Unify instance config types and add bulk GET/PUT API

Move all typed settings (GlobalSettings, WorkerGroupConfig, etc.) from
windmill-operator/crd.rs into windmill-common/instance_config.rs so both
the API server and operator share a single source of truth. Add diff/apply
logic (Merge mode for UI, Replace mode for operator) and InstanceConfig::from_db().

Add GET/PUT /settings/instance_config endpoints so the frontend loads all
settings in 1 call instead of 42, and saves with a single bulk PUT. The
backend handles the diff internally, running pre-write hooks for changed keys.

Refactor windmill-operator/db_sync.rs to use the shared diff+apply functions
and slim crd.rs down to the CRD wrapper with re-exports.

Includes 32 unit tests and 30 integration tests covering serialization,
diff logic, DB roundtrips, protected settings, and edge cases.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: add Form/YAML toggle to instance settings UI

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: show Form/YAML toggle regardless of hideTabs prop

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* refactor: replace toggle button group with simple YAML toggle

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* feat: mask sensitive fields in YAML view with show/hide toggle

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* feat: hide internal settings and mask sensitive fields in YAML view

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* feat: hide jwt_secret and min_keep_alive_version from API and config exports

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* all

* feat: add secretKeyRef support for sensitive fields in operator CRD

Allow sensitive fields (license_key, hub_api_secret, scim_token,
smtp_password, OAuthClient.secret, custom PG user_pwd) to reference
Kubernetes Secrets via the standard secretKeyRef pattern instead of
inlining values as plaintext YAML. The reconciler resolves all refs
by reading K8s Secrets before syncing to the database.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* all

* all

* all

* fix: merge main and update dev environment docs

Resolve merge conflicts from origin/main, fix duplicate
UV_INDEX_STRATEGY_SETTING import, and add Playwright MCP
testing instructions to CLAUDE.md.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* all

* fix: init tracing for CLI subcommands and deduplicate setting side-effects

Initialize tracing subscriber before early-return CLI paths (sync-config,
operator) so tracing calls are not silently dropped. Refactor
set_global_setting_internal to call run_setting_pre_write_hook instead of
duplicating the side-effect logic.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: add `wmill instance get-config` CLI command

Dumps the current instance config (global settings + worker configs) as
YAML. Supports --output-file to write to a file instead of stdout.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* all

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-14 22:42:01 +00:00
Diego Imbert
a9dbd1f73f fix: Missing app policy for datatable (#7944) 2026-02-14 08:48:16 +01:00
Ruben Fiszel
6215760b12 fix: allow renaming of backend runnables in the UI 2026-02-13 21:43:13 +00:00
Alexander Petric
92cd7fee0b git init script update lockfile (#7942) 2026-02-13 20:32:03 +00:00
Ruben Fiszel
4fe9314a3a nit: remove temporary release-as override
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 17:27:40 +00:00
Ruben Fiszel
9e7b1783b8 fix: override release-please to 1.635.0 to prevent unintended 2.0 bump
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 17:12:52 +00:00
Ruben Fiszel
581dde8d0b feat: add maven settings.xml support for java private registries (#7940)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 16:57:30 +00:00
centdix
37d1277b91 feat(cli): add lint command (#7917)
* feat(yaml-validator)!: unify flow, schedule, and trigger validation

- replace FlowValidator with WindmillYamlValidator.validate(doc, target)

- generate schedule/trigger schemas from backend OpenAPI and OpenFlow refs

- add schedule/trigger/filename-target tests and update AI agent fixtures

- bump windmill-yaml-validator to 2.0.0

BREAKING CHANGE: FlowValidator and validateFlow() are replaced by WindmillYamlValidator.validate(doc, target).

* add lint command

* add deno-compat script and docs for local yaml-validator testing

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: make nullable fields pass yaml validation

Add nullable: true to static_asset_config and authentication_resource_path
in HttpTrigger schema. Post-process generated JSON schemas to add null to
enums with nullable: true (AJV doesn't handle OpenAPI 3.0 nullable + enum).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: add nullable to all Option<T> fields in trigger and schedule OpenAPI schemas

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(frontend): handle nullable fields from updated OpenAPI types

Add ?? undefined coalescing at assignment sites where generated types
now include | null from the OpenAPI nullable additions.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(lint): show allowed values in enum validation errors

Instead of "must be equal to one of the allowed values", now shows
"must be one of: 'r', 'w', 'rw'" for enum validation failures.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: add nullable to Edit/New trigger and schedule OpenAPI schemas

Ensures create/update request body types accept null for the same
fields that GET response types return as nullable, enabling clean
round-tripping without type mismatches.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* use published package

* publish

* refactor(lint): remove unused --includes/--excludes/--extra-includes CLI options

These options were defined but never wired to the file filtering logic.
The lint command still respects includes/excludes from wmill.yaml via
mergeConfigWithConfigFile.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat(lint): handle additionalProperties errors and expand test coverage

Add formatting for AJV additionalProperties keyword to show the unknown
property name. Add unit tests for all formatValidationError branches and
integration tests for --json report shape, --fail-on-warn with mixed
files, non-existent directory, and enum error output.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* test: add realistic validator tests for schedules, triggers, and edge cases

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: add email trigger validation support

Add email trigger schema generation, validation, and linting. Email
triggers are no longer skipped with a warning — they are validated
like all other trigger types.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* chore(cli): bump windmill-yaml-validator to 1.1.1 (email trigger support)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* publish

* rm

* fix: address PR review feedback for lint command

- Add email to trigger kinds test loop instead of separate test
- Add email to ValidationTarget docs in README
- Type formatYamlDiagnostics param directly instead of unsafe cast
- Destructure json option before mergeConfigWithConfigFile for clarity

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat(cli): add --lint option to sync push command

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-13 16:41:04 +00:00
Ruben Fiszel
a9e4a5c8e7 windows build nit 2026-02-13 15:16:13 +00:00
Ruben Fiszel
b9e7476571 restrict number of workspace forks in CE 2026-02-13 14:58:57 +00:00
Ruben Fiszel
097c5bc8f3 chore(main): release 1.634.6 (#7938)
* chore(main): release 1.634.6

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2026-02-13 14:45:33 +00:00
Ruben Fiszel
caccdd553a fix: full build fix with new rustup config 2026-02-13 14:40:02 +00:00
Ruben Fiszel
1d0703ca8f chore(main): release 1.634.5 (#7937)
* chore(main): release 1.634.5

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2026-02-13 14:16:48 +00:00
Ruben Fiszel
7d88676b15 rust nits 2026-02-13 14:13:11 +00:00
Ruben Fiszel
50f04fe8d4 rust nits 2026-02-13 14:08:42 +00:00
Ruben Fiszel
e144432a16 fix: rust + java works with just /tmp mounted 2026-02-13 14:03:45 +00:00
claude[bot]
077eb91c0f feat: add list instance emails endpoint and use Select in AddUser
Add a new GET /users/list_instance_emails endpoint that returns all
instance user emails (from the password table). This endpoint is only
available on non-cloud instances.

In the frontend AddUser component, when the instance emails are
available (non-cloud), replace the email text input with a Select
component that allows both selecting from existing instance users and
typing custom emails.

Closes #7902

Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
2026-02-11 11:24:13 +00:00
198 changed files with 16195 additions and 1831 deletions

View File

@@ -0,0 +1,756 @@
# Skill: Adding Native Trigger Services
This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications.
## Architecture Overview
The native trigger system consists of:
1. **Database Layer** - PostgreSQL tables and enum types
2. **Backend Rust Implementation** - Core trait, handlers, and service modules in the `windmill-native-triggers` crate
3. **Frontend Svelte Components** - Configuration forms and UI components
### Key Files
| Component | Path |
|-----------|------|
| Core module with `External` trait | `backend/windmill-native-triggers/src/lib.rs` |
| Generic CRUD handlers | `backend/windmill-native-triggers/src/handler.rs` |
| Background sync logic | `backend/windmill-native-triggers/src/sync.rs` |
| OAuth/workspace integration | `backend/windmill-native-triggers/src/workspace_integrations.rs` |
| Re-export shim (windmill-api) | `backend/windmill-api/src/native_triggers/mod.rs` |
| TriggerKind enum | `backend/windmill-common/src/triggers.rs` |
| JobTriggerKind enum | `backend/windmill-common/src/jobs.rs` |
| Frontend service registry | `frontend/src/lib/components/triggers/native/utils.ts` |
| Frontend trigger utilities | `frontend/src/lib/components/triggers/utils.ts` |
| Trigger badges (icons + counts) | `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte` |
| Workspace integrations UI | `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte` |
| OAuth config form component | `frontend/src/lib/components/workspaceSettings/OAuthClientConfig.svelte` |
| OpenAPI spec | `backend/windmill-api/openapi.yaml` |
| Reference: Nextcloud module | `backend/windmill-native-triggers/src/nextcloud/` |
| Reference: Google module | `backend/windmill-native-triggers/src/google/` |
### Crate Structure
The native trigger code lives in the `windmill-native-triggers` crate (`backend/windmill-native-triggers/`). The `windmill-api` crate re-exports everything via a shim:
```rust
// backend/windmill-api/src/native_triggers/mod.rs
pub use windmill_native_triggers::*;
```
All new service modules go in `backend/windmill-native-triggers/src/`.
---
## Core Concepts
### The `External` Trait
Every native trigger service implements the `External` trait defined in `lib.rs`:
```rust
#[async_trait]
pub trait External: Send + Sync + 'static {
// Associated types:
type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync;
type TriggerData: Debug + Serialize + Send + Sync;
type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync;
type CreateResponse: DeserializeOwned + Send + Sync;
// Constants:
const SUPPORT_WEBHOOK: bool;
const SERVICE_NAME: ServiceName;
const DISPLAY_NAME: &'static str;
const TOKEN_ENDPOINT: &'static str;
const REFRESH_ENDPOINT: &'static str;
const AUTH_ENDPOINT: &'static str;
// Required methods:
async fn create(&self, w_id, oauth_data, webhook_token, data, db, tx) -> Result<Self::CreateResponse>;
async fn update(&self, w_id, oauth_data, external_id, webhook_token, data, db, tx) -> Result<serde_json::Value>;
async fn get(&self, w_id, oauth_data, external_id, db, tx) -> Result<Self::TriggerData>;
async fn delete(&self, w_id, oauth_data, external_id, db, tx) -> Result<()>;
async fn exists(&self, w_id, oauth_data, external_id, db, tx) -> Result<bool>;
async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors);
fn external_id_and_metadata_from_response(&self, resp) -> (String, Option<serde_json::Value>);
// Methods with defaults:
async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result<PushArgsOwned>;
fn service_config_from_create_response(&self, data, resp) -> Option<serde_json::Value>;
fn additional_routes(&self) -> axum::Router;
async fn http_client_request<T, B>(&self, url, method, workspace_id, tx, db, headers, body) -> Result<T>;
}
```
Key design points:
- **`update()` returns `serde_json::Value`** - the resolved service_config to store. Each service is responsible for building the final config.
- **`maintain_triggers()`** - periodic background maintenance. Each service implements its own strategy (Nextcloud: reconcile with external state; Google: renew expiring channels).
- **No `list_all()` in the trait** - services that need it (Nextcloud) implement it privately; services that don't (Google) use different maintenance strategies.
- **No `get_external_id_from_trigger_data()` or `extract_service_config_from_trigger_data()`** - removed in favor of the `maintain_triggers` pattern.
### Create Lifecycle: Two Paths
The `create_native_trigger` handler in `handler.rs` supports two creation flows, controlled by `service_config_from_create_response()`:
**Path A: Short (Google pattern)** - `service_config_from_create_response()` returns `Some(config)`:
1. `create()` registers on external service
2. `external_id_and_metadata_from_response()` extracts the ID
3. `service_config_from_create_response()` builds the config directly from input data + response metadata
4. Stores trigger in DB -- done, no extra round-trip
Use this when the external_id is known before the create call (e.g., Google generates the channel_id as a UUID upfront and includes it in the webhook URL).
**Path B: Long (Nextcloud pattern)** - `service_config_from_create_response()` returns `None` (default):
1. `create()` registers on external service (webhook URL has no external_id yet)
2. `external_id_and_metadata_from_response()` extracts the ID
3. `update()` is called to fix the webhook URL with the now-known external_id
4. `update()` returns the resolved service_config
5. Stores trigger in DB
Use this when the external_id is assigned by the remote service and the webhook URL needs to be corrected after creation.
### OAuth Token Storage (Three-Table Pattern)
OAuth tokens are stored across three tables, NOT in `workspace_integrations.oauth_data` directly:
| Table | What's Stored |
|-------|---------------|
| `workspace_integrations` | `oauth_data` JSON with `base_url`, `client_id`, `client_secret`, `instance_shared` flag; `resource_path` pointing to the variable |
| `variable` | Encrypted `access_token` (at the path stored in `resource_path`), linked to `account` via `account` column |
| `account` | `refresh_token`, keyed by `workspace_id` + `client` (service name) + `is_workspace_integration = true` |
The `decrypt_oauth_data()` function in `lib.rs` assembles these into a unified struct:
```rust
pub struct OAuthConfig {
pub base_url: String,
pub access_token: String, // decrypted from variable
pub refresh_token: Option<String>, // from account table
pub client_id: String, // from oauth_data or instance settings
pub client_secret: String, // from oauth_data or instance settings
}
```
Instance-level sharing: when `oauth_data.instance_shared == true`, `client_id` and `client_secret` are read from global settings instead of workspace_integrations.
### URL Resolution
The `resolve_endpoint()` helper handles both absolute and relative OAuth URLs:
```rust
pub fn resolve_endpoint(base_url: &str, endpoint: &str) -> String {
if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
endpoint.to_string() // Google: absolute URLs
} else {
format!("{}{}", base_url, endpoint) // Nextcloud: relative paths
}
}
```
### ServiceName Methods
`ServiceName` is the central registry enum. Each variant must implement these match arms:
| Method | Purpose |
|--------|---------|
| `as_str()` | Lowercase identifier (e.g., `"google"`) |
| `as_trigger_kind()` | Maps to `TriggerKind` enum |
| `as_job_trigger_kind()` | Maps to `JobTriggerKind` enum |
| `token_endpoint()` | OAuth token endpoint (relative or absolute) |
| `auth_endpoint()` | OAuth authorization endpoint |
| `oauth_scopes()` | Space-separated OAuth scopes |
| `resource_type()` | Resource type for token storage (e.g., `"gworkspace"`) |
| `extra_auth_params()` | Extra OAuth params (e.g., Google needs `access_type=offline`, `prompt=consent`) |
| `integration_service()` | Maps to the workspace integration service (usually `*self`) |
| `TryFrom<String>` | Parse from string |
| `Display` | Delegates to `as_str()` |
---
## Step-by-Step Implementation Guide
### Step 1: Database Migration
Create a new migration file: `backend/migrations/YYYYMMDDHHMMSS_newservice_trigger.up.sql`
```sql
-- Add the service to the native_trigger_service enum
ALTER TYPE native_trigger_service ADD VALUE IF NOT EXISTS 'newservice';
-- Add to TRIGGER_KIND enum (used for trigger tracking)
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'newservice';
-- Add to job_trigger_kind enum (used for job tracking)
ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'newservice';
```
Also create the corresponding down migration.
### Step 2: Update windmill-common Enums
#### `backend/windmill-common/src/triggers.rs`
Add variant to `TriggerKind` enum, and update `to_key()` and `fmt()` implementations.
#### `backend/windmill-common/src/jobs.rs`
Add variant to `JobTriggerKind` enum and update the `Display` implementation.
### Step 3: Backend Service Module
Create a new directory: `backend/windmill-native-triggers/src/newservice/`
#### `mod.rs` - Type Definitions
```rust
use serde::{Deserialize, Serialize};
pub mod external;
// pub mod routes; // Only if you need additional service-specific routes
/// OAuth data deserialized from the three-table pattern.
/// The actual structure is built by decrypt_oauth_data() from variable + account + workspace_integrations.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct NewServiceOAuthData {
pub base_url: String, // from workspace_integrations.oauth_data
pub access_token: String, // decrypted from variable table
pub refresh_token: Option<String>, // from account table
// Note: client_id and client_secret are in OAuthConfig, not here
// unless the service needs them at runtime for API calls
}
/// Configuration provided by user when creating/updating a trigger.
/// Stored as JSON in native_trigger.service_config.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NewServiceConfig {
// Service-specific configuration fields
pub folder_path: String,
pub file_filter: Option<String>,
}
/// Data retrieved from the external service about a trigger.
/// Returned by the get() method and shown in the UI.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NewServiceTriggerData {
pub folder_path: String,
pub file_filter: Option<String>,
// Fields that shouldn't affect service_config comparison should use #[serde(skip_serializing)]
}
/// Response from external service when creating a trigger/webhook.
#[derive(Debug, Deserialize)]
pub struct CreateTriggerResponse {
pub id: String,
}
/// Handler struct (stateless, used for routing)
#[derive(Copy, Clone)]
pub struct NewService;
```
#### `external.rs` - External Trait Implementation
```rust
use async_trait::async_trait;
use reqwest::Method;
use sqlx::PgConnection;
use std::collections::HashMap;
use windmill_common::{
error::{Error, Result},
BASE_URL, DB,
};
use crate::{
generate_webhook_service_url, External, NativeTrigger, NativeTriggerData, ServiceName,
sync::{SyncError, TriggerSyncInfo},
};
use super::{NewService, NewServiceConfig, NewServiceOAuthData, NewServiceTriggerData, CreateTriggerResponse};
#[async_trait]
impl External for NewService {
type ServiceConfig = NewServiceConfig;
type TriggerData = NewServiceTriggerData;
type OAuthData = NewServiceOAuthData;
type CreateResponse = CreateTriggerResponse;
const SERVICE_NAME: ServiceName = ServiceName::NewService;
const DISPLAY_NAME: &'static str = "New Service";
const SUPPORT_WEBHOOK: bool = true;
const TOKEN_ENDPOINT: &'static str = "/oauth/token";
const REFRESH_ENDPOINT: &'static str = "/oauth/token";
const AUTH_ENDPOINT: &'static str = "/oauth/authorize";
async fn create(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<Self::CreateResponse> {
let base_url = &*BASE_URL.read().await;
// external_id is None during create (we get it from the response)
let webhook_url = generate_webhook_service_url(
base_url, w_id, &data.script_path, data.is_flow,
None, Self::SERVICE_NAME, webhook_token,
);
let url = format!("{}/api/webhooks/create", oauth_data.base_url);
let payload = serde_json::json!({
"callback_url": webhook_url,
"folder_path": data.service_config.folder_path,
});
let response: CreateTriggerResponse = self
.http_client_request(&url, Method::POST, w_id, tx, db, None, Some(&payload))
.await?;
Ok(response)
}
/// Update returns the resolved service_config as JSON.
/// For services using the update+get pattern, call self.get() and serialize.
async fn update(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<serde_json::Value> {
let base_url = &*BASE_URL.read().await;
let webhook_url = generate_webhook_service_url(
base_url, w_id, &data.script_path, data.is_flow,
Some(external_id), Self::SERVICE_NAME, webhook_token,
);
let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id);
let payload = serde_json::json!({
"callback_url": webhook_url,
"folder_path": data.service_config.folder_path,
});
let _: serde_json::Value = self
.http_client_request(&url, Method::PUT, w_id, tx, db, None, Some(&payload))
.await?;
// Fetch back the updated state to get the resolved config
let trigger_data = self.get(w_id, oauth_data, external_id, db, tx).await?;
serde_json::to_value(&trigger_data)
.map_err(|e| Error::InternalErr(format!("Failed to serialize trigger data: {}", e)))
}
async fn get(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<Self::TriggerData> {
let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id);
self.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, None, None).await
}
async fn delete(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<()> {
let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id);
let _: serde_json::Value = self
.http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, None, None)
.await
.or_else(|e| match &e {
Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null),
_ => Err(e),
})?;
Ok(())
}
async fn exists(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<bool> {
match self.get(w_id, oauth_data, external_id, db, tx).await {
Ok(_) => Ok(true),
Err(Error::NotFound(_)) => Ok(false),
Err(e) => Err(e),
}
}
/// Background maintenance. Choose the right pattern for your service:
/// - For services with queryable external state: use reconcile_with_external_state()
/// - For channel-based services with expiration: implement renewal logic
async fn maintain_triggers(
&self,
db: &DB,
workspace_id: &str,
triggers: &[NativeTrigger],
oauth_data: &Self::OAuthData,
synced: &mut Vec<TriggerSyncInfo>,
errors: &mut Vec<SyncError>,
) {
// Option A: Reconcile with external state (Nextcloud pattern)
// Fetch all triggers from external service and compare with DB
let external_triggers = match self.list_all(workspace_id, oauth_data, db).await {
Ok(triggers) => triggers,
Err(e) => {
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to list triggers: {}", e),
error_type: "api_error".to_string(),
});
return;
}
};
// Convert to (external_id, config_json) pairs
let external_pairs: Vec<(String, serde_json::Value)> = external_triggers
.into_iter()
.map(|t| (t.id.clone(), serde_json::to_value(&t).unwrap_or_default()))
.collect();
crate::sync::reconcile_with_external_state(
db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors,
).await;
}
fn external_id_and_metadata_from_response(
&self,
resp: &Self::CreateResponse,
) -> (String, Option<serde_json::Value>) {
(resp.id.clone(), None)
}
// service_config_from_create_response: NOT overridden (returns None).
// This means the handler uses the update+get pattern after create.
// Override and return Some(...) to skip the update+get cycle (Google pattern).
}
impl NewService {
/// Private helper to list all triggers from the external service.
async fn list_all(
&self,
w_id: &str,
oauth_data: &<Self as External>::OAuthData,
db: &DB,
) -> Result<Vec<<Self as External>::TriggerData>> {
// Implementation depends on the external service's API
todo!()
}
}
```
### Step 4: Update lib.rs Registry
In `backend/windmill-native-triggers/src/lib.rs`:
```rust
// Service modules - add new services here:
#[cfg(feature = "native_trigger")]
pub mod newservice; // <-- Add this
// ServiceName enum - add variant:
pub enum ServiceName {
Nextcloud,
Google,
NewService, // <-- Add this
}
// Then add match arms in ALL ServiceName methods:
// as_str(), as_trigger_kind(), as_job_trigger_kind(), token_endpoint(),
// auth_endpoint(), oauth_scopes(), resource_type(), extra_auth_params(),
// integration_service(), TryFrom<String>, Display
```
### Step 5: Update handler.rs Routes
In `backend/windmill-native-triggers/src/handler.rs`:
```rust
pub fn generate_native_trigger_routers() -> Router {
// ...
#[cfg(feature = "native_trigger")]
{
use crate::newservice::NewService;
return router
.nest("/nextcloud", service_routes(NextCloud))
.nest("/google", service_routes(Google))
.nest("/newservice", service_routes(NewService)); // <-- Add this
}
// ...
}
```
### Step 6: Update sync.rs
In `backend/windmill-native-triggers/src/sync.rs`:
```rust
pub async fn sync_all_triggers(db: &DB) -> Result<BackgroundSyncResult> {
// ...
#[cfg(feature = "native_trigger")]
{
use crate::newservice::NewService;
// ... existing service syncs ...
// New service sync
let (service_name, result) = sync_service_triggers(db, NewService).await;
total_synced += result.synced_triggers.len();
total_errors += result.errors.len();
service_results.insert(service_name, result);
}
// ...
}
```
### Step 7: Frontend Service Registry
In `frontend/src/lib/components/triggers/native/utils.ts`:
Add to `NATIVE_TRIGGER_SERVICES`, `getTriggerIconName()`, and `getServiceIcon()`.
### Step 8: Frontend Trigger Form Component
Create: `frontend/src/lib/components/triggers/native/services/newservice/NewServiceTriggerForm.svelte`
### Step 9: Frontend Icon Component
Create: `frontend/src/lib/components/icons/NewServiceIcon.svelte`
### Step 10: Update NativeTriggerEditor
Check `frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte` to ensure it dynamically loads form components based on service name.
### Step 11: Workspace Integration UI
Add your service to the `supportedServices` map in `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte`:
```typescript
const supportedServices: Record<string, ServiceConfig> = {
// ... existing services ...
newservice: {
name: 'newservice',
displayName: 'New Service',
description: 'Connect to New Service for triggers',
icon: NewServiceIcon,
docsUrl: 'https://www.windmill.dev/docs/integrations/newservice',
requiresBaseUrl: false, // false for cloud services, true for self-hosted
setupInstructions: [
'Step 1: Create an OAuth app on the service',
'Step 2: Configure the redirect URI shown below',
'Step 3: Enter the client credentials below'
]
}
}
```
### Step 12: Update `frontend/src/lib/components/triggers/utils.ts`
Update ALL of these maps/functions:
1. `triggerIconMap` - import and add icon
2. `triggerDisplayNamesMap` - add display name
3. `triggerTypeOrder` in `sortTriggers()` - add type
4. `getLightConfig()` - add case for your service
5. `getTriggerLabel()` - add case for your service
6. `jobTriggerKinds` - add to array
7. `countPropertyMap` - add count property
8. `triggerSaveFunctions` - add save function
### Step 13: Update TriggersBadge Component
In `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte`:
1. Import the icon
2. Add to `baseConfig` with `countKey` (the dynamic `availableNativeServices` loop does NOT set `countKey`)
3. Add to the `allTypes` array
### Step 14: Update OpenAPI Spec and Regenerate Types
Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then:
```bash
cd frontend && npm run generate-backend-client
```
---
## Special Patterns
### Unified Service with `trigger_type` (Google Pattern)
When a single service handles multiple trigger types (e.g., Google Drive + Calendar share OAuth and API patterns), use a single `ServiceName` variant with a discriminator field:
```rust
pub enum GoogleTriggerType { Drive, Calendar }
pub struct GoogleServiceConfig {
pub trigger_type: GoogleTriggerType,
// Drive-specific fields (only used when trigger_type = Drive)
pub resource_id: Option<String>,
pub resource_name: Option<String>,
// Calendar-specific fields (only used when trigger_type = Calendar)
pub calendar_id: Option<String>,
pub calendar_name: Option<String>,
// Metadata set after creation
pub google_resource_id: Option<String>,
pub expiration: Option<String>,
}
```
Branch in trait methods based on `trigger_type`. Frontend uses a `ToggleButtonGroup` to switch between types. This keeps the codebase simpler (one service, one OAuth flow, one set of routes).
See `backend/windmill-native-triggers/src/google/` for the reference implementation.
### Skipping update+get After Create (Google Pattern)
Override `service_config_from_create_response()` to return `Some(config)` when the external_id is known before the create call:
```rust
fn service_config_from_create_response(
&self,
data: &NativeTriggerData<Self::ServiceConfig>,
resp: &Self::CreateResponse,
) -> Option<serde_json::Value> {
// Clone input config, add metadata from response
let mut config = data.service_config.clone();
config.google_resource_id = Some(resp.resource_id.clone());
config.expiration = Some(resp.expiration.clone());
Some(serde_json::to_value(&config).unwrap())
}
```
### Services with Absolute OAuth Endpoints (Google)
Unlike self-hosted services where OAuth endpoints are relative paths appended to `base_url`, services like Google have absolute URLs:
```rust
// Nextcloud: relative paths
ServiceName::Nextcloud => "/apps/oauth2/api/v1/token",
// Google: absolute URLs
ServiceName::Google => "https://oauth2.googleapis.com/token",
```
The `resolve_endpoint()` function handles both. For services with absolute endpoints:
- `base_url` can be empty
- `requiresBaseUrl: false` in the frontend workspace integration config
- Add `extra_auth_params()` if needed (Google requires `access_type=offline` and `prompt=consent`)
### Channel-Based Push Notifications with Renewal (Google Pattern)
For services using expiring watch channels instead of persistent webhooks:
1. Store expiration in `service_config` (as part of `ServiceConfig`)
2. In `maintain_triggers()`, implement renewal logic instead of using `reconcile_with_external_state()`:
```rust
async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors) {
for trigger in triggers {
if should_renew_channel(trigger) {
self.renew_channel(db, trigger, oauth_data).await;
}
}
}
```
3. Renewal: best-effort stop old channel, create new one with same external_id, update service_config with new expiration
4. Google example: Drive channels expire in 24h (renew when <1h left), Calendar channels expire in 7 days (renew when <1 day left)
### reconcile_with_external_state (Nextcloud Pattern)
The reusable function in `sync.rs` compares external triggers with DB state:
- Triggers missing externally: sets error "Trigger no longer exists on external service"
- Triggers present externally: clears errors, updates service_config if it differs
Usage in `maintain_triggers()`:
```rust
let external_pairs: Vec<(String, serde_json::Value)> = /* fetch from external */;
crate::sync::reconcile_with_external_state(
db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors,
).await;
```
### Webhook Payload Processing
Override `prepare_webhook()` to parse service-specific payloads into script/flow args:
```rust
async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result<PushArgsOwned> {
let mut args = HashMap::new();
args.insert("event_type".to_string(), Box::new(headers.get("x-event-type").cloned()) as _);
args.insert("payload".to_string(), Box::new(serde_json::from_str::<serde_json::Value>(&body)?) as _);
Ok(PushArgsOwned { extra: None, args })
}
```
Then register in `prepare_native_trigger_args()` in `lib.rs`:
```rust
pub async fn prepare_native_trigger_args(service_name, db, w_id, headers, body) -> Result<Option<PushArgsOwned>> {
match service_name {
ServiceName::Google => { /* ... */ Ok(Some(args)) }
ServiceName::NewService => { /* ... */ Ok(Some(args)) }
ServiceName::Nextcloud => Ok(None), // Uses default body parsing
}
}
```
### Instance-Level OAuth Credentials
When `workspace_integrations.oauth_data.instance_shared == true`, `decrypt_oauth_data()` reads `client_id` and `client_secret` from instance-level global settings instead of workspace-level. This allows admins to share OAuth app credentials across workspaces.
The frontend handles this via the `generate_instance_connect_url` endpoint in `workspace_integrations.rs`.
---
## Testing Checklist
- [ ] Database migration runs successfully
- [ ] `cargo check -p windmill-native-triggers --features native_trigger` passes
- [ ] `npx svelte-check --threshold error` passes (in frontend/)
- [ ] Service appears in workspace integrations list
- [ ] OAuth flow completes successfully
- [ ] Can create a new trigger
- [ ] Can view trigger details
- [ ] Can update trigger configuration
- [ ] Can delete trigger
- [ ] Webhook receives and processes payloads
- [ ] Background sync works correctly (reconciliation or channel renewal)
- [ ] Error handling works (expired tokens, service unavailable)
---
## Reference Implementations
### Nextcloud (Self-Hosted, Update+Get Pattern)
| File | Purpose |
|------|---------|
| `nextcloud/mod.rs` | Types: NextCloudOAuthData, NextcloudServiceConfig, NextCloudTriggerData |
| `nextcloud/external.rs` | External trait: uses update+get pattern, reconcile_with_external_state for sync |
| `nextcloud/routes.rs` | Additional route: `GET /events` |
Key patterns: relative OAuth endpoints, base_url required, list_all + reconcile for sync, update returns JSON from get().
### Google (Cloud, Unified Service, Short Create)
| File | Purpose |
|------|---------|
| `google/mod.rs` | Types: GoogleServiceConfig with trigger_type discriminator, GoogleTriggerType enum |
| `google/external.rs` | External trait: overrides service_config_from_create_response, channel renewal for sync |
| `google/routes.rs` | Additional routes: `GET /calendars`, `GET /drive/files`, `GET /drive/shared_drives` |
Key patterns: absolute OAuth endpoints, empty base_url, trigger_type for Drive/Calendar, expiring watch channels with renewal, service_config_from_create_response skips update+get, get() reconstructs data from stored service_config (no external "get channel" API).

View File

@@ -3,6 +3,10 @@
"svelte": {
"type": "http",
"url": "https://mcp.svelte.dev/mcp"
},
"playwright": {
"command": "npx",
"args": ["@playwright/mcp@latest"]
}
}
}

View File

@@ -1,5 +1,59 @@
# Changelog
## [1.636.0](https://github.com/windmill-labs/windmill/compare/v1.635.1...v1.636.0) (2026-02-16)
### Features
* allow adding workspace scripts and flows as AI chat context ([#7882](https://github.com/windmill-labs/windmill/issues/7882)) ([5b8ec50](https://github.com/windmill-labs/windmill/commit/5b8ec502fef8fb439200e18b8c610d0f5998b6df))
* google native triggers ([#7837](https://github.com/windmill-labs/windmill/issues/7837)) ([6f24f19](https://github.com/windmill-labs/windmill/commit/6f24f1939d75a597acc74c1589794d511e041baa))
### Bug Fixes
* mark base_url as unsaved when using browser fallback ([#7964](https://github.com/windmill-labs/windmill/issues/7964)) ([e7b0b00](https://github.com/windmill-labs/windmill/commit/e7b0b00f5696828dec094155298d0c9dc033b355))
## [1.635.1](https://github.com/windmill-labs/windmill/compare/v1.635.0...v1.635.1) (2026-02-15)
### Bug Fixes
* pin strum_macros to 0.27 to match strum version ([#7957](https://github.com/windmill-labs/windmill/issues/7957)) ([68f766e](https://github.com/windmill-labs/windmill/commit/68f766e1ae54dbe2fe42769559d81d4d76a409ef))
## [1.635.0](https://github.com/windmill-labs/windmill/compare/v1.634.6...v1.635.0) (2026-02-15)
### Features
* add Kubernetes operator and instance settings YAML editor ([#7836](https://github.com/windmill-labs/windmill/issues/7836)) ([82e5f6d](https://github.com/windmill-labs/windmill/commit/82e5f6de48e246a49b25e7d4ea7be65122e8772c))
* add maven settings.xml support for java private registries ([#7940](https://github.com/windmill-labs/windmill/issues/7940)) ([581dde8](https://github.com/windmill-labs/windmill/commit/581dde8d0bc4428a5e95fcb5341239231ab36ef6))
* **cli:** add `lint` command ([#7917](https://github.com/windmill-labs/windmill/issues/7917)) ([37d1277](https://github.com/windmill-labs/windmill/commit/37d1277b91d1b8a03e327b0585f547037482498d))
* handle $var: and $res: in arrays for transform_json_value ([#7949](https://github.com/windmill-labs/windmill/issues/7949)) ([e4a34d0](https://github.com/windmill-labs/windmill/commit/e4a34d031b2bdb1b73a2a7ca68544fa34f83ed0f))
* IaC hints, YAML editor for worker configs ([#7956](https://github.com/windmill-labs/windmill/issues/7956)) ([8b8e33e](https://github.com/windmill-labs/windmill/commit/8b8e33e2dc1a2b4c0effab70463f6d4b402a0f7f))
* open-source worker group configuration UI ([#7954](https://github.com/windmill-labs/windmill/issues/7954)) ([6cf3f5f](https://github.com/windmill-labs/windmill/commit/6cf3f5f4a35a6139b5cdf9f44af29c3941f19645))
### Bug Fixes
* allow renaming of backend runnables in the UI ([6215760](https://github.com/windmill-labs/windmill/commit/6215760b1294d55245909a1c1de6c4cc8cef320a))
* **go:** preserve proxy envs for go mod tidy/download ([#7946](https://github.com/windmill-labs/windmill/issues/7946)) ([8410b59](https://github.com/windmill-labs/windmill/commit/8410b59a8f23d62c57e497d170449643b46595a0))
* Missing app policy for datatable ([#7944](https://github.com/windmill-labs/windmill/issues/7944)) ([a9dbd1f](https://github.com/windmill-labs/windmill/commit/a9dbd1f73fca9100b64106281802c43881181e78))
* strip slack_oauth_client_secret from get_settings for non-admins ([#7950](https://github.com/windmill-labs/windmill/issues/7950)) ([43218c6](https://github.com/windmill-labs/windmill/commit/43218c62852490d0efafa8f94385bfe0e8f2ad82))
## [1.634.6](https://github.com/windmill-labs/windmill/compare/v1.634.5...v1.634.6) (2026-02-13)
### Bug Fixes
* full build fix with new rustup config ([caccdd5](https://github.com/windmill-labs/windmill/commit/caccdd553ad72ff26c2c7c45f0ff3a25bd19a49f))
## [1.634.5](https://github.com/windmill-labs/windmill/compare/v1.634.4...v1.634.5) (2026-02-13)
### Bug Fixes
* rust + java works with just /tmp mounted ([e144432](https://github.com/windmill-labs/windmill/commit/e144432a168178a531aa146def0aff478f3d1586))
## [1.634.4](https://github.com/windmill-labs/windmill/compare/v1.634.3...v1.634.4) (2026-02-13)

View File

@@ -20,6 +20,26 @@ When implementing new features in Windmill, follow these best practices:
- Backend (Rust): see `backend/CLAUDE.md` and the `rust-backend` skill: `.claude/skills/rust-backend/SKILL.md`
- Frontend (Svelte 5): see `frontend/CLAUDE.md` and the `svelte-frontend` skill: `.claude/skills/svelte-frontend/SKILL.md`
## Dev Environment
- **Backend**: `cargo run` from `backend/` (API at http://localhost:8000)
- **Frontend**: `REMOTE=http://localhost:8000 npm run dev` from `frontend/`
- The `REMOTE` env var configures the Vite proxy target. Without it, API calls proxy to `https://app.windmill.dev` instead of the local backend.
- The dev server starts on port 3000 (or 3001+ if 3000 is in use).
- **Default login**: `admin@windmill.dev` / `changeme`
- **Instance settings**: navigate to `/#superadmin-settings` (opens the drawer overlay)
## UI Testing with Playwright MCP
When testing the frontend with the Playwright MCP tools:
1. **Start servers**: Launch backend (`cargo run`) and frontend (`REMOTE=http://localhost:8000 npm run dev`) as background tasks
2. **Wait for readiness**: Backend takes ~60s to compile; check output for `health check completed`. Frontend starts in ~5s.
3. **Login flow**: Navigate to `/user/login`, click "Log in without third-party", fill email/password, submit
4. **Instance settings drawer**: Navigate to `/#superadmin-settings` to open the drawer directly
5. **Toggle components**: The YAML toggle uses a custom `<Toggle>` component where the checkbox is visually hidden (`sr-only`). Click the wrapper `<label>` element (the parent container with `cursor=pointer`), not the checkbox ref directly.
6. **Console errors to ignore**: `critical_alerts` 404s are expected on CE builds (EE-only endpoint). VSCode worker 404s are dev-mode artifacts.
## Code Validation (MUST DO)
After making code changes, you MUST run the appropriate checks and fix all errors before considering the work done:

View File

@@ -241,7 +241,7 @@ RUN mkdir -p /tmp/windmill/cache && \
cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \
chmod -R a+rw /tmp/windmill/cache && \
rm -rf /tmp/build_cache && \
mkdir -p -m 777 /tmp/windmill/cache/uv /tmp/windmill/cache/go
mkdir -p -m 777 /tmp/windmill/cache/uv /tmp/windmill/cache/go /tmp/windmill/cache/rustup /tmp/windmill/cache/cargo
# Runtime cache locations
ENV UV_CACHE_DIR=/tmp/windmill/cache/uv
@@ -264,8 +264,8 @@ COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
ENV RUSTUP_HOME="/usr/local/rustup"
ENV CARGO_HOME="/usr/local/cargo"
ENV RUSTUP_HOME="/tmp/windmill/cache/rustup"
ENV CARGO_HOME="/tmp/windmill/cache/cargo"
ENV LD_LIBRARY_PATH="."
# nsjail runtime deps and binary

View File

@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n oauth_data as \"oauth_data!: sqlx::types::Json<WorkspaceOAuthConfig>\",\n service_name as \"service_name!: ServiceName\"\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n ",
"query": "\n SELECT\n oauth_data as \"oauth_data: sqlx::types::Json<WorkspaceOAuthConfig>\",\n service_name as \"service_name!: ServiceName\",\n resource_path\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "oauth_data!: sqlx::types::Json<WorkspaceOAuthConfig>",
"name": "oauth_data: sqlx::types::Json<WorkspaceOAuthConfig>",
"type_info": "Jsonb"
},
{
@@ -22,6 +22,11 @@
}
}
}
},
{
"ordinal": 2,
"name": "resource_path",
"type_info": "Text"
}
],
"parameters": {
@@ -30,9 +35,10 @@
]
},
"nullable": [
true,
false,
false
true
]
},
"hash": "5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee"
"hash": "0010ef26da16facd1c2c832601ac687c4c27de46a90f45496b8446af1a9d0578"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM resource WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "05e05a9b979941c7a11cd881da652f459e4a0444d63a96deba4a879fbe1124ff"
}

View File

@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n custom_path\n FROM \n app\n WHERE \n custom_path IN (\n SELECT \n custom_path\n FROM \n app\n GROUP \n BY custom_path\n HAVING COUNT(*) > 1\n )\n ORDER BY custom_path\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "custom_path",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
true
]
},
"hash": "11e24f758a70cd5f3a240bc81a05f40754826db0ee1194409227597a98603e92"
}

View File

@@ -1,11 +1,10 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_integrations\n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ",
"query": "DELETE FROM native_trigger WHERE workspace_id = $1 AND service_name = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Text",
{
"Custom": {
@@ -22,5 +21,5 @@
},
"nullable": []
},
"hash": "3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3"
"hash": "1af48c42255f1c973b4a9c9a58050bf5ec1ee6f93f0a90c1c7d0c0fcd816702d"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, mcp_server_url FROM account WHERE workspace_id = $1 AND id = $2",
"query": "SELECT client, refresh_token, grant_type, cc_client_id, cc_client_secret, cc_token_url, mcp_server_url, is_workspace_integration FROM account WHERE workspace_id = $1 AND id = $2",
"describe": {
"columns": [
{
@@ -37,6 +37,11 @@
"ordinal": 6,
"name": "mcp_server_url",
"type_info": "Text"
},
{
"ordinal": 7,
"name": "is_workspace_integration",
"type_info": "Bool"
}
],
"parameters": {
@@ -52,8 +57,9 @@
true,
true,
true,
true
true,
false
]
},
"hash": "e26ccc6607a9c78c1a8c1fd7b3bec931cf0ed27f79f852ae7f63a0ed6e12042f"
"hash": "1ba2e23d4ba816048ec1e88af9e342867fc0443cabea16d111afa2b91d3fe03b"
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n custom_path\n FROM\n app\n WHERE\n custom_path IN (\n SELECT\n custom_path\n FROM\n app\n GROUP\n BY custom_path\n HAVING COUNT(*) > 1\n )\n ORDER BY custom_path\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "custom_path",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
true
]
},
"hash": "26b35cf50959b1b1fd7e1cb33c65da40d29e20fd16b02355ba073f420c03a767"
}

View File

@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO account (workspace_id, client, expires_at, refresh_token, is_workspace_integration)\n VALUES ($1, $2, now() + ($3 || ' seconds')::interval, $4, true)\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "26beff5e94b68703ad81ef9dd2d08869eb3bb7659efd9bac04cdf98ae963063d"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM account WHERE workspace_id = $1 AND client = $2 AND is_workspace_integration = true RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "27065225c6affd26f1533dacffe1c38321511b5a7dd2a7e9435c04868188fd44"
}

View File

@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) FROM native_trigger WHERE workspace_id = 'test-workspace' AND service_name = 'google'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "2e5dd992b0bfd7550d6f4cb5424a1c14352527b98249bce286790641bf56491e"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ('oauths', $1)\n ON CONFLICT (name) DO UPDATE SET value = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "4be53f0b801ebc1a33a184556fd138fdec8082f31f56d7023cf8c6311964f3b0"
}

View File

@@ -15,7 +15,7 @@
]
},
"nullable": [
true
null
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) FROM variable WHERE workspace_id = 'test-workspace' AND path = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "607c13333627e79557d3b6f6f68eee0a5dbe7cd4643e4bf99a592eb1bb82580c"
}

View File

@@ -0,0 +1,19 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource (workspace_id, path, value, resource_type, description, created_by)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, path) DO UPDATE\n SET value = EXCLUDED.value, resource_type = EXCLUDED.resource_type",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Varchar",
"Text",
"Varchar"
]
},
"nullable": []
},
"hash": "6868520d496afe306bbd93293076ea4bb155097d1e8d3ffe5b75dd80ced735de"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT refresh_token FROM account WHERE workspace_id = $1 AND id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "refresh_token",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Int4"
]
},
"nullable": [
false
]
},
"hash": "6b4d48527af6f1411dc5e03f9144fb127488a79ac53a154d71253628320b1084"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE account SET\n refresh_token = $1,\n expires_at = now() + interval '1 hour',\n refresh_error = NULL\n WHERE workspace_id = $2 AND client = $3 AND is_workspace_integration = true",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "6e83478011a8f65bf294ad7886139369e386f6a552c6626be48ecaa0e5ab78a7"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n oauth_data,\n created_at,\n updated_at,\n created_by\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n AND service_name = $2\n ",
"query": "\n SELECT\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n oauth_data,\n resource_path,\n created_at,\n updated_at,\n created_by\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n AND service_name = $2\n ",
"describe": {
"columns": [
{
@@ -30,16 +30,21 @@
},
{
"ordinal": 3,
"name": "resource_path",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"ordinal": 5,
"name": "updated_at",
"type_info": "Timestamptz"
},
{
"ordinal": 5,
"ordinal": 6,
"name": "created_by",
"type_info": "Varchar"
}
@@ -63,11 +68,12 @@
"nullable": [
false,
false,
false,
true,
true,
false,
false,
false
]
},
"hash": "3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2"
"hash": "7443ba1f922e190bb9a0ca313847f8d27c35a6ee3aff20157d6285e73aa923ef"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO account (workspace_id, client, expires_at, refresh_token, is_workspace_integration)\n VALUES ('test-workspace', $1, now() + interval '1 hour', $2, true)\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
},
"hash": "7a57f58e809e482a599722d3887fb7e115506ff1e5ec9cf6dd2af84ed9a78632"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE account SET\n expires_at = now() + interval '1 hour',\n refresh_error = NULL\n WHERE workspace_id = $1 AND client = $2 AND is_workspace_integration = true",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "7b1d29170c9c4ad4e3a15c4e8acbeb6769dc6fc97269beee607a5625a495a121"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2\n AND oauth_data IS NOT NULL\n )\n ",
"query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations wi\n WHERE wi.workspace_id = $1\n AND wi.service_name = $2\n AND wi.oauth_data IS NOT NULL\n )\n ",
"describe": {
"columns": [
{
@@ -29,5 +29,5 @@
null
]
},
"hash": "95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7"
"hash": "823fc5f998fe747ec8537752d9eb7ef548b2fd9ee1f5380084f27796c2bcc8ad"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM variable WHERE workspace_id = $1 AND account = ANY($2)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int4Array"
]
},
"nullable": []
},
"hash": "826a4216830f6a930c382209a20bc7f8b460064480e080b989e31df3d6a30e31"
}

View File

@@ -0,0 +1,39 @@
{
"db_name": "PostgreSQL",
"query": "SELECT external_id, webhook_token_prefix FROM native_trigger WHERE workspace_id = $1 AND service_name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "external_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "webhook_token_prefix",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud",
"google"
]
}
}
}
]
},
"nullable": [
false,
false
]
},
"hash": "83d6e371ca84903e9f487afc065353a9f7be86ff752612909587ec3cb770cb75"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_integrations SET resource_path = REGEXP_REPLACE(resource_path, 'u/' || $2 || '/(.*)', 'u/' || $1 || '/\\1') WHERE resource_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "89791ad1a0862b6475ebfdeb54b0101e124fcf9d12e93d84b44457c72c7604a5"
}

View File

@@ -1,12 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM native_trigger\n WHERE\n workspace_id = $1 AND\n service_name = $2 AND\n external_id = $3\n )\n ",
"query": "\n SELECT service_config\n FROM native_trigger\n WHERE external_id = $1 AND service_name = $2 AND workspace_id = $3\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
"name": "service_config",
"type_info": "Jsonb"
}
],
"parameters": {
@@ -27,8 +27,8 @@
]
},
"nullable": [
null
true
]
},
"hash": "06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf"
"hash": "8a4d42e373043bc509985ab320894bf3e6afa7b2019cc4739baac9165f7ead9e"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE account SET refresh_token = $1 WHERE workspace_id = 'test-workspace' AND id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Int4"
]
},
"nullable": []
},
"hash": "8e5881225f4bf7243bd40397ac8b8708fb6ce6c0ba0d263bb7eeb404f5dd62ff"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM resource WHERE workspace_id = $1 AND resource_type = $2 AND path LIKE 'u/%/native_%'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "8f33846f6c25a78267a5c8143b414f033280ebf57b7b9568dc2fe31bc625020d"
}

View File

@@ -26,7 +26,7 @@
]
},
"nullable": [
false
true
]
},
"hash": "9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e"

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) FROM resource WHERE workspace_id = 'test-workspace' AND path = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "9242b5a866d0dd489bebe4284413d37202be70068affca92c45d112e0210538a"
}

View File

@@ -0,0 +1,33 @@
{
"db_name": "PostgreSQL",
"query": "SELECT resource_path FROM workspace_integrations WHERE workspace_id = $1 AND service_name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "resource_path",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud",
"google"
]
}
}
}
]
},
"nullable": [
true
]
},
"hash": "993e514229a1d508a44ef07b4399f9697cdefaa1f45ca665f72bc6fcf2797c7e"
}

View File

@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE path = 'g/all/setup_app')",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "a264bbd8dbabb03854bd25350a7aeda0704770eb200bae635f1933eece90c9d6"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO workspace_integrations (\n workspace_id,\n service_name,\n oauth_data,\n created_by,\n created_at,\n updated_at\n ) VALUES (\n $1, $2, $3, $4, now(), now()\n )\n ON CONFLICT (workspace_id, service_name)\n DO UPDATE SET\n oauth_data = $3,\n updated_at = now()\n ",
"query": "\n INSERT INTO workspace_integrations (\n workspace_id,\n service_name,\n oauth_data,\n resource_path,\n created_by,\n created_at,\n updated_at\n ) VALUES (\n $1, $2, $3, $4, $5, now(), now()\n )\n ON CONFLICT (workspace_id, service_name)\n DO UPDATE SET\n oauth_data = $3,\n resource_path = $4,\n updated_at = now()\n ",
"describe": {
"columns": [],
"parameters": {
@@ -18,10 +18,11 @@
}
},
"Jsonb",
"Text",
"Varchar"
]
},
"nullable": []
},
"hash": "2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8"
"hash": "a588f4caa014008b50eccd09122b09f8a098e58791893bacaaf2ff67a30c031c"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\",\n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\",\n EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS \"gcp_used!\",\n EXISTS(SELECT 1 FROM email_trigger WHERE workspace_id = $1) AS \"email_used!\",\n EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS \"nextcloud_used!\"\n ",
"query": "\n SELECT\n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\",\n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\",\n EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS \"gcp_used!\",\n EXISTS(SELECT 1 FROM email_trigger WHERE workspace_id = $1) AS \"email_used!\",\n EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS \"nextcloud_used!\",\n EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'google'::native_trigger_service) AS \"google_used!\"\n ",
"describe": {
"columns": [
{
@@ -52,6 +52,11 @@
"ordinal": 9,
"name": "nextcloud_used!",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "google_used!",
"type_info": "Bool"
}
],
"parameters": {
@@ -69,8 +74,9 @@
null,
null,
null,
null,
null
]
},
"hash": "d14e982c3d74499ec4bc62118e0edf065799eec5cf16f439b5f7568f392e60c3"
"hash": "a6a25545af16db9f03552ce2ac178cfda3f2ced1b8f1e60bdfc7d84c642903d2"
}

View File

@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, account, is_oauth)\n VALUES ($1, $2, $3, true, $4, $5, true)\n ON CONFLICT (workspace_id, path) DO UPDATE\n SET value = EXCLUDED.value, account = EXCLUDED.account",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Int4"
]
},
"nullable": []
},
"hash": "ab5720c0af66aba9fd7d6b842f098878c431d52bdd7b71355fc7192144b91300"
}

View File

@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM native_trigger WHERE workspace_id = $1 AND script_path = $2 AND is_flow = $3 AND service_name = 'google'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "bc1298e492d3008386d9b1b449a98156fe186832494d16a434921727e5d3314d"
}

View File

@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value, account FROM variable WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "account",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true
]
},
"hash": "c0aff25f0cc3b71842b0ba9ae55b6bc5eca203bf02f46164db08580d128b860a"
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) FROM account WHERE workspace_id = 'test-workspace' AND id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int4"
]
},
"nullable": [
null
]
},
"hash": "c1757ea525295ac9a0681be83a1f9d1e70944f65562e38c078b683e09cd9fb09"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by)\n VALUES ('test-workspace', $1, $2, $3, '{}'::jsonb, 'test-user')",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb",
"Varchar"
]
},
"nullable": []
},
"hash": "c2bf1109d208d3aa989b2e12c0380f54638edc40788d7417a08d08a267426b5e"
}

View File

@@ -0,0 +1,33 @@
{
"db_name": "PostgreSQL",
"query": "SELECT oauth_data FROM workspace_integrations\n WHERE workspace_id = $1 AND service_name = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "oauth_data",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "native_trigger_service",
"kind": {
"Enum": [
"nextcloud",
"google"
]
}
}
}
]
},
"nullable": [
true
]
},
"hash": "e241023a0d7b24adf7940ae764f14136b6d19fefbd8389e5ecd3bfc9bd652632"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE variable SET value = $1 WHERE workspace_id = 'test-workspace' AND path = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "e4c508a9bc69ccb4b32cf50caa97f9a2ff7c5990df953296d7227c1c81bc5130"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, account, is_oauth)\n VALUES ('test-workspace', $1, $2, true, 'test oauth token', $3, true)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int4"
]
},
"nullable": []
},
"hash": "e80dc984cd1d2388cbf17206ad059137cf7f92d0222382af1a66de807f3138e8"
}

View File

@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email FROM password ORDER BY email LIMIT 1000",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "f11d3a43f804a33e412a69be54824d7666d4dac0b824095c4c4fefb1f802c6f7"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_integrations SET resource_path = $1 WHERE workspace_id = $2 AND resource_path = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "f5460eb13ea4f0e9896928a3266e419090f72e54f7347b35254a661116ba822d"
}

466
backend/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.634.4"
version = "1.636.0"
authors.workspace = true
edition.workspace = true
@@ -52,6 +52,7 @@ members = [
"./windmill-audit",
"./windmill-git-sync",
"./windmill-autoscaling",
"./windmill-operator",
"./windmill-indexer",
"./windmill-macros",
"./windmill-oauth",
@@ -74,7 +75,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.634.4"
version = "1.636.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -95,7 +96,7 @@ lto = "thin"
[features]
default = []
private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"]
private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"]
agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
local_reports = ["windmill-common/local_reports"]
@@ -135,6 +136,8 @@ zip = ["windmill-api/zip"]
static_frontend = ["windmill-api/static_frontend"]
scoped_cache = ["windmill-common/scoped_cache"]
no_auth = ["windmill-api/no_auth"]
operator = ["dep:windmill-operator"]
test_job_debouncing = []
private_registry_test = []
# Languages
python = ["windmill-worker/python", "windmill-api/python", "windmill-test-utils/python"]
@@ -163,11 +166,11 @@ oss_core = [
"static_frontend", "mcp", "bedrock", "inline_preview",
"quickjs"
]
ce_core = ["oss_core", "private"]
ce_core = ["oss_core", "private", "operator"]
ee_core = [
"enterprise", "stripe", "prometheus", "cloud",
"kafka", "sqs_trigger", "nats", "gcp_trigger",
"jemalloc", "otel"
"jemalloc", "otel", "operator"
]
ee_server = ["enterprise_saml", "tantivy", "agent_worker_server", "local_reports"]
# Edition meta-features: CE variants
@@ -203,8 +206,10 @@ windmill-api-settings.workspace = true
windmill-worker.workspace = true
windmill-indexer = { workspace = true, optional = true }
windmill-autoscaling = { workspace = true, optional = true }
windmill-operator = { workspace = true, optional = true }
futures.workspace = true
tracing.workspace = true
tracing-subscriber.workspace = true
sqlx.workspace = true
sql-builder.workspace = true
rand.workspace = true
@@ -266,6 +271,7 @@ windmill-common = { path = "./windmill-common", default-features = false }
windmill-audit = { path = "./windmill-audit" }
windmill-git-sync = { path = "./windmill-git-sync" }
windmill-autoscaling = { path = "./windmill-autoscaling" }
windmill-operator = { path = "./windmill-operator" }
windmill-indexer = {path = "./windmill-indexer"}
windmill-mcp = {path = "./windmill-mcp"}
windmill-oauth = {path = "./windmill-oauth"}
@@ -558,6 +564,7 @@ backon = "1.3.0"
flume = { version = "0.11.1", features = ["async"] }
kube = { version = "1.1.0", features = ["runtime", "derive"] }
schemars = "0.8"
k8s-openapi = { version = "0.25.0", features = ["latest"] }
libloading = "0.8.8"
@@ -577,7 +584,7 @@ tree-sitter-ruby = "0.23.0"
oracle = { version = "0.6.3", features = ["chrono"] }
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}
strum = { version = "0.27", features = ["derive"] }
strum_macros = "^0"
strum_macros = "0.27"
hudsucker = { version = "0.22", features = ["rcgen-ca", "native-tls-client"] }
hyper-http-proxy = { version = "1", default-features = false, features = ["native-tls"] }
rcgen = "0.13"

View File

@@ -1 +1 @@
e7f80bca9320580e1cb96b4f4ca9942649abce7f
9f6e1e533df7711600ec2b8d5f0c958448db1a20

View File

@@ -0,0 +1,11 @@
-- Note: PostgreSQL does not support removing enum values directly.
-- This down migration is a placeholder. To fully reverse, you would need to:
-- 1. Create a new enum type without the values
-- 2. Update all columns to use the new type
-- 3. Drop the old type
-- 4. Rename the new type
-- For now, we just document what was added:
-- Removed from native_trigger_service: 'google'
-- Removed from TRIGGER_KIND: 'google'
-- Removed from job_trigger_kind: 'google'

View File

@@ -0,0 +1,10 @@
-- Add Google to native_trigger_service enum
-- 'google' is a unified service that handles both Drive and Calendar triggers
-- The trigger_type field in service_config determines which Google service is used
ALTER TYPE native_trigger_service ADD VALUE IF NOT EXISTS 'google';
-- Add to TRIGGER_KIND enum (used for trigger tracking)
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'google';
-- Add to job_trigger_kind enum (used for job tracking)
ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'google';

View File

@@ -0,0 +1,3 @@
ALTER TABLE account DROP COLUMN IF EXISTS is_workspace_integration;
ALTER TABLE workspace_integrations ALTER COLUMN oauth_data SET NOT NULL;
ALTER TABLE workspace_integrations DROP COLUMN IF EXISTS resource_path;

View File

@@ -0,0 +1,10 @@
-- Migrate native trigger OAuth tokens from workspace_integrations to account+variable+resource pattern
-- Add flag to distinguish workspace integration accounts from regular user OAuth accounts
ALTER TABLE account ADD COLUMN is_workspace_integration BOOLEAN NOT NULL DEFAULT false;
-- Make oauth_data nullable since it will only store client config (no tokens) going forward
ALTER TABLE workspace_integrations ALTER COLUMN oauth_data DROP NOT NULL;
-- Add resource_path column to workspace_integrations
ALTER TABLE workspace_integrations ADD COLUMN IF NOT EXISTS resource_path TEXT;

View File

@@ -10,10 +10,11 @@ use monitor::{
load_base_url, load_otel, reload_critical_alerts_on_db_oversize,
reload_delete_logs_periodically_setting, reload_indexer_config,
reload_instance_python_version_setting, reload_maven_repos_setting,
reload_no_default_maven_setting, reload_nuget_config_setting,
reload_powershell_repo_pat_setting, reload_powershell_repo_url_setting,
reload_ruby_repos_setting, reload_timeout_wait_result_setting,
send_current_log_file_to_object_store, send_logs_to_object_store, WORKERS_NAMES,
reload_maven_settings_xml_setting, reload_no_default_maven_setting,
reload_nuget_config_setting, reload_powershell_repo_pat_setting,
reload_powershell_repo_url_setting, reload_ruby_repos_setting,
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store, WORKERS_NAMES,
};
use rand::Rng;
use sqlx::{Pool, Postgres};
@@ -41,16 +42,16 @@ use windmill_common::{
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
JOB_ISOLATION_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, UV_INDEX_STRATEGY_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TEAMS_SETTING,
TIMEOUT_WAIT_RESULT_SETTING,
TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING,
},
scripts::ScriptLang,
stats_oss::schedule_stats,
@@ -99,12 +100,11 @@ use crate::monitor::{
reload_app_workspaced_route_setting, reload_base_url_setting,
reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting,
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_job_isolation_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting,
reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key,
reload_npm_config_registry_setting,
reload_otel_tracing_proxy_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration,
reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting,
reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key,
reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration,
};
#[cfg(feature = "parquet")]
@@ -484,6 +484,9 @@ fn print_help() {
println!(" version Show Windmill version and exit");
println!(" cache [hubPaths.json] Pre-cache hub scripts (default: ./hubPaths.json)");
println!(" cache-rt Pre-cache hub resource types");
println!(" sync-config <file> Sync instance config from a YAML file to the database");
println!(" operator Run the Kubernetes operator (watches WindmillInstance CRDs)");
println!(" operator crd Print the WindmillInstance CRD YAML to stdout");
println!();
println!("Environment variables (name = default):");
println!(" DATABASE_URL = <required> The Postgres database url.");
@@ -607,6 +610,43 @@ async fn windmill_main() -> anyhow::Result<()> {
cache_hub_resource_types().await?;
return Ok(());
}
"sync-config" => {
tracing_subscriber::fmt::init();
let path = std::env::args().nth(2).unwrap_or_else(|| {
eprintln!("Usage: windmill sync-config <file>");
std::process::exit(1);
});
let contents = tokio::fs::read_to_string(&path)
.await
.with_context(|| format!("Could not read config file: {path}"))?;
let mut config: windmill_common::instance_config::InstanceConfig =
serde_yml::from_str(&contents)
.with_context(|| format!("Could not parse YAML from: {path}"))?;
windmill_common::instance_config::resolve_env_refs(&mut config.global_settings)
.map_err(|var| anyhow::anyhow!("environment variable '{var}' not found"))?;
tracing::info!("Connecting to database...");
let db = crate::db_connect::initial_connection().await?;
config.sync_to_db(&db).await?;
tracing::info!("Synced instance config from {path}");
return Ok(());
}
#[cfg(feature = "operator")]
"operator" => {
let sub_arg = std::env::args().nth(2).unwrap_or_default();
if sub_arg == "crd" {
windmill_operator::print_crd_yaml();
return Ok(());
}
tracing_subscriber::fmt::init();
tracing::info!("Starting Windmill Kubernetes operator...");
tracing::info!("Connecting to database...");
let db = crate::db_connect::initial_connection().await?;
tracing::info!("Database connected. Starting controller...");
windmill_operator::run(db).await?;
return Ok(());
}
_ => {}
}
@@ -1582,6 +1622,7 @@ async fn process_notify_event(
POWERSHELL_REPO_URL_SETTING => reload_powershell_repo_url_setting(conn).await,
POWERSHELL_REPO_PAT_SETTING => reload_powershell_repo_pat_setting(conn).await,
MAVEN_REPOS_SETTING => reload_maven_repos_setting(conn).await,
MAVEN_SETTINGS_XML_SETTING => reload_maven_settings_xml_setting(conn).await,
NO_DEFAULT_MAVEN_SETTING => reload_no_default_maven_setting(conn).await,
RUBY_REPOS_SETTING => reload_ruby_repos_setting(conn).await,
HUB_API_SECRET_SETTING => reload_hub_api_secret_setting(conn).await,

View File

@@ -88,10 +88,10 @@ use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload};
use windmill_worker::{
result_processor::handle_job_error, JobCompletedSender, JobIsolationLevel,
OtelTracingProxySettings, SameWorkerSender, BUNFIG_INSTALL_SCOPES, CARGO_REGISTRIES,
INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS,
NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NUGET_CONFIG,
OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT,
POWERSHELL_REPO_URL, UV_INDEX_STRATEGY,
INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR,
MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE,
NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UV_INDEX_STRATEGY,
};
#[cfg(feature = "parquet")]
@@ -329,6 +329,7 @@ pub async fn initial_load(
reload_powershell_repo_url_setting(&conn).await;
reload_powershell_repo_pat_setting(&conn).await;
reload_maven_repos_setting(&conn).await;
reload_maven_settings_xml_setting(&conn).await;
reload_no_default_maven_setting(&conn).await;
reload_ruby_repos_setting(&conn).await;
reload_cargo_registries_setting(&conn).await;
@@ -1336,6 +1337,39 @@ pub async fn reload_maven_repos_setting(conn: &Connection) {
.await;
}
pub async fn reload_maven_settings_xml_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
windmill_common::global_settings::MAVEN_SETTINGS_XML_SETTING,
"MAVEN_SETTINGS_XML",
MAVEN_SETTINGS_XML.clone(),
)
.await;
if !cfg!(feature = "enterprise") {
return;
}
let settings_xml = MAVEN_SETTINGS_XML.read().await.clone();
match settings_xml {
Some(ref content) if !content.trim().is_empty() => {
let m2_dir = format!("{JAVA_HOME_DIR}/.m2");
if let Err(e) = tokio::fs::create_dir_all(&m2_dir).await {
tracing::error!("Failed to create .m2 directory: {e:#}");
return;
}
let settings_path = format!("{m2_dir}/settings.xml");
if let Err(e) = tokio::fs::write(&settings_path, content).await {
tracing::error!("Failed to write Maven settings.xml: {e:#}");
}
}
_ => {
let settings_path = format!("{JAVA_HOME_DIR}/.m2/settings.xml");
let _ = tokio::fs::remove_file(&settings_path).await;
}
}
}
pub async fn reload_no_default_maven_setting(conn: &Connection) {
let value = load_value_from_global_settings_with_conn(
conn,

View File

@@ -0,0 +1,994 @@
/*!
* Integration tests for windmill-common instance_config module.
*
* Tests verify the DB-level operations:
* - `InstanceConfig::from_db()` reads global_settings + worker configs
* - `apply_settings_diff()` applies upserts and deletes to global_settings
* - `apply_configs_diff()` applies upserts and deletes to config table
* - Full roundtrip: write → read → modify → diff → apply → read → verify
*
* Note: the test DB is created from migrations which seed default settings
* and worker configs. Tests either clean up first or assert on specific keys
* rather than exact counts.
*/
use std::collections::BTreeMap;
use sqlx::{Pool, Postgres};
use windmill_common::instance_config::{
apply_configs_diff, apply_settings_diff, diff_global_settings, diff_worker_configs, ApplyMode,
ConfigsDiff, InstanceConfig, SettingsDiff,
};
// ========================================================================
// Helpers
// ========================================================================
async fn get_global_setting(db: &Pool<Postgres>, name: &str) -> Option<serde_json::Value> {
sqlx::query_as::<_, (serde_json::Value,)>("SELECT value FROM global_settings WHERE name = $1")
.bind(name)
.fetch_optional(db)
.await
.expect("query should succeed")
.map(|(v,)| v)
}
async fn get_config(db: &Pool<Postgres>, name: &str) -> Option<serde_json::Value> {
sqlx::query_as::<_, (serde_json::Value,)>("SELECT config FROM config WHERE name = $1")
.bind(name)
.fetch_optional(db)
.await
.expect("query should succeed")
.map(|(v,)| v)
}
async fn insert_global_setting(db: &Pool<Postgres>, name: &str, value: serde_json::Value) {
sqlx::query(
"INSERT INTO global_settings (name, value) VALUES ($1, $2) \
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value",
)
.bind(name)
.bind(&value)
.execute(db)
.await
.expect("insert should succeed");
}
async fn insert_config(db: &Pool<Postgres>, name: &str, config: serde_json::Value) {
sqlx::query(
"INSERT INTO config (name, config) VALUES ($1, $2) \
ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config",
)
.bind(name)
.bind(&config)
.execute(db)
.await
.expect("insert should succeed");
}
async fn count_global_settings(db: &Pool<Postgres>) -> i64 {
sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM global_settings")
.fetch_one(db)
.await
.expect("count query should succeed")
.0
}
/// Clear all migration-seeded data so tests start from a clean slate.
async fn clear_settings_and_configs(db: &Pool<Postgres>) {
sqlx::query("DELETE FROM global_settings")
.execute(db)
.await
.expect("clear global_settings should succeed");
sqlx::query("DELETE FROM config WHERE name LIKE 'worker__%'")
.execute(db)
.await
.expect("clear worker configs should succeed");
}
// ========================================================================
// InstanceConfig::from_db() tests
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_from_db_empty(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
let config = InstanceConfig::from_db(&db)
.await
.expect("from_db should succeed on empty DB");
assert!(config.global_settings.base_url.is_none());
assert!(config.global_settings.license_key.is_none());
assert!(config.worker_configs.is_empty());
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_with_typed_global_settings(db: Pool<Postgres>) {
insert_global_setting(&db, "base_url", serde_json::json!("https://windmill.test")).await;
insert_global_setting(&db, "retention_period_secs", serde_json::json!(86400)).await;
insert_global_setting(&db, "expose_metrics", serde_json::json!(true)).await;
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(
config.global_settings.base_url.as_deref(),
Some("https://windmill.test")
);
assert_eq!(config.global_settings.retention_period_secs, Some(86400));
assert_eq!(config.global_settings.expose_metrics, Some(true));
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_with_structured_settings(db: Pool<Postgres>) {
insert_global_setting(
&db,
"smtp_settings",
serde_json::json!({
"smtp_host": "mail.test.com",
"smtp_port": 587,
"smtp_tls_implicit": false
}),
)
.await;
insert_global_setting(
&db,
"otel",
serde_json::json!({
"metrics_enabled": true,
"logs_enabled": false,
"otel_exporter_otlp_endpoint": "http://otel:4317"
}),
)
.await;
let config = InstanceConfig::from_db(&db).await.unwrap();
let smtp = config.global_settings.smtp_settings.as_ref().unwrap();
assert_eq!(smtp.smtp_host.as_deref(), Some("mail.test.com"));
assert_eq!(smtp.smtp_port, Some(587));
assert_eq!(smtp.smtp_tls_implicit, Some(false));
let otel = config.global_settings.otel.as_ref().unwrap();
assert_eq!(otel.metrics_enabled, Some(true));
assert_eq!(otel.logs_enabled, Some(false));
assert_eq!(
otel.otel_exporter_otlp_endpoint.as_deref(),
Some("http://otel:4317")
);
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_unknown_settings_go_to_extra(db: Pool<Postgres>) {
insert_global_setting(&db, "future_setting_xyz", serde_json::json!({"nested": 42})).await;
insert_global_setting(&db, "another_unknown", serde_json::json!("hello")).await;
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(
config.global_settings.extra["future_setting_xyz"],
serde_json::json!({"nested": 42})
);
assert_eq!(
config.global_settings.extra["another_unknown"],
serde_json::json!("hello")
);
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_with_worker_configs(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
insert_config(
&db,
"worker__default",
serde_json::json!({"init_bash": "echo default"}),
)
.await;
insert_config(
&db,
"worker__gpu",
serde_json::json!({
"dedicated_worker": "ws:f/gpu_script",
"worker_tags": ["gpu", "cuda"]
}),
)
.await;
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(config.worker_configs.len(), 2);
assert_eq!(
config.worker_configs["default"].init_bash.as_deref(),
Some("echo default")
);
assert_eq!(
config.worker_configs["gpu"].dedicated_worker.as_deref(),
Some("ws:f/gpu_script")
);
assert_eq!(
config.worker_configs["gpu"].worker_tags.as_ref().unwrap(),
&["gpu", "cuda"]
);
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_ignores_non_worker_configs(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Insert a config without worker__ prefix — should not appear
insert_config(&db, "server_config", serde_json::json!({"important": true})).await;
insert_config(
&db,
"worker__actual",
serde_json::json!({"init_bash": "echo hi"}),
)
.await;
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(config.worker_configs.len(), 1);
assert!(config.worker_configs.contains_key("actual"));
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_worker_config_prefix_stripping(db: Pool<Postgres>) {
insert_config(
&db,
"worker__my_group_name",
serde_json::json!({"cache_clear": 5}),
)
.await;
let config = InstanceConfig::from_db(&db).await.unwrap();
assert!(
config.worker_configs.contains_key("my_group_name"),
"worker__ prefix should be stripped"
);
assert_eq!(config.worker_configs["my_group_name"].cache_clear, Some(5));
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_worker_config_unknown_fields_in_extra(db: Pool<Postgres>) {
insert_config(
&db,
"worker__test_extra",
serde_json::json!({
"init_bash": "echo hello",
"future_field": 999,
"another_future": {"nested": true}
}),
)
.await;
let config = InstanceConfig::from_db(&db).await.unwrap();
let wc = &config.worker_configs["test_extra"];
assert_eq!(wc.init_bash.as_deref(), Some("echo hello"));
assert_eq!(wc.extra["future_field"], serde_json::json!(999));
assert_eq!(
wc.extra["another_future"],
serde_json::json!({"nested": true})
);
}
// ========================================================================
// apply_settings_diff() tests
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_upserts_only(db: Pool<Postgres>) {
let diff = SettingsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert("key_a".to_string(), serde_json::json!("val_a"));
m.insert("key_b".to_string(), serde_json::json!(123));
m
},
deletes: vec![],
};
apply_settings_diff(&db, &diff).await.unwrap();
assert_eq!(
get_global_setting(&db, "key_a").await,
Some(serde_json::json!("val_a"))
);
assert_eq!(
get_global_setting(&db, "key_b").await,
Some(serde_json::json!(123))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_deletes_only(db: Pool<Postgres>) {
insert_global_setting(&db, "to_delete_1", serde_json::json!("bye")).await;
insert_global_setting(&db, "to_delete_2", serde_json::json!("gone")).await;
insert_global_setting(&db, "to_keep", serde_json::json!("stay")).await;
let diff = SettingsDiff {
upserts: BTreeMap::new(),
deletes: vec!["to_delete_1".to_string(), "to_delete_2".to_string()],
};
apply_settings_diff(&db, &diff).await.unwrap();
assert!(get_global_setting(&db, "to_delete_1").await.is_none());
assert!(get_global_setting(&db, "to_delete_2").await.is_none());
assert!(get_global_setting(&db, "to_keep").await.is_some());
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_upserts_and_deletes(db: Pool<Postgres>) {
insert_global_setting(&db, "old_key", serde_json::json!("old")).await;
let diff = SettingsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert("new_key".to_string(), serde_json::json!("new"));
m
},
deletes: vec!["old_key".to_string()],
};
apply_settings_diff(&db, &diff).await.unwrap();
assert!(get_global_setting(&db, "old_key").await.is_none());
assert_eq!(
get_global_setting(&db, "new_key").await,
Some(serde_json::json!("new"))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_empty_noop(db: Pool<Postgres>) {
insert_global_setting(&db, "preexisting", serde_json::json!("value")).await;
let diff = SettingsDiff::default();
apply_settings_diff(&db, &diff).await.unwrap();
assert_eq!(
get_global_setting(&db, "preexisting").await,
Some(serde_json::json!("value"))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_upsert_overwrites(db: Pool<Postgres>) {
insert_global_setting(&db, "overwrite_me", serde_json::json!("old_value")).await;
let diff = SettingsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert("overwrite_me".to_string(), serde_json::json!("new_value"));
m
},
deletes: vec![],
};
apply_settings_diff(&db, &diff).await.unwrap();
assert_eq!(
get_global_setting(&db, "overwrite_me").await,
Some(serde_json::json!("new_value"))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_complex_json(db: Pool<Postgres>) {
let complex_value = serde_json::json!({
"host": "smtp.example.com",
"port": 587,
"auth": {"user": "admin", "pass": "secret"},
"tags": [1, 2, 3],
"nested": {"deep": {"value": null}}
});
let diff = SettingsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert("complex_setting".to_string(), complex_value.clone());
m
},
deletes: vec![],
};
apply_settings_diff(&db, &diff).await.unwrap();
let stored = get_global_setting(&db, "complex_setting").await.unwrap();
assert_eq!(stored, complex_value);
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_settings_diff_delete_nonexistent_is_noop(db: Pool<Postgres>) {
let diff =
SettingsDiff { upserts: BTreeMap::new(), deletes: vec!["does_not_exist".to_string()] };
// Should not error
apply_settings_diff(&db, &diff).await.unwrap();
}
// ========================================================================
// apply_configs_diff() tests
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_apply_configs_diff_upserts_with_prefix(db: Pool<Postgres>) {
let diff = ConfigsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert(
"mygroup".to_string(),
serde_json::json!({"init_bash": "echo hi"}),
);
m
},
deletes: vec![],
};
apply_configs_diff(&db, &diff).await.unwrap();
let stored = get_config(&db, "worker__mygroup").await.unwrap();
assert_eq!(stored["init_bash"], "echo hi");
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_configs_diff_deletes_with_prefix(db: Pool<Postgres>) {
insert_config(&db, "worker__to_remove", serde_json::json!({"a": 1})).await;
let diff = ConfigsDiff { upserts: BTreeMap::new(), deletes: vec!["to_remove".to_string()] };
apply_configs_diff(&db, &diff).await.unwrap();
assert!(get_config(&db, "worker__to_remove").await.is_none());
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_configs_diff_upsert_overwrites(db: Pool<Postgres>) {
insert_config(&db, "worker__grp", serde_json::json!({"old": true})).await;
let diff = ConfigsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert("grp".to_string(), serde_json::json!({"new": true}));
m
},
deletes: vec![],
};
apply_configs_diff(&db, &diff).await.unwrap();
let stored = get_config(&db, "worker__grp").await.unwrap();
assert_eq!(stored, serde_json::json!({"new": true}));
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_configs_diff_empty_noop(db: Pool<Postgres>) {
insert_config(&db, "worker__keep", serde_json::json!({"keep": true})).await;
let diff = ConfigsDiff::default();
apply_configs_diff(&db, &diff).await.unwrap();
assert!(get_config(&db, "worker__keep").await.is_some());
}
#[sqlx::test(fixtures("base"))]
async fn test_apply_configs_diff_does_not_touch_non_worker_configs(db: Pool<Postgres>) {
insert_config(&db, "server_config", serde_json::json!({"x": 1})).await;
let diff = ConfigsDiff { upserts: BTreeMap::new(), deletes: vec!["server_config".to_string()] };
apply_configs_diff(&db, &diff).await.unwrap();
// The delete targets "worker__server_config", not "server_config"
assert!(
get_config(&db, "server_config").await.is_some(),
"Non-worker config should not be affected"
);
}
// ========================================================================
// Full roundtrip tests
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_roundtrip_write_read_modify_apply(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Step 1: Seed initial state
insert_global_setting(&db, "base_url", serde_json::json!("https://v1.test")).await;
insert_global_setting(&db, "retention_period_secs", serde_json::json!(3600)).await;
insert_config(
&db,
"worker__default",
serde_json::json!({"init_bash": "echo v1"}),
)
.await;
// Step 2: Read via from_db
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(
config.global_settings.base_url.as_deref(),
Some("https://v1.test")
);
assert_eq!(config.global_settings.retention_period_secs, Some(3600));
assert_eq!(config.worker_configs.len(), 1);
// Step 3: Modify — change base_url, add a new setting, remove retention
let mut desired_settings = config.global_settings.clone();
desired_settings.base_url = Some("https://v2.test".to_string());
desired_settings.expose_metrics = Some(true);
desired_settings.retention_period_secs = None;
let current_map = config.global_settings.to_settings_map();
let desired_map = desired_settings.to_settings_map();
// Step 4: Diff + apply (Merge mode — no deletes)
let diff = diff_global_settings(&current_map, &desired_map, ApplyMode::Merge);
assert!(diff.upserts.contains_key("base_url"));
assert!(diff.upserts.contains_key("expose_metrics"));
assert!(diff.deletes.is_empty());
apply_settings_diff(&db, &diff).await.unwrap();
// Step 5: Read back and verify
let config2 = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(
config2.global_settings.base_url.as_deref(),
Some("https://v2.test")
);
assert_eq!(config2.global_settings.expose_metrics, Some(true));
// retention_period_secs still in DB because Merge mode doesn't delete
assert_eq!(config2.global_settings.retention_period_secs, Some(3600));
}
#[sqlx::test(fixtures("base"))]
async fn test_roundtrip_replace_mode_deletes(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Seed
insert_global_setting(&db, "base_url", serde_json::json!("https://old.test")).await;
insert_global_setting(&db, "retention_period_secs", serde_json::json!(7200)).await;
insert_global_setting(&db, "expose_metrics", serde_json::json!(false)).await;
let config = InstanceConfig::from_db(&db).await.unwrap();
let current_map = config.global_settings.to_settings_map();
// Desired: only base_url — retention and expose_metrics should be deleted
let mut desired_map = BTreeMap::new();
desired_map.insert(
"base_url".to_string(),
serde_json::json!("https://old.test"),
);
let diff = diff_global_settings(&current_map, &desired_map, ApplyMode::Replace);
assert!(diff.upserts.is_empty());
assert!(diff.deletes.contains(&"retention_period_secs".to_string()));
assert!(diff.deletes.contains(&"expose_metrics".to_string()));
apply_settings_diff(&db, &diff).await.unwrap();
assert!(get_global_setting(&db, "retention_period_secs")
.await
.is_none());
assert!(get_global_setting(&db, "expose_metrics").await.is_none());
assert!(get_global_setting(&db, "base_url").await.is_some());
}
#[sqlx::test(fixtures("base"))]
async fn test_roundtrip_worker_configs(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Seed two worker configs
insert_config(
&db,
"worker__default",
serde_json::json!({"init_bash": "echo default"}),
)
.await;
insert_config(
&db,
"worker__legacy",
serde_json::json!({"init_bash": "echo legacy"}),
)
.await;
// Read
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(config.worker_configs.len(), 2);
// Desired: replace default, add gpu, remove legacy
let current_map: BTreeMap<String, serde_json::Value> = config
.worker_configs
.iter()
.map(|(k, v)| (k.clone(), serde_json::to_value(v).unwrap()))
.collect();
let mut desired_map = BTreeMap::new();
desired_map.insert(
"default".to_string(),
serde_json::json!({"init_bash": "echo default v2"}),
);
desired_map.insert(
"gpu".to_string(),
serde_json::json!({"dedicated_worker": "ws:f/gpu"}),
);
let diff = diff_worker_configs(&current_map, &desired_map, ApplyMode::Replace);
assert!(diff.upserts.contains_key("default")); // changed
assert!(diff.upserts.contains_key("gpu")); // new
assert_eq!(diff.deletes, vec!["legacy".to_string()]);
apply_configs_diff(&db, &diff).await.unwrap();
// Verify
let config2 = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(config2.worker_configs.len(), 2);
assert_eq!(
config2.worker_configs["default"].init_bash.as_deref(),
Some("echo default v2")
);
assert_eq!(
config2.worker_configs["gpu"].dedicated_worker.as_deref(),
Some("ws:f/gpu")
);
assert!(!config2.worker_configs.contains_key("legacy"));
}
#[sqlx::test(fixtures("base"))]
async fn test_roundtrip_to_settings_map_from_db_consistency(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Write a GlobalSettings via to_settings_map + apply, then read back via from_db
let original = windmill_common::instance_config::GlobalSettings {
base_url: Some("https://roundtrip.test".to_string()),
retention_period_secs: Some(43200),
expose_metrics: Some(true),
smtp_settings: Some(windmill_common::instance_config::SmtpSettings {
smtp_host: Some("smtp.roundtrip.test".to_string()),
smtp_port: Some(465),
..Default::default()
}),
custom_tags: Some(vec!["tag1".to_string(), "tag2".to_string()]),
..Default::default()
};
let map = original.to_settings_map();
let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![] };
apply_settings_diff(&db, &diff).await.unwrap();
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(config.global_settings.base_url, original.base_url);
assert_eq!(
config.global_settings.retention_period_secs,
original.retention_period_secs
);
assert_eq!(
config.global_settings.expose_metrics,
original.expose_metrics
);
assert_eq!(
config
.global_settings
.smtp_settings
.as_ref()
.unwrap()
.smtp_host,
original.smtp_settings.as_ref().unwrap().smtp_host
);
assert_eq!(
config
.global_settings
.smtp_settings
.as_ref()
.unwrap()
.smtp_port,
original.smtp_settings.as_ref().unwrap().smtp_port
);
assert_eq!(config.global_settings.custom_tags, original.custom_tags);
}
#[sqlx::test(fixtures("base"))]
async fn test_idempotent_apply(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
let diff = SettingsDiff {
upserts: {
let mut m = BTreeMap::new();
m.insert("idem_key".to_string(), serde_json::json!("idem_value"));
m
},
deletes: vec![],
};
// Apply twice
apply_settings_diff(&db, &diff).await.unwrap();
apply_settings_diff(&db, &diff).await.unwrap();
assert_eq!(
get_global_setting(&db, "idem_key").await,
Some(serde_json::json!("idem_value"))
);
assert_eq!(count_global_settings(&db).await, 1);
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_mixed_typed_and_extra(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Insert both typed and untyped settings
insert_global_setting(&db, "base_url", serde_json::json!("https://mixed.test")).await;
insert_global_setting(&db, "expose_metrics", serde_json::json!(true)).await;
insert_global_setting(
&db,
"unknown_future_setting",
serde_json::json!({"key": "val"}),
)
.await;
insert_global_setting(&db, "another_custom", serde_json::json!([1, 2, 3])).await;
let config = InstanceConfig::from_db(&db).await.unwrap();
// Typed fields
assert_eq!(
config.global_settings.base_url.as_deref(),
Some("https://mixed.test")
);
assert_eq!(config.global_settings.expose_metrics, Some(true));
// Extra fields
assert_eq!(
config.global_settings.extra["unknown_future_setting"],
serde_json::json!({"key": "val"})
);
assert_eq!(
config.global_settings.extra["another_custom"],
serde_json::json!([1, 2, 3])
);
// Roundtrip: to_settings_map should include everything
let map = config.global_settings.to_settings_map();
assert!(map.contains_key("base_url"));
assert!(map.contains_key("expose_metrics"));
assert!(map.contains_key("unknown_future_setting"));
assert!(map.contains_key("another_custom"));
assert_eq!(map.len(), 4);
}
#[sqlx::test(fixtures("base"))]
async fn test_full_config_roundtrip(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Seed a realistic configuration
insert_global_setting(
&db,
"base_url",
serde_json::json!("https://prod.windmill.dev"),
)
.await;
insert_global_setting(&db, "license_key", serde_json::json!("prod-license-key")).await;
insert_global_setting(&db, "retention_period_secs", serde_json::json!(2592000)).await;
insert_global_setting(
&db,
"smtp_settings",
serde_json::json!({
"smtp_host": "smtp.prod.com",
"smtp_port": 587,
"smtp_from": "noreply@prod.com",
"smtp_tls_implicit": true
}),
)
.await;
insert_global_setting(
&db,
"critical_error_channels",
serde_json::json!([
{"email": "admin@prod.com"},
{"slack_channel": "#prod-alerts"}
]),
)
.await;
insert_global_setting(
&db,
"otel",
serde_json::json!({
"metrics_enabled": true,
"tracing_enabled": true,
"otel_exporter_otlp_endpoint": "http://otel-collector:4317"
}),
)
.await;
insert_config(
&db,
"worker__default",
serde_json::json!({
"init_bash": "apt-get update",
"worker_tags": ["default", "deno", "bun"],
"cache_clear": 7
}),
)
.await;
insert_config(
&db,
"worker__gpu",
serde_json::json!({
"dedicated_worker": "ws:f/gpu_inference",
"autoscaling": {
"enabled": true,
"min_workers": 0,
"max_workers": 4,
"integration": {"type": "kubernetes"}
}
}),
)
.await;
// Read full config
let config = InstanceConfig::from_db(&db).await.unwrap();
assert_eq!(
config.global_settings.base_url.as_deref(),
Some("https://prod.windmill.dev")
);
assert_eq!(
config
.global_settings
.license_key
.as_ref()
.and_then(|v| v.as_literal()),
Some("prod-license-key")
);
assert_eq!(config.global_settings.retention_period_secs, Some(2592000));
let smtp = config.global_settings.smtp_settings.as_ref().unwrap();
assert_eq!(smtp.smtp_host.as_deref(), Some("smtp.prod.com"));
assert_eq!(smtp.smtp_from.as_deref(), Some("noreply@prod.com"));
let channels = config
.global_settings
.critical_error_channels
.as_ref()
.unwrap();
assert_eq!(channels.len(), 2);
let otel = config.global_settings.otel.as_ref().unwrap();
assert_eq!(otel.metrics_enabled, Some(true));
assert_eq!(otel.tracing_enabled, Some(true));
assert_eq!(config.worker_configs.len(), 2);
assert_eq!(config.worker_configs["default"].cache_clear, Some(7));
let gpu_auto = config.worker_configs["gpu"].autoscaling.as_ref().unwrap();
assert!(gpu_auto.enabled);
assert_eq!(gpu_auto.min_workers, Some(0));
assert_eq!(gpu_auto.max_workers, Some(4));
// Verify settings count matches
let settings_map = config.global_settings.to_settings_map();
let db_count = count_global_settings(&db).await;
assert_eq!(
settings_map.len() as i64,
db_count,
"to_settings_map should produce same count as DB rows"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_diff_apply_only_touches_changed_rows(db: Pool<Postgres>) {
clear_settings_and_configs(&db).await;
// Seed 3 settings
insert_global_setting(&db, "unchanged_1", serde_json::json!("val1")).await;
insert_global_setting(&db, "unchanged_2", serde_json::json!("val2")).await;
insert_global_setting(&db, "to_change", serde_json::json!("old")).await;
let mut current = BTreeMap::new();
current.insert("unchanged_1".to_string(), serde_json::json!("val1"));
current.insert("unchanged_2".to_string(), serde_json::json!("val2"));
current.insert("to_change".to_string(), serde_json::json!("old"));
let mut desired = current.clone();
desired.insert("to_change".to_string(), serde_json::json!("new"));
desired.insert("added".to_string(), serde_json::json!("fresh"));
let diff = diff_global_settings(&current, &desired, ApplyMode::Merge);
// Only "to_change" and "added" should be in upserts
assert_eq!(diff.upserts.len(), 2);
assert!(diff.upserts.contains_key("to_change"));
assert!(diff.upserts.contains_key("added"));
assert!(!diff.upserts.contains_key("unchanged_1"));
assert!(!diff.upserts.contains_key("unchanged_2"));
apply_settings_diff(&db, &diff).await.unwrap();
// Verify all 4 settings present
assert_eq!(count_global_settings(&db).await, 4);
assert_eq!(
get_global_setting(&db, "to_change").await,
Some(serde_json::json!("new"))
);
assert_eq!(
get_global_setting(&db, "added").await,
Some(serde_json::json!("fresh"))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_from_db_reads_migration_defaults(db: Pool<Postgres>) {
// Verify that from_db correctly reads the migration-seeded state
// without clearing — tests that pre-existing data is properly deserialized
let config = InstanceConfig::from_db(&db).await.unwrap();
// Migrations seed at least base_url, license_key, etc.
// Just verify from_db doesn't error and returns a populated struct
let map = config.global_settings.to_settings_map();
assert!(
!map.is_empty(),
"Migration-seeded DB should produce non-empty settings"
);
assert!(
!config.worker_configs.is_empty(),
"Migration-seeded DB should have worker configs"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_replace_mode_protects_settings_in_integration(db: Pool<Postgres>) {
// Seed a protected setting
insert_global_setting(
&db,
"ducklake_settings",
serde_json::json!({"ducklakes": {}}),
)
.await;
insert_global_setting(&db, "ducklake_user_pg_pwd", serde_json::json!("secret_pwd")).await;
insert_global_setting(
&db,
"custom_instance_pg_databases",
serde_json::json!({"databases": {}}),
)
.await;
insert_global_setting(&db, "normal_setting", serde_json::json!("will_be_deleted")).await;
// Read current state
let config = InstanceConfig::from_db(&db).await.unwrap();
let current_map = config.global_settings.to_settings_map();
// Desired: only keep_me — everything else should be deleted except protected
let mut desired_map = BTreeMap::new();
desired_map.insert("keep_me".to_string(), serde_json::json!("yes"));
let diff = diff_global_settings(&current_map, &desired_map, ApplyMode::Replace);
// Protected keys should NOT be in deletes
assert!(
!diff.deletes.contains(&"ducklake_settings".to_string()),
"ducklake_settings is protected"
);
assert!(
!diff.deletes.contains(&"ducklake_user_pg_pwd".to_string()),
"ducklake_user_pg_pwd is protected"
);
assert!(
!diff
.deletes
.contains(&"custom_instance_pg_databases".to_string()),
"custom_instance_pg_databases is protected"
);
// But normal_setting should be deleted
assert!(diff.deletes.contains(&"normal_setting".to_string()));
apply_settings_diff(&db, &diff).await.unwrap();
// Verify protected settings survived
assert!(get_global_setting(&db, "ducklake_settings").await.is_some());
assert!(get_global_setting(&db, "ducklake_user_pg_pwd")
.await
.is_some());
assert!(get_global_setting(&db, "custom_instance_pg_databases")
.await
.is_some());
// Normal setting is gone
assert!(get_global_setting(&db, "normal_setting").await.is_none());
// New setting is present
assert_eq!(
get_global_setting(&db, "keep_me").await,
Some(serde_json::json!("yes"))
);
}

View File

@@ -0,0 +1,448 @@
/*!
* Integration tests for windmill-operator db_sync module.
*
* Tests verify full declarative sync of global_settings and worker configs:
* - Upsert desired settings into DB
* - Delete settings present in DB but absent from desired state
* - Protect certain internal settings from deletion
*/
#[cfg(feature = "operator")]
mod tests {
use std::collections::BTreeMap;
use sqlx::{Pool, Postgres};
// ========================================================================
// Helpers
// ========================================================================
async fn get_global_setting(db: &Pool<Postgres>, name: &str) -> Option<serde_json::Value> {
sqlx::query_as::<_, (serde_json::Value,)>(
"SELECT value FROM global_settings WHERE name = $1",
)
.bind(name)
.fetch_optional(db)
.await
.expect("query should succeed")
.map(|(v,)| v)
}
async fn get_config(db: &Pool<Postgres>, name: &str) -> Option<serde_json::Value> {
sqlx::query_as::<_, (serde_json::Value,)>("SELECT config FROM config WHERE name = $1")
.bind(name)
.fetch_optional(db)
.await
.expect("query should succeed")
.map(|(v,)| v)
}
async fn insert_global_setting(db: &Pool<Postgres>, name: &str, value: serde_json::Value) {
sqlx::query(
"INSERT INTO global_settings (name, value) VALUES ($1, $2) \
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value",
)
.bind(name)
.bind(&value)
.execute(db)
.await
.expect("insert should succeed");
}
async fn insert_config(db: &Pool<Postgres>, name: &str, config: serde_json::Value) {
sqlx::query(
"INSERT INTO config (name, config) VALUES ($1, $2) \
ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config",
)
.bind(name)
.bind(&config)
.execute(db)
.await
.expect("insert should succeed");
}
// ========================================================================
// sync_global_settings tests
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_upserts(db: Pool<Postgres>) {
let mut desired = BTreeMap::new();
desired.insert(
"test_op_setting_a".to_string(),
serde_json::json!("value_a"),
);
desired.insert("test_op_setting_b".to_string(), serde_json::json!(42));
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
assert_eq!(
get_global_setting(&db, "test_op_setting_a").await,
Some(serde_json::json!("value_a"))
);
assert_eq!(
get_global_setting(&db, "test_op_setting_b").await,
Some(serde_json::json!(42))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_updates_existing(db: Pool<Postgres>) {
// Pre-populate a setting
insert_global_setting(&db, "test_op_existing", serde_json::json!("old")).await;
// Sync with new value
let mut desired = BTreeMap::new();
desired.insert("test_op_existing".to_string(), serde_json::json!("new"));
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
assert_eq!(
get_global_setting(&db, "test_op_existing").await,
Some(serde_json::json!("new"))
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_deletes_absent(db: Pool<Postgres>) {
// Pre-populate settings
insert_global_setting(&db, "test_op_keep", serde_json::json!("keep")).await;
insert_global_setting(&db, "test_op_remove", serde_json::json!("remove")).await;
// Sync with only one of them
let mut desired = BTreeMap::new();
desired.insert("test_op_keep".to_string(), serde_json::json!("keep"));
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
assert!(get_global_setting(&db, "test_op_keep").await.is_some());
assert!(
get_global_setting(&db, "test_op_remove").await.is_none(),
"Setting absent from desired should be deleted"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_protects_ducklake(db: Pool<Postgres>) {
// Pre-populate a protected setting
insert_global_setting(
&db,
"ducklake_settings",
serde_json::json!({"protected": true}),
)
.await;
// Sync with empty desired — protected key should survive
let desired = BTreeMap::new();
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
assert!(
get_global_setting(&db, "ducklake_settings").await.is_some(),
"Protected setting ducklake_settings should not be deleted"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_protects_all_protected_keys(db: Pool<Postgres>) {
let protected_keys = [
"ducklake_user_pg_pwd",
"ducklake_settings",
"custom_instance_pg_databases",
];
for key in &protected_keys {
insert_global_setting(&db, key, serde_json::json!("protected_value")).await;
}
// Sync with empty desired
let desired = BTreeMap::new();
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
for key in &protected_keys {
assert!(
get_global_setting(&db, key).await.is_some(),
"Protected key {key} should not be deleted"
);
}
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_empty_desired(db: Pool<Postgres>) {
// Pre-populate a non-protected setting
insert_global_setting(&db, "test_op_ephemeral", serde_json::json!("gone")).await;
let desired = BTreeMap::new();
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
assert!(
get_global_setting(&db, "test_op_ephemeral").await.is_none(),
"Non-protected settings should be deleted when desired is empty"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_global_settings_complex_json_values(db: Pool<Postgres>) {
let mut desired = BTreeMap::new();
desired.insert(
"test_op_complex".to_string(),
serde_json::json!({
"host": "smtp.example.com",
"port": 587,
"tls": true,
"nested": {"array": [1, 2, 3]}
}),
);
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("sync should succeed");
let stored = get_global_setting(&db, "test_op_complex")
.await
.expect("Setting should exist");
assert_eq!(stored["host"], "smtp.example.com");
assert_eq!(stored["port"], 587);
assert_eq!(stored["nested"]["array"][1], 2);
}
// ========================================================================
// sync_worker_configs tests
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_sync_worker_configs_upserts_with_prefix(db: Pool<Postgres>) {
let mut desired = BTreeMap::new();
desired.insert(
"test_group".to_string(),
serde_json::json!({"dedicated_worker": false}),
);
windmill_operator::db_sync::sync_worker_configs(&db, &desired)
.await
.expect("sync should succeed");
let config = get_config(&db, "worker__test_group")
.await
.expect("Config should exist with worker__ prefix");
assert_eq!(config["dedicated_worker"], false);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_worker_configs_updates_existing(db: Pool<Postgres>) {
// Pre-populate
insert_config(
&db,
"worker__test_wc_existing",
serde_json::json!({"old": true}),
)
.await;
let mut desired = BTreeMap::new();
desired.insert(
"test_wc_existing".to_string(),
serde_json::json!({"new": true}),
);
windmill_operator::db_sync::sync_worker_configs(&db, &desired)
.await
.expect("sync should succeed");
let config = get_config(&db, "worker__test_wc_existing")
.await
.expect("Config should exist");
assert_eq!(config["new"], true);
assert!(config.get("old").is_none());
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_worker_configs_deletes_absent(db: Pool<Postgres>) {
// Pre-populate two worker configs
insert_config(
&db,
"worker__test_wc_keep",
serde_json::json!({"keep": true}),
)
.await;
insert_config(
&db,
"worker__test_wc_remove",
serde_json::json!({"remove": true}),
)
.await;
// Sync with only one
let mut desired = BTreeMap::new();
desired.insert(
"test_wc_keep".to_string(),
serde_json::json!({"keep": true}),
);
windmill_operator::db_sync::sync_worker_configs(&db, &desired)
.await
.expect("sync should succeed");
assert!(get_config(&db, "worker__test_wc_keep").await.is_some());
assert!(
get_config(&db, "worker__test_wc_remove").await.is_none(),
"Worker config absent from desired should be deleted"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_worker_configs_does_not_touch_non_worker_configs(db: Pool<Postgres>) {
// Insert a non-worker config (no worker__ prefix)
insert_config(&db, "server_config", serde_json::json!({"important": true})).await;
// Sync with empty worker configs
let desired = BTreeMap::new();
windmill_operator::db_sync::sync_worker_configs(&db, &desired)
.await
.expect("sync should succeed");
assert!(
get_config(&db, "server_config").await.is_some(),
"Non-worker configs should not be touched"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_worker_configs_multiple_groups(db: Pool<Postgres>) {
let mut desired = BTreeMap::new();
desired.insert(
"default".to_string(),
serde_json::json!({"init_bash": "echo default"}),
);
desired.insert(
"gpu".to_string(),
serde_json::json!({"dedicated_worker": true}),
);
desired.insert(
"native".to_string(),
serde_json::json!({"init_bash": "echo native"}),
);
windmill_operator::db_sync::sync_worker_configs(&db, &desired)
.await
.expect("sync should succeed");
assert!(get_config(&db, "worker__default").await.is_some());
assert!(get_config(&db, "worker__gpu").await.is_some());
assert!(get_config(&db, "worker__native").await.is_some());
}
#[sqlx::test(fixtures("base"))]
async fn test_sync_worker_configs_empty_desired(db: Pool<Postgres>) {
insert_config(
&db,
"worker__test_wc_gone",
serde_json::json!({"ephemeral": true}),
)
.await;
let desired = BTreeMap::new();
windmill_operator::db_sync::sync_worker_configs(&db, &desired)
.await
.expect("sync should succeed");
assert!(
get_config(&db, "worker__test_wc_gone").await.is_none(),
"Worker config should be deleted when desired is empty"
);
}
// ========================================================================
// End-to-end: both syncs together
// ========================================================================
#[sqlx::test(fixtures("base"))]
async fn test_full_declarative_sync(db: Pool<Postgres>) {
// Pre-populate some existing state
insert_global_setting(&db, "test_op_stale_setting", serde_json::json!("stale")).await;
insert_config(
&db,
"worker__test_stale_group",
serde_json::json!({"stale": true}),
)
.await;
// Define desired state
let mut global_settings = BTreeMap::new();
global_settings.insert(
"test_op_base_url".to_string(),
serde_json::json!("https://windmill.example.com"),
);
global_settings.insert(
"test_op_license_key".to_string(),
serde_json::json!("my-license"),
);
let mut worker_configs = BTreeMap::new();
worker_configs.insert(
"default".to_string(),
serde_json::json!({"init_bash": "echo hello"}),
);
// Sync both
windmill_operator::db_sync::sync_global_settings(&db, &global_settings)
.await
.expect("global sync should succeed");
windmill_operator::db_sync::sync_worker_configs(&db, &worker_configs)
.await
.expect("worker sync should succeed");
// Verify desired state is present
assert_eq!(
get_global_setting(&db, "test_op_base_url").await,
Some(serde_json::json!("https://windmill.example.com"))
);
assert_eq!(
get_global_setting(&db, "test_op_license_key").await,
Some(serde_json::json!("my-license"))
);
assert!(get_config(&db, "worker__default").await.is_some());
// Verify stale state is removed
assert!(
get_global_setting(&db, "test_op_stale_setting")
.await
.is_none(),
"Stale global setting should be removed"
);
assert!(
get_config(&db, "worker__test_stale_group").await.is_none(),
"Stale worker config should be removed"
);
}
#[sqlx::test(fixtures("base"))]
async fn test_idempotent_sync(db: Pool<Postgres>) {
let mut desired = BTreeMap::new();
desired.insert("test_op_idempotent".to_string(), serde_json::json!("value"));
// Run sync twice — should be idempotent
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("first sync should succeed");
windmill_operator::db_sync::sync_global_settings(&db, &desired)
.await
.expect("second sync should succeed");
assert_eq!(
get_global_setting(&db, "test_op_idempotent").await,
Some(serde_json::json!("value"))
);
}
}

View File

@@ -23,7 +23,7 @@ use windmill_common::{
DB,
};
use windmill_api_auth::{ApiAuthed, require_devops_role};
use windmill_api_auth::{require_devops_role, ApiAuthed};
pub fn global_service() -> Router {
Router::new()
@@ -128,12 +128,16 @@ async fn update_config(
require_devops_role(&db, &authed.email).await?;
#[cfg(not(feature = "enterprise"))]
if name.starts_with("worker__") {
return Err(error::Error::BadRequest(
"Worker groups configurable from UI available only in the enterprise version"
.to_string(),
));
}
let config = if name.starts_with("worker__") {
// In CE, only allow setting worker_tags, cache_clear, and init_bash
serde_json::json!({
"worker_tags": config.get("worker_tags"),
"cache_clear": config.get("cache_clear"),
"init_bash": config.get("init_bash")
})
} else {
config
};
if name.starts_with("worker__") {
let periodic_script_bash = config

View File

@@ -19,6 +19,8 @@ mcp = []
windmill-test-utils.workspace = true
windmill-api-client.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-native-triggers = { workspace = true, features = ["native_trigger"] }
windmill-api-auth.workspace = true
sqlx.workspace = true
serde_json.workspace = true
serde.workspace = true

View File

@@ -0,0 +1,595 @@
/*!
* Integration tests for the native trigger system (Google).
*
* Tests cover 4 business-logic areas:
* 1. Resource path change — cleanup old path, recreate at new path
* 2. Config loading — workspace-level, instance-level, token update
* 3. Channel expiration renewal — should_renew_channel pure logic
* 4. Delete workspace integration — full cascade, cleanup preserves triggers, parse_stop_channel_params
*/
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_api_auth::ApiAuthed;
use windmill_common::variables::{build_crypt, encrypt};
use windmill_native_triggers::{
decrypt_oauth_data, delete_native_trigger, delete_workspace_integration,
get_workspace_integration,
google::{parse_stop_channel_params, should_renew_channel},
store_native_trigger, store_workspace_integration, NativeTriggerConfig, OAuthConfig,
ServiceName,
};
// ============================================================================
// Helpers
// ============================================================================
async fn insert_test_script(db: &Pool<Postgres>, path: &str) -> anyhow::Result<i64> {
let hash: i64 = rand::random::<i64>().unsigned_abs() as i64;
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, summary, description, content,
created_by, language, kind, lock)
VALUES ('test-workspace', $1, $2, '', '', 'def main(): pass',
'test-user', 'python3', 'script', '')",
)
.bind(hash)
.bind(path)
.execute(db)
.await?;
Ok(hash)
}
fn test_authed() -> ApiAuthed {
ApiAuthed {
email: "test@windmill.dev".to_string(),
username: "test-user".to_string(),
is_admin: true,
is_operator: false,
groups: vec!["all".to_string()],
folders: vec![],
scopes: None,
username_override: None,
token_prefix: None,
}
}
/// Set up a complete workspace integration with account+variable+resource.
/// Returns (resource_path, account_id).
async fn setup_oauth_integration(
db: &Pool<Postgres>,
service_name: ServiceName,
resource_path: &str,
access_token: &str,
refresh_token: &str,
oauth_data_override: Option<serde_json::Value>,
) -> anyhow::Result<i32> {
// 1. Create account with is_workspace_integration=true
let account_id: i32 = sqlx::query_scalar!(
"INSERT INTO account (workspace_id, client, expires_at, refresh_token, is_workspace_integration)
VALUES ('test-workspace', $1, now() + interval '1 hour', $2, true)
RETURNING id",
service_name.as_str(),
refresh_token,
)
.fetch_one(db)
.await?;
// 2. Encrypt and create variable
let mc = build_crypt(db, "test-workspace").await?;
let encrypted = encrypt(&mc, access_token);
sqlx::query!(
"INSERT INTO variable (workspace_id, path, value, is_secret, description, account, is_oauth)
VALUES ('test-workspace', $1, $2, true, 'test oauth token', $3, true)",
resource_path,
encrypted,
account_id,
)
.execute(db)
.await?;
// 3. Create resource
let resource_value = json!({ "token": format!("$var:{}", resource_path) });
sqlx::query!(
"INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by)
VALUES ('test-workspace', $1, $2, $3, '{}'::jsonb, 'test-user')",
resource_path,
resource_value,
service_name.resource_type(),
)
.execute(db)
.await?;
// 4. Store workspace integration with resource_path
let oauth_data = oauth_data_override.unwrap_or_else(|| {
json!({
"client_id": "test-client-id",
"client_secret": "test-client-secret",
"base_url": "https://example.com",
"resource_path": resource_path,
})
});
let authed = test_authed();
let mut tx = db.begin().await?;
store_workspace_integration(
&mut *tx,
&authed,
"test-workspace",
service_name,
oauth_data,
Some(resource_path),
)
.await?;
tx.commit().await?;
Ok(account_id)
}
fn now_ms() -> i64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis() as i64
}
// ============================================================================
// 1. Resource Path Change
// ============================================================================
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_resource_path_change(db: Pool<Postgres>) -> anyhow::Result<()> {
let path_a = "u/test-user/native_gworkspace";
setup_oauth_integration(
&db,
ServiceName::Google,
path_a,
"token-a",
"refresh-a",
None,
)
.await?;
// Verify decrypt works at path A
let config: OAuthConfig =
decrypt_oauth_data(&db, "test-workspace", ServiceName::Google).await?;
assert_eq!(config.access_token, "token-a");
// Cleanup old path
let mut tx = db.begin().await?;
windmill_native_triggers::workspace_integrations::cleanup_oauth_resource(
&mut *tx,
"test-workspace",
ServiceName::Google,
)
.await;
tx.commit().await?;
// Recreate at path B
let path_b = "u/test-user/native_gworkspace_v2";
setup_oauth_integration(
&db,
ServiceName::Google,
path_b,
"token-b",
"refresh-b",
None,
)
.await?;
// Path A resources should be gone
let var_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM variable WHERE workspace_id = 'test-workspace' AND path = $1",
path_a,
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(var_count, 0, "variable at old path should be deleted");
let res_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM resource WHERE workspace_id = 'test-workspace' AND path = $1",
path_a,
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(res_count, 0, "resource at old path should be deleted");
// Path B should work
let config: OAuthConfig =
decrypt_oauth_data(&db, "test-workspace", ServiceName::Google).await?;
assert_eq!(config.access_token, "token-b");
assert_eq!(config.refresh_token.as_deref(), Some("refresh-b"));
Ok(())
}
// ============================================================================
// 2. Config Loading — workspace vs instance + token update
// ============================================================================
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_decrypt_workspace_level(db: Pool<Postgres>) -> anyhow::Result<()> {
let resource_path = "u/test-user/native_gworkspace";
setup_oauth_integration(
&db,
ServiceName::Google,
resource_path,
"ws-access-token",
"ws-refresh-token",
None,
)
.await?;
let config: OAuthConfig =
decrypt_oauth_data(&db, "test-workspace", ServiceName::Google).await?;
assert_eq!(config.access_token, "ws-access-token");
assert_eq!(config.refresh_token.as_deref(), Some("ws-refresh-token"));
assert_eq!(config.client_id, "test-client-id");
assert_eq!(config.client_secret, "test-client-secret");
assert_eq!(config.base_url, "https://example.com");
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_decrypt_instance_level(db: Pool<Postgres>) -> anyhow::Result<()> {
// Insert instance-level credentials into global_settings
sqlx::query!(
"INSERT INTO global_settings (name, value) VALUES ('oauths', $1)
ON CONFLICT (name) DO UPDATE SET value = $1",
json!({
"gworkspace": {
"id": "instance-client-id",
"secret": "instance-client-secret"
}
}),
)
.execute(&db)
.await?;
let resource_path = "u/test-user/native_gworkspace";
let oauth_data = json!({
"instance_shared": true,
"base_url": "https://accounts.google.com",
"resource_path": resource_path,
});
setup_oauth_integration(
&db,
ServiceName::Google,
resource_path,
"inst-access-token",
"inst-refresh-token",
Some(oauth_data),
)
.await?;
let config: OAuthConfig =
decrypt_oauth_data(&db, "test-workspace", ServiceName::Google).await?;
assert_eq!(config.client_id, "instance-client-id");
assert_eq!(config.client_secret, "instance-client-secret");
assert_eq!(config.access_token, "inst-access-token");
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_token_update_persists(db: Pool<Postgres>) -> anyhow::Result<()> {
let resource_path = "u/test-user/native_gworkspace";
let account_id = setup_oauth_integration(
&db,
ServiceName::Google,
resource_path,
"old-access-token",
"old-refresh-token",
None,
)
.await?;
// Verify old tokens
let config: OAuthConfig =
decrypt_oauth_data(&db, "test-workspace", ServiceName::Google).await?;
assert_eq!(config.access_token, "old-access-token");
// Simulate token refresh: update variable + account
let mc = build_crypt(&db, "test-workspace").await?;
let new_encrypted = encrypt(&mc, "new-access-token");
sqlx::query!(
"UPDATE variable SET value = $1 WHERE workspace_id = 'test-workspace' AND path = $2",
new_encrypted,
resource_path,
)
.execute(&db)
.await?;
sqlx::query!(
"UPDATE account SET refresh_token = $1 WHERE workspace_id = 'test-workspace' AND id = $2",
"new-refresh-token",
account_id,
)
.execute(&db)
.await?;
// Verify new tokens
let config: OAuthConfig =
decrypt_oauth_data(&db, "test-workspace", ServiceName::Google).await?;
assert_eq!(config.access_token, "new-access-token");
assert_eq!(config.refresh_token.as_deref(), Some("new-refresh-token"));
Ok(())
}
// ============================================================================
// 3. Channel Expiration Renewal — should_renew_channel
// ============================================================================
#[test]
fn test_should_renew_drive_channel_expired() {
let config = json!({
"triggerType": "drive",
"expiration": (now_ms() - 1000).to_string(),
});
assert!(should_renew_channel(&config));
}
#[test]
fn test_should_renew_drive_channel_within_window() {
// 30 minutes remaining — within the 1-hour Drive renewal window
let config = json!({
"triggerType": "drive",
"expiration": (now_ms() + 30 * 60 * 1000).to_string(),
});
assert!(should_renew_channel(&config));
}
#[test]
fn test_should_renew_drive_channel_not_yet() {
// 2 hours remaining — outside the 1-hour Drive renewal window
let config = json!({
"triggerType": "drive",
"expiration": (now_ms() + 2 * 60 * 60 * 1000).to_string(),
});
assert!(!should_renew_channel(&config));
}
#[test]
fn test_should_renew_calendar_channel_within_window() {
// 12 hours remaining — within the 1-day Calendar renewal window
let config = json!({
"triggerType": "calendar",
"expiration": (now_ms() + 12 * 60 * 60 * 1000).to_string(),
});
assert!(should_renew_channel(&config));
}
#[test]
fn test_should_renew_calendar_channel_not_yet() {
// 2 days remaining — outside the 1-day Calendar renewal window
let config = json!({
"triggerType": "calendar",
"expiration": (now_ms() + 2 * 24 * 60 * 60 * 1000).to_string(),
});
assert!(!should_renew_channel(&config));
}
#[test]
fn test_should_renew_channel_zero_expiration() {
let config = json!({
"triggerType": "drive",
"expiration": "0",
});
assert!(!should_renew_channel(&config));
}
#[test]
fn test_should_renew_channel_missing_fields() {
assert!(!should_renew_channel(&json!({})));
}
// ============================================================================
// 4. Delete Workspace Integration
// ============================================================================
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_delete_integration_full_cascade(db: Pool<Postgres>) -> anyhow::Result<()> {
let resource_path = "u/test-user/native_gworkspace";
let account_id = setup_oauth_integration(
&db,
ServiceName::Google,
resource_path,
"token",
"refresh",
None,
)
.await?;
// Add a native trigger linked to this integration
insert_test_script(&db, "f/test/handler").await?;
let trigger_config = NativeTriggerConfig {
script_path: "f/test/handler".to_string(),
is_flow: false,
webhook_token: "abcdefghij1234567890".to_string(),
};
store_native_trigger(
&db,
"test-workspace",
ServiceName::Google,
"ext-1",
&trigger_config,
json!({"triggerType": "drive"}),
)
.await?;
// Step 1: Delete triggers
let deleted =
delete_native_trigger(&db, "test-workspace", ServiceName::Google, "ext-1").await?;
assert!(deleted);
// Step 2: Cleanup OAuth resources
let mut tx = db.begin().await?;
windmill_native_triggers::workspace_integrations::cleanup_oauth_resource(
&mut *tx,
"test-workspace",
ServiceName::Google,
)
.await;
tx.commit().await?;
// Step 3: Delete workspace integration
let mut tx = db.begin().await?;
let deleted =
delete_workspace_integration(&mut *tx, "test-workspace", ServiceName::Google).await?;
tx.commit().await?;
assert!(deleted);
// Verify everything is gone
let var_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM variable WHERE workspace_id = 'test-workspace' AND path = $1",
resource_path,
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(var_count, 0);
let acc_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM account WHERE workspace_id = 'test-workspace' AND id = $1",
account_id,
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(acc_count, 0);
let res_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM resource WHERE workspace_id = 'test-workspace' AND path = $1",
resource_path,
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(res_count, 0);
assert!(
get_workspace_integration(&db, "test-workspace", ServiceName::Google)
.await
.is_err()
);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_cleanup_preserves_triggers(db: Pool<Postgres>) -> anyhow::Result<()> {
let resource_path = "u/test-user/native_gworkspace";
setup_oauth_integration(
&db,
ServiceName::Google,
resource_path,
"token",
"refresh",
None,
)
.await?;
// Create a trigger
insert_test_script(&db, "f/test/handler").await?;
let trigger_config = NativeTriggerConfig {
script_path: "f/test/handler".to_string(),
is_flow: false,
webhook_token: "abcdefghij1234567890".to_string(),
};
store_native_trigger(
&db,
"test-workspace",
ServiceName::Google,
"ext-1",
&trigger_config,
json!({"triggerType": "drive"}),
)
.await?;
// Cleanup OAuth only — should NOT remove the trigger
let mut tx = db.begin().await?;
windmill_native_triggers::workspace_integrations::cleanup_oauth_resource(
&mut *tx,
"test-workspace",
ServiceName::Google,
)
.await;
tx.commit().await?;
// OAuth resources gone
let var_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM variable WHERE workspace_id = 'test-workspace' AND path = $1",
resource_path,
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(var_count, 0);
// Trigger still exists
let trigger_count: i64 = sqlx::query_scalar!(
"SELECT count(*) FROM native_trigger WHERE workspace_id = 'test-workspace' AND service_name = 'google'"
)
.fetch_one(&db)
.await?
.unwrap_or(0);
assert_eq!(trigger_count, 1, "trigger should survive OAuth cleanup");
Ok(())
}
// --- parse_stop_channel_params ---
#[test]
fn test_parse_stop_channel_params_drive() {
let config = json!({
"triggerType": "drive",
"googleResourceId": "res-123",
});
let (resource_id, url) = parse_stop_channel_params(&config);
assert_eq!(resource_id, "res-123");
assert!(
url.contains("googleapis.com/drive/v3/channels/stop"),
"url={}",
url
);
}
#[test]
fn test_parse_stop_channel_params_calendar() {
let config = json!({
"triggerType": "calendar",
"googleResourceId": "res-456",
});
let (resource_id, url) = parse_stop_channel_params(&config);
assert_eq!(resource_id, "res-456");
assert!(
url.contains("googleapis.com/calendar/v3/channels/stop"),
"url={}",
url
);
}
#[test]
fn test_parse_stop_channel_params_default() {
// Missing triggerType defaults to Drive
let config = json!({ "googleResourceId": "res-789" });
let (resource_id, url) = parse_stop_channel_params(&config);
assert_eq!(resource_id, "res-789");
assert!(url.contains("drive/v3/channels/stop"), "url={}", url);
}
#[test]
fn test_parse_stop_channel_params_missing_resource_id() {
let config = json!({ "triggerType": "drive" });
let (resource_id, _url) = parse_stop_channel_params(&config);
assert_eq!(resource_id, "");
}

View File

@@ -26,7 +26,6 @@ use axum::{
routing::{get, post},
Json, Router,
};
#[cfg(feature = "enterprise")]
use axum::extract::Query;
use serde_json::json;
@@ -45,6 +44,7 @@ use windmill_common::{
CRITICAL_ALERT_MUTE_UI_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
},
instance_config::{self, ApplyMode, InstanceConfig},
server::Smtp,
};
use windmill_common::{error::to_anyhow, PgDatabase};
@@ -58,6 +58,10 @@ pub fn global_service() -> Router {
post(set_global_setting).get(get_global_setting),
)
.route("/list_global", get(list_global_settings))
.route(
"/instance_config",
get(get_instance_config).put(set_instance_config),
)
.route("/test_smtp", post(test_email))
.route("/test_license_key", post(test_license_key))
.route("/send_stats", post(send_stats))
@@ -271,9 +275,40 @@ pub async fn set_global_setting_internal(
key: String,
value: serde_json::Value,
) -> error::Result<()> {
match key.as_str() {
run_setting_pre_write_hook(db, &key, &value).await?;
match value {
serde_json::Value::Null => {
delete_global_setting(db, &key).await?;
}
serde_json::Value::String(x) if x.is_empty() => {
delete_global_setting(db, &key).await?;
}
v => {
sqlx::query!(
"INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
key,
v
)
.execute(db)
.await?;
tracing::info!("Set global setting {} to {}", key, v);
}
};
Ok(())
}
/// Run side-effect hooks for specific settings before writing to DB.
/// Extracted from `set_global_setting_internal` for reuse by the bulk endpoint.
async fn run_setting_pre_write_hook(
db: &DB,
key: &str,
value: &serde_json::Value,
) -> error::Result<()> {
match key {
AUTOMATE_USERNAME_CREATION_SETTING => {
if value.clone().as_bool().unwrap_or(false) {
if value.as_bool().unwrap_or(false) {
generate_instance_username_for_all_users(db)
.await
.map_err(|err| {
@@ -285,14 +320,14 @@ pub async fn set_global_setting_internal(
}
}
CRITICAL_ALERT_MUTE_UI_SETTING => {
if value.clone().as_bool().unwrap_or(false) {
if value.as_bool().unwrap_or(false) {
sqlx::query!("UPDATE alerts SET acknowledged = true")
.execute(db)
.await?;
}
}
APP_WORKSPACED_ROUTE_SETTING => {
let serde_json::Value::Bool(workspaced_route) = &value else {
let serde_json::Value::Bool(workspaced_route) = value else {
return Err(error::Error::BadRequest(format!(
"{} setting Expected to be boolean",
APP_WORKSPACED_ROUTE_SETTING
@@ -312,15 +347,15 @@ pub async fn set_global_setting_internal(
SELECT
path,
custom_path
FROM
FROM
app
WHERE
WHERE
custom_path IN (
SELECT
SELECT
custom_path
FROM
FROM
app
GROUP
GROUP
BY custom_path
HAVING COUNT(*) > 1
)
@@ -356,25 +391,79 @@ pub async fn set_global_setting_internal(
}
_ => {}
}
Ok(())
}
match value {
serde_json::Value::Null => {
delete_global_setting(db, &key).await?;
// ---------------------------------------------------------------------------
// Bulk instance config endpoints
// ---------------------------------------------------------------------------
async fn get_instance_config(
Extension(db): Extension<DB>,
authed: ApiAuthed,
) -> JsonResult<InstanceConfig> {
require_super_admin(&db, &authed.email).await?;
let config = InstanceConfig::from_db(&db)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
Ok(Json(config))
}
async fn set_instance_config(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Json(desired): Json<InstanceConfig>,
) -> error::Result<()> {
require_super_admin(&db, &authed.email).await?;
let current = InstanceConfig::from_db(&db)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
let desired_map = desired.global_settings.to_settings_map();
if !desired_map.is_empty() {
let current_map = current.global_settings.to_settings_map();
let settings_diff =
instance_config::diff_global_settings(&current_map, &desired_map, ApplyMode::Merge);
for (key, value) in &settings_diff.upserts {
run_setting_pre_write_hook(&db, key, value).await?;
}
serde_json::Value::String(x) if x.is_empty() => {
delete_global_setting(db, &key).await?;
}
v => {
sqlx::query!(
"INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
key,
v
)
.execute(db)
.await?;
tracing::info!("Set global setting {} to {}", key, v);
}
};
instance_config::apply_settings_diff(&db, &settings_diff)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
}
if !desired.worker_configs.is_empty() {
let current_wc: std::collections::BTreeMap<String, serde_json::Value> = current
.worker_configs
.iter()
.map(|(k, v)| {
(
k.clone(),
serde_json::to_value(v)
.expect("WorkerGroupConfig serialization cannot fail"),
)
})
.collect();
let desired_wc: std::collections::BTreeMap<String, serde_json::Value> = desired
.worker_configs
.iter()
.map(|(k, v)| {
(
k.clone(),
serde_json::to_value(v)
.expect("WorkerGroupConfig serialization cannot fail"),
)
})
.collect();
let configs_diff =
instance_config::diff_worker_configs(&current_wc, &desired_wc, ApplyMode::Merge);
instance_config::apply_configs_diff(&db, &configs_diff)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
}
Ok(())
}

View File

@@ -75,6 +75,7 @@ pub fn workspaced_service() -> Router {
.route("/whoami", get(whoami))
.route("/leave", post(leave_workspace))
.route("/username_to_email/:username", get(username_to_email))
.route("/list_instance_emails", get(list_instance_emails))
}
pub fn global_service() -> Router {
@@ -420,6 +421,22 @@ async fn list_users_as_super_admin(
Ok(Json(rows))
}
async fn list_instance_emails(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<Vec<String>> {
if *CLOUD_HOSTED {
return Err(Error::BadRequest(
"This endpoint is not available on cloud hosted instances".to_string(),
));
}
require_admin(authed.is_admin, &authed.username)?;
let rows = sqlx::query_scalar!("SELECT email FROM password ORDER BY email LIMIT 1000")
.fetch_all(&db)
.await?;
Ok(Json(rows))
}
#[derive(Serialize, Deserialize)]
struct Progress {
progress: u64,
@@ -945,7 +962,7 @@ async fn join_workspace<'c>(
.await?
.map(|v| v.as_bool())
.flatten()
.unwrap_or(false);
.unwrap_or(true);
let username = if automate_username_creation {
if username.is_some() && username.unwrap().len() > 0 {

View File

@@ -25,8 +25,8 @@ use regex::Regex;
use hex;
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use strum::IntoEnumIterator;
use uuid::Uuid;
use strum::{IntoEnumIterator};
use windmill_audit::audit_oss::{audit_log, AuditAuthorable};
use windmill_audit::ActionKind;
use windmill_common::db::UserDB;
@@ -39,7 +39,9 @@ use windmill_common::workspaces::GitRepositorySettings;
#[cfg(feature = "enterprise")]
use windmill_common::workspaces::WorkspaceDeploymentUISettings;
use windmill_common::workspaces::{
check_user_against_rule, get_datatable_resource_from_db_unchecked, DataTable, DataTableCatalogResourceType, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings
check_user_against_rule, get_datatable_resource_from_db_unchecked, DataTable,
DataTableCatalogResourceType, ProtectionRuleKind, ProtectionRules, ProtectionRuleset,
RuleCheckResult, WorkspaceGitSyncSettings,
};
use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType};
use windmill_common::PgDatabase;
@@ -601,7 +603,10 @@ async fn get_settings(
tx.commit().await?;
let settings = not_found_if_none(settings, "workspace settings", &w_id)?;
let mut settings = not_found_if_none(settings, "workspace settings", &w_id)?;
if !authed.is_admin {
settings.slack_oauth_client_secret = None;
}
Ok(Json(settings))
}
@@ -2494,6 +2499,7 @@ struct UsedTriggers {
pub gcp_used: bool,
pub email_used: bool,
pub nextcloud_used: bool,
pub google_used: bool,
}
async fn get_used_triggers(
@@ -2515,7 +2521,8 @@ async fn get_used_triggers(
EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!",
EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS "gcp_used!",
EXISTS(SELECT 1 FROM email_trigger WHERE workspace_id = $1) AS "email_used!",
EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS "nextcloud_used!"
EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS "nextcloud_used!",
EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'google'::native_trigger_service) AS "google_used!"
"#,
w_id
)
@@ -2748,7 +2755,7 @@ async fn create_workspace(
.await?
.map(|v| v.as_bool())
.flatten()
.unwrap_or(false);
.unwrap_or(true);
let username = if automate_username_creation {
if nw.username.is_some() && nw.username.unwrap().len() > 0 {
@@ -3418,6 +3425,9 @@ async fn create_workspace_fork(
)));
}
#[cfg(not(feature = "enterprise"))]
_check_nb_of_workspaces(&db).await?;
if *DISABLE_WORKSPACE_FORK {
require_super_admin(&db, &authed.email).await?;
}
@@ -3804,7 +3814,7 @@ async fn add_user(
.await?
.map(|v| v.as_bool())
.flatten()
.unwrap_or(false);
.unwrap_or(true);
let username = if automate_username_creation {
if nu.username.is_some() && nu.username.unwrap().len() > 0 {
@@ -4366,9 +4376,13 @@ async fn list_protection_rules(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ProtectionRulesetResponse>> {
let rules =
(*windmill_common::workspaces::get_protection_rules(&w_id, &db).await?).clone();
Ok(Json(rules.into_iter().map(ProtectionRulesetResponse::from).collect()))
let rules = (*windmill_common::workspaces::get_protection_rules(&w_id, &db).await?).clone();
Ok(Json(
rules
.into_iter()
.map(ProtectionRulesetResponse::from)
.collect(),
))
}
/// Create a new protection rule

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.634.4
version: 1.636.0
title: Windmill API
contact:
@@ -1483,6 +1483,39 @@ paths:
items:
$ref: "#/components/schemas/GlobalSetting"
/settings/instance_config:
get:
summary: get full instance config (global settings + worker configs)
operationId: getInstanceConfig
tags:
- setting
responses:
"200":
description: full instance configuration
content:
application/json:
schema:
$ref: "#/components/schemas/InstanceConfig"
put:
summary: update instance config (bulk upsert, no deletes). Empty or missing global_settings/worker_configs are skipped.
operationId: setInstanceConfig
tags:
- setting
requestBody:
description: full instance configuration to apply
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/InstanceConfig"
responses:
"200":
description: instance config updated
content:
text/plain:
schema:
type: string
/min_keep_alive_version:
get:
summary: get minimum worker versions required to stay alive
@@ -2364,6 +2397,24 @@ paths:
items:
$ref: "#/components/schemas/GlobalUserInfo"
/w/{workspace}/users/list_instance_emails:
get:
summary: list all instance user emails (only on non-cloud instances, requires workspace admin)
operationId: listInstanceEmails
tags:
- user
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: list of instance user emails
content:
application/json:
schema:
type: array
items:
type: string
/w/{workspace}/workspaces/list_pending_invites:
get:
summary: list pending invites for a workspace
@@ -3625,6 +3676,8 @@ paths:
type: boolean
nextcloud_used:
type: boolean
google_used:
type: boolean
required:
- http_routes_used
- websocket_used
@@ -3636,6 +3689,7 @@ paths:
- sqs_used
- email_used
- nextcloud_used
- google_used
/w/{workspace}/users/list:
get:
summary: list users
@@ -12287,6 +12341,55 @@ paths:
schema:
type: string
/w/{workspace}/native_triggers/integrations/{service_name}/instance_sharing_available:
get:
summary: check if instance-level credential sharing is available for a service
operationId: checkInstanceSharingAvailable
tags:
- workspace_integration
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: service_name
in: path
required: true
schema:
$ref: "#/components/schemas/NativeServiceName"
responses:
"200":
description: whether instance sharing is available
content:
application/json:
schema:
type: boolean
/w/{workspace}/native_triggers/integrations/{service_name}/generate_instance_connect_url:
post:
summary: generate connect url using instance-level credentials
operationId: generateInstanceConnectUrl
tags:
- workspace_integration
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: service_name
in: path
required: true
schema:
$ref: "#/components/schemas/NativeServiceName"
requestBody:
description: redirect_uri
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/RedirectUri"
responses:
"200":
description: authorization URL using instance credentials
content:
application/json:
schema:
type: string
/w/{workspace}/native_triggers/integrations/{service_name}/delete:
delete:
summary: delete native trigger service
@@ -12308,7 +12411,7 @@ paths:
schema:
type: string
/w/{workspace}/native_triggers/integrations/{service_name}/callback/{code}/{state}:
/w/{workspace}/native_triggers/integrations/{service_name}/callback:
post:
summary: native trigger service oauth callback
operationId: nativeTriggerServiceCallback
@@ -12321,23 +12424,26 @@ paths:
required: true
schema:
$ref: "#/components/schemas/NativeServiceName"
- name: code
in: path
required: true
schema:
type: string
- name: state
in: path
required: true
schema:
type: string
requestBody:
description: redirect_uri
description: OAuth callback data
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/RedirectUri"
type: object
properties:
code:
type: string
state:
type: string
redirect_uri:
type: string
resource_path:
type: string
required:
- code
- state
- redirect_uri
responses:
"200":
description: native trigger service oauth completed
@@ -12577,6 +12683,91 @@ paths:
items:
$ref: "#/components/schemas/NextCloudEventType"
/w/{workspace}/native_triggers/google/calendars:
get:
summary: list Google Calendars for the authenticated user
operationId: listGoogleCalendars
tags:
- native_trigger
parameters:
- name: workspace
in: path
required: true
schema:
type: string
responses:
"200":
description: list of Google Calendars
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/GoogleCalendarEntry"
/w/{workspace}/native_triggers/google/drive/files:
get:
summary: list or search Google Drive files
operationId: listGoogleDriveFiles
tags:
- native_trigger
parameters:
- name: workspace
in: path
required: true
schema:
type: string
- name: q
in: query
description: search query to filter files by name
schema:
type: string
- name: parent_id
in: query
description: folder ID to list children of
schema:
type: string
- name: page_token
in: query
description: token for next page of results
schema:
type: string
- name: shared_with_me
in: query
description: if true, list files shared with the user
schema:
type: boolean
default: false
responses:
"200":
description: list of Google Drive files
content:
application/json:
schema:
$ref: "#/components/schemas/GoogleDriveFilesResponse"
/w/{workspace}/native_triggers/google/drive/shared_drives:
get:
summary: list shared drives accessible to the user
operationId: listGoogleSharedDrives
tags:
- native_trigger
parameters:
- name: workspace
in: path
required: true
schema:
type: string
responses:
"200":
description: list of shared drives
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/SharedDriveEntry"
/native_triggers/{service_name}/w/{workspace_id}/webhook/{internal_id}:
post:
summary: receive webhook from external native trigger service
@@ -19697,6 +19888,7 @@ components:
description: True if script_path points to a flow, false if it points to a script
args:
$ref: "#/components/schemas/ScriptArgs"
nullable: true
extra_perms:
type: object
additionalProperties:
@@ -19707,57 +19899,74 @@ components:
description: Email of the user who owns this schedule, used for permissioned_as
error:
type: string
nullable: true
description: Last error message if the schedule failed to trigger
on_failure:
type: string
nullable: true
description: Path to a script or flow to run when the scheduled job fails
on_failure_times:
type: number
nullable: true
description: Number of consecutive failures before the on_failure handler is triggered (default 1)
on_failure_exact:
type: boolean
nullable: true
description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N
on_failure_extra_args:
$ref: "#/components/schemas/ScriptArgs"
nullable: true
on_recovery:
type: string
nullable: true
description: Path to a script or flow to run when the schedule recovers after failures
on_recovery_times:
type: number
nullable: true
description: Number of consecutive successes before the on_recovery handler is triggered (default 1)
on_recovery_extra_args:
$ref: "#/components/schemas/ScriptArgs"
nullable: true
on_success:
type: string
nullable: true
description: Path to a script or flow to run after each successful execution
on_success_extra_args:
$ref: "#/components/schemas/ScriptArgs"
nullable: true
ws_error_handler_muted:
type: boolean
description: If true, the workspace-level error handler will not be triggered for this schedule's failures
retry:
$ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
nullable: true
summary:
type: string
nullable: true
description: Short summary describing the purpose of this schedule
description:
type: string
nullable: true
description: Detailed description of what this schedule does
no_flow_overlap:
type: boolean
description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs)
tag:
type: string
nullable: true
description: Worker tag to route jobs to specific worker groups
paused_until:
type: string
format: date-time
nullable: true
description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time
cron_version:
type: string
nullable: true
description: Cron parser version. Use 'v2' for extended syntax with additional features
dynamic_skip:
type: string
nullable: true
description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false)
required:
- path
@@ -19819,60 +20028,77 @@ components:
type: boolean
description: True if script_path points to a flow, false if it points to a script
args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
enabled:
type: boolean
description: Whether the schedule is currently active and will trigger jobs
on_failure:
type: string
nullable: true
description: Path to a script or flow to run when the scheduled job fails
on_failure_times:
type: number
nullable: true
description: Number of consecutive failures before the on_failure handler is triggered (default 1)
on_failure_exact:
type: boolean
nullable: true
description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N
on_failure_extra_args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
on_recovery:
type: string
nullable: true
description: Path to a script or flow to run when the schedule recovers after failures
on_recovery_times:
type: number
nullable: true
description: Number of consecutive successes before the on_recovery handler is triggered (default 1)
on_recovery_extra_args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
on_success:
type: string
nullable: true
description: Path to a script or flow to run after each successful execution
on_success_extra_args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
ws_error_handler_muted:
type: boolean
description: If true, the workspace-level error handler will not be triggered for this schedule's failures
retry:
nullable: true
$ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
no_flow_overlap:
type: boolean
description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs)
summary:
type: string
nullable: true
description: Short summary describing the purpose of this schedule
description:
type: string
nullable: true
description: Detailed description of what this schedule does
tag:
type: string
nullable: true
description: Worker tag to route jobs to specific worker groups
paused_until:
type: string
nullable: true
format: date-time
description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time
cron_version:
type: string
nullable: true
description: Cron parser version. Use 'v2' for extended syntax with additional features
dynamic_skip:
type: string
nullable: true
description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false)
required:
- path
@@ -19892,57 +20118,74 @@ components:
type: string
description: IANA timezone for the schedule (e.g., 'UTC', 'Europe/Paris', 'America/New_York')
args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
on_failure:
type: string
nullable: true
description: Path to a script or flow to run when the scheduled job fails
on_failure_times:
type: number
nullable: true
description: Number of consecutive failures before the on_failure handler is triggered (default 1)
on_failure_exact:
type: boolean
nullable: true
description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N
on_failure_extra_args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
on_recovery:
type: string
nullable: true
description: Path to a script or flow to run when the schedule recovers after failures
on_recovery_times:
type: number
nullable: true
description: Number of consecutive successes before the on_recovery handler is triggered (default 1)
on_recovery_extra_args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
on_success:
type: string
nullable: true
description: Path to a script or flow to run after each successful execution
on_success_extra_args:
nullable: true
$ref: "#/components/schemas/ScriptArgs"
ws_error_handler_muted:
type: boolean
description: If true, the workspace-level error handler will not be triggered for this schedule's failures
retry:
nullable: true
$ref: "../../openflow.openapi.yaml#/components/schemas/Retry"
no_flow_overlap:
type: boolean
description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs)
summary:
type: string
nullable: true
description: Short summary describing the purpose of this schedule
description:
type: string
nullable: true
description: Detailed description of what this schedule does
tag:
type: string
nullable: true
description: Worker tag to route jobs to specific worker groups
paused_until:
type: string
nullable: true
format: date-time
description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time
cron_version:
type: string
nullable: true
description: Cron parser version. Use 'v2' for extended syntax with additional features
dynamic_skip:
type: string
nullable: true
description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false)
required:
- schedule
@@ -19965,6 +20208,7 @@ components:
- mqtt
- sqs
- gcp
- google
TriggerMode:
description: job trigger mode
@@ -20154,6 +20398,7 @@ components:
description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /.
static_asset_config:
type: object
nullable: true
description: Configuration for serving static assets (s3 bucket, storage path, filename)
properties:
s3:
@@ -20172,12 +20417,15 @@ components:
description: HTTP method (get, post, put, delete, patch) that triggers this endpoint
authentication_resource_path:
type: string
nullable: true
description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods)
summary:
type: string
nullable: true
description: Short summary describing the purpose of this trigger
description:
type: string
nullable: true
description: Detailed description of what this trigger does
request_type:
$ref: "#/components/schemas/HttpRequestType"
@@ -20234,12 +20482,15 @@ components:
description: If true, the route includes the workspace ID in the path
summary:
type: string
nullable: true
description: Short summary describing the purpose of this trigger
description:
type: string
nullable: true
description: Detailed description of what this trigger does
static_asset_config:
type: object
nullable: true
description: Configuration for serving static assets (s3 bucket, storage path, filename)
properties:
s3:
@@ -20261,6 +20512,7 @@ components:
description: HTTP method (get, post, put, delete, patch) that triggers this endpoint
authentication_resource_path:
type: string
nullable: true
description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods)
is_async:
type: boolean
@@ -20315,15 +20567,18 @@ components:
description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /.
summary:
type: string
nullable: true
description: Short summary describing the purpose of this trigger
description:
type: string
nullable: true
description: Detailed description of what this trigger does
workspaced_route:
type: boolean
description: If true, the route includes the workspace ID in the path
static_asset_config:
type: object
nullable: true
description: Configuration for serving static assets (s3 bucket, storage path, filename)
properties:
s3:
@@ -20339,6 +20594,7 @@ components:
- s3
authentication_resource_path:
type: string
nullable: true
description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods)
is_flow:
type: boolean
@@ -20416,6 +20672,8 @@ components:
type: number
nextcloud_count:
type: number
google_count:
type: number
WebsocketTrigger:
allOf:
@@ -20449,12 +20707,14 @@ components:
- value
initial_messages:
type: array
nullable: true
description: Messages to send immediately after connecting (can be raw strings or computed by runnables)
items:
$ref: "#/components/schemas/WebsocketTriggerInitialMessage"
url_runnable_args:
description: Arguments to pass to the script/flow that computes the WebSocket URL
$ref: "#/components/schemas/ScriptArgs"
nullable: true
can_return_message:
type: boolean
description: If true, the script can return a message to send back through the WebSocket
@@ -20508,11 +20768,13 @@ components:
- value
initial_messages:
type: array
nullable: true
description: Messages to send immediately after connecting (can be raw strings or computed by runnables)
items:
$ref: "#/components/schemas/WebsocketTriggerInitialMessage"
url_runnable_args:
description: Arguments to pass to the script/flow that computes the WebSocket URL
nullable: true
$ref: "#/components/schemas/ScriptArgs"
can_return_message:
type: boolean
@@ -20568,11 +20830,13 @@ components:
- value
initial_messages:
type: array
nullable: true
description: Messages to send immediately after connecting (can be raw strings or computed by runnables)
items:
$ref: "#/components/schemas/WebsocketTriggerInitialMessage"
url_runnable_args:
description: Arguments to pass to the script/flow that computes the WebSocket URL
nullable: true
$ref: "#/components/schemas/ScriptArgs"
can_return_message:
type: boolean
@@ -20674,15 +20938,19 @@ components:
description: Array of MQTT topics to subscribe to, each with topic name and QoS level
v3_config:
$ref: "#/components/schemas/MqttV3Config"
nullable: true
description: MQTT v3 specific configuration (clean_session)
v5_config:
$ref: "#/components/schemas/MqttV5Config"
nullable: true
description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval)
client_id:
type: string
nullable: true
description: MQTT client ID for this connection
client_version:
$ref: "#/components/schemas/MqttClientVersion"
nullable: true
description: MQTT protocol version ('v3' or 'v5')
server_id:
type: string
@@ -20720,14 +20988,18 @@ components:
description: Array of MQTT topics to subscribe to, each with topic name and QoS level
client_id:
type: string
nullable: true
description: MQTT client ID for this connection
v3_config:
nullable: true
$ref: "#/components/schemas/MqttV3Config"
description: MQTT v3 specific configuration (clean_session)
v5_config:
nullable: true
$ref: "#/components/schemas/MqttV5Config"
description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval)
client_version:
nullable: true
$ref: "#/components/schemas/MqttClientVersion"
description: MQTT protocol version ('v3' or 'v5')
path:
@@ -20770,14 +21042,18 @@ components:
description: Array of MQTT topics to subscribe to, each with topic name and QoS level
client_id:
type: string
nullable: true
description: MQTT client ID for this connection
v3_config:
nullable: true
$ref: "#/components/schemas/MqttV3Config"
description: MQTT v3 specific configuration (clean_session)
v5_config:
nullable: true
$ref: "#/components/schemas/MqttV5Config"
description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval)
client_version:
nullable: true
$ref: "#/components/schemas/MqttClientVersion"
description: MQTT protocol version ('v3' or 'v5')
path:
@@ -20851,6 +21127,7 @@ components:
$ref: "#/components/schemas/DeliveryType"
delivery_config:
$ref: "#/components/schemas/PushConfig"
nullable: true
subscription_mode:
$ref: "#/components/schemas/SubscriptionMode"
last_server_ping:
@@ -20904,6 +21181,7 @@ components:
delivery_type:
$ref: "#/components/schemas/DeliveryType"
delivery_config:
nullable: true
$ref: "#/components/schemas/PushConfig"
path:
type: string
@@ -20980,6 +21258,7 @@ components:
description: Path to the AWS resource containing credentials or OIDC configuration
message_attributes:
type: array
nullable: true
items:
type: string
description: Array of SQS message attribute names to include with each message
@@ -21071,6 +21350,7 @@ components:
description: Path to the AWS resource containing credentials or OIDC configuration
message_attributes:
type: array
nullable: true
items:
type: string
description: Array of SQS message attribute names to include with each message
@@ -21116,6 +21396,7 @@ components:
description: Path to the AWS resource containing credentials or OIDC configuration
message_attributes:
type: array
nullable: true
items:
type: string
description: Array of SQS message attribute names to include with each message
@@ -21522,9 +21803,11 @@ components:
description: If true, uses NATS JetStream for durable message delivery
stream_name:
type: string
nullable: true
description: JetStream stream name (required when use_jetstream is true)
consumer_name:
type: string
nullable: true
description: JetStream consumer name (required when use_jetstream is true)
subjects:
type: array
@@ -21576,9 +21859,11 @@ components:
description: If true, uses NATS JetStream for durable message delivery
stream_name:
type: string
nullable: true
description: JetStream stream name (required when use_jetstream is true)
consumer_name:
type: string
nullable: true
description: JetStream consumer name (required when use_jetstream is true)
subjects:
type: array
@@ -21616,9 +21901,11 @@ components:
description: If true, uses NATS JetStream for durable message delivery
stream_name:
type: string
nullable: true
description: JetStream stream name (required when use_jetstream is true)
consumer_name:
type: string
nullable: true
description: JetStream consumer name (required when use_jetstream is true)
subjects:
type: array
@@ -22873,6 +23160,26 @@ components:
- name
- value
InstanceConfig:
type: object
description: Unified instance configuration combining global settings and worker group configs
properties:
global_settings:
type: object
description: >
Global settings keyed by setting name. Known fields include base_url,
license_key, retention_period_secs, smtp_settings, otel, etc.
Unknown fields are preserved as-is.
additionalProperties: true
worker_configs:
type: object
description: >
Worker group configurations keyed by group name (e.g. "default", "gpu").
Each value contains worker_tags, init_bash, autoscaling, etc.
additionalProperties:
type: object
additionalProperties: true
Config:
type: object
properties:
@@ -23367,6 +23674,7 @@ components:
type: string
enum:
- nextcloud
- google
NativeTrigger:
type: object
@@ -23449,6 +23757,10 @@ components:
oauth_data:
nullable: true
$ref: "#/components/schemas/WorkspaceOAuthConfig"
resource_path:
type: string
nullable: true
description: Path to the resource storing the OAuth token
required:
- service_name
@@ -23570,3 +23882,57 @@ components:
- id
- name
- path
GoogleCalendarEntry:
type: object
properties:
id:
type: string
summary:
type: string
primary:
type: boolean
default: false
required:
- id
- summary
GoogleDriveFile:
type: object
properties:
id:
type: string
name:
type: string
mime_type:
type: string
is_folder:
type: boolean
default: false
required:
- id
- name
- mime_type
GoogleDriveFilesResponse:
type: object
properties:
files:
type: array
items:
$ref: "#/components/schemas/GoogleDriveFile"
next_page_token:
type: string
required:
- files
SharedDriveEntry:
type: object
properties:
id:
type: string
name:
type: string
required:
- id
- name

View File

@@ -3413,29 +3413,43 @@ pub async fn get_args_and_trigger_metadata(
// Build trigger metadata if this is a native trigger request
#[cfg(feature = "native_trigger")]
let trigger_metadata = if let Some(service_name_str) = &run_query.service_name {
use crate::native_triggers::ServiceName;
let (trigger_metadata, native_args) = if let Some(service_name_str) = &run_query.service_name {
use crate::native_triggers::{prepare_native_trigger_args, ServiceName};
let service_name = ServiceName::try_from(service_name_str.to_owned())?;
Some(TriggerMetadata::new(
let metadata = Some(TriggerMetadata::new(
run_query.trigger_external_id.clone(),
service_name.as_job_trigger_kind(),
))
));
let body = match &args.body {
crate::args::RawBody::Json(s) => s.clone(),
crate::args::RawBody::Text(s) => s.clone(),
_ => String::new(),
};
let native =
prepare_native_trigger_args(service_name, db, w_id, &args.metadata.headers, body)
.await?;
(metadata, native)
} else {
None
(None, None)
};
#[cfg(not(feature = "native_trigger"))]
let trigger_metadata: Option<TriggerMetadata> = None;
#[cfg(not(feature = "native_trigger"))]
let native_args: Option<windmill_queue::PushArgsOwned> = None;
let args = args
.to_args_from_runnable(
let args = if let Some(prepared) = native_args {
prepared
} else {
args.to_args_from_runnable(
&authed,
&db,
&w_id,
runnable_id,
run_query.skip_preprocessor,
)
.await?;
.await?
};
Ok((args, trigger_metadata))
}

View File

@@ -136,6 +136,7 @@ pub struct TriggersCount {
sqs_count: i64,
gcp_count: i64,
nextcloud_count: i64,
google_count: i64,
}
pub async fn get_triggers_count_internal(
@@ -305,6 +306,16 @@ pub async fn get_triggers_count_internal(
.await?
.unwrap_or(0);
let google_count = sqlx::query_scalar!(
"SELECT COUNT(*) FROM native_trigger WHERE workspace_id = $1 AND script_path = $2 AND is_flow = $3 AND service_name = 'google'",
w_id,
path,
is_flow,
)
.fetch_one(db)
.await?
.unwrap_or(0);
Ok(axum::Json(TriggersCount {
primary_schedule: primary_schedule
.map(|s| windmill_trigger::handler::TriggerPrimarySchedule { schedule: s }),
@@ -321,5 +332,6 @@ pub async fn get_triggers_count_internal(
gcp_count,
sqs_count,
nextcloud_count,
google_count,
}))
}

View File

@@ -335,6 +335,15 @@ async fn update_username_in_workpsace<'c>(
).execute(&mut **tx)
.await?;
sqlx::query!(
r#"UPDATE workspace_integrations SET resource_path = REGEXP_REPLACE(resource_path, 'u/' || $2 || '/(.*)', 'u/' || $1 || '/\1') WHERE resource_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#,
new_username,
old_username,
w_id
)
.execute(&mut **tx)
.await?;
sqlx::query!(
"UPDATE variable SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3",
new_username,

View File

@@ -7,6 +7,7 @@ edition.workspace = true
[features]
default = []
enterprise = ["dep:aws-config"]
instance_config_schema = ["dep:schemars"]
local_reports = ["dep:rsa", "dep:aes-gcm"]
private = ["dep:aws-sdk-rds"]
jemalloc = ["dep:tikv-jemalloc-ctl"]
@@ -96,6 +97,7 @@ windmill-parser.workspace = true
jsonwebtoken.workspace = true
backon.workspace = true
openidconnect = { workspace = true, optional = true }
schemars = { workspace = true, optional = true }
strum.workspace = true
strum_macros.workspace = true
windmill-types.workspace = true

View File

@@ -14,6 +14,7 @@ pub const NUGET_CONFIG_SETTING: &str = "nuget_config";
pub const POWERSHELL_REPO_URL_SETTING: &str = "powershell_repo_url";
pub const POWERSHELL_REPO_PAT_SETTING: &str = "powershell_repo_pat";
pub const MAVEN_REPOS_SETTING: &str = "maven_repos";
pub const MAVEN_SETTINGS_XML_SETTING: &str = "maven_settings_xml";
pub const NO_DEFAULT_MAVEN_SETTING: &str = "no_default_maven";
pub const RUBY_REPOS_SETTING: &str = "ruby_repos";
pub const CARGO_REGISTRIES_SETTING: &str = "cargo_registries";
@@ -144,6 +145,68 @@ pub async fn load_value_from_global_settings(
Ok(r)
}
/// Read OAuth client_id and client_secret from instance-level global settings.
/// `oauth_key` is the key under `oauths` (e.g., "gworkspace", "nextcloud").
pub async fn get_instance_oauth_credentials(
db: &Pool<Postgres>,
oauth_key: &str,
) -> error::Result<(String, String)> {
let oauths_value = load_value_from_global_settings(db, OAUTH_SETTING)
.await?
.ok_or_else(|| {
error::Error::InternalErr("Instance OAuth settings not found".to_string())
})?;
let entry = oauths_value.get(oauth_key).ok_or_else(|| {
error::Error::InternalErr(format!("No {} entry in instance OAuth settings", oauth_key))
})?;
let id = entry
.get("id")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let secret = entry
.get("secret")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
if id.is_empty() || secret.is_empty() {
return Err(error::Error::InternalErr(format!(
"Instance OAuth credentials for {} are incomplete",
oauth_key
)));
}
Ok((id, secret))
}
/// Map service client name to the OAuth settings key in global_settings.
/// e.g. "google" -> "gworkspace", "nextcloud" -> "nextcloud"
pub fn workspace_integration_oauth_key(client_name: &str) -> &str {
match client_name {
"google" => "gworkspace",
other => other,
}
}
/// Resolve the token endpoint URL for a workspace integration service.
pub fn workspace_integration_token_endpoint(client_name: &str, base_url: &str) -> String {
match client_name {
"google" => "https://oauth2.googleapis.com/token".to_string(),
_ => format!("{}/apps/oauth2/api/v1/token", base_url),
}
}
/// Resolve the auth endpoint URL for a workspace integration service.
pub fn workspace_integration_auth_endpoint(client_name: &str, base_url: &str) -> String {
match client_name {
"google" => "https://accounts.google.com/o/oauth2/v2/auth".to_string(),
_ => format!("{}/apps/oauth2/authorize", base_url),
}
}
pub async fn set_value_in_global_settings(
db: &Pool<Postgres>,
setting_name: &str,

File diff suppressed because it is too large Load Diff

View File

@@ -55,6 +55,7 @@ pub mod flow_status;
pub mod flows;
pub mod global_settings;
pub mod indexer;
pub mod instance_config;
pub mod job_metrics;
#[cfg(all(feature = "parquet", feature = "private"))]
pub mod job_s3_helpers_ee;

View File

@@ -0,0 +1,531 @@
use async_trait::async_trait;
use reqwest::Method;
use serde_json::value::RawValue;
use sqlx::PgConnection;
use std::collections::HashMap;
use windmill_common::{
error::{Error, Result},
worker::to_raw_value,
BASE_URL, DB,
};
use windmill_queue::PushArgsOwned;
use crate::{
generate_webhook_service_url, get_token_by_prefix,
sync::{SyncAction, SyncError, TriggerSyncInfo},
update_native_trigger_error, update_native_trigger_service_config, External, NativeTrigger,
NativeTriggerData, ServiceName,
};
use super::{
endpoints, routes, CreateWatchResponse, Google, GoogleOAuthData, GoogleServiceConfig,
GoogleTriggerType, StopChannelRequest, WatchRequest,
};
#[async_trait]
impl External for Google {
type ServiceConfig = GoogleServiceConfig;
// Google has no "get channel" API, so TriggerData is never constructed.
// The trait default for get() returns Ok(None).
type TriggerData = ();
type OAuthData = GoogleOAuthData;
type CreateResponse = CreateWatchResponse;
const SERVICE_NAME: ServiceName = ServiceName::Google;
const DISPLAY_NAME: &'static str = "Google";
const SUPPORT_WEBHOOK: bool = true;
const TOKEN_ENDPOINT: &'static str = "https://oauth2.googleapis.com/token";
const REFRESH_ENDPOINT: &'static str = "https://oauth2.googleapis.com/token";
const AUTH_ENDPOINT: &'static str = "https://accounts.google.com/o/oauth2/v2/auth";
async fn create(
&self,
w_id: &str,
_oauth_data: &Self::OAuthData,
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
_tx: &mut PgConnection,
) -> Result<Self::CreateResponse> {
let channel_id = uuid::Uuid::new_v4().to_string();
self.create_watch_channel(w_id, &channel_id, webhook_token, data, db)
.await
}
async fn update(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<serde_json::Value> {
// Google doesn't support updating watch channels — delete old, create new.
let _ = self.delete(w_id, oauth_data, external_id, db, tx).await;
// Reuse the same channel ID so external_id stays permanent
let resp = self
.create_watch_channel(w_id, external_id, webhook_token, data, db)
.await?;
self.service_config_from_create_response(data, &resp)
.ok_or_else(|| {
Error::InternalErr(
"Failed to build service_config from create response".to_string(),
)
})
}
async fn delete(
&self,
w_id: &str,
_oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<()> {
// Get the stored trigger to find the google_resource_id and trigger_type
let trigger = sqlx::query_scalar!(
r#"
SELECT service_config
FROM native_trigger
WHERE external_id = $1 AND service_name = $2 AND workspace_id = $3
"#,
external_id,
ServiceName::Google as ServiceName,
w_id
)
.fetch_optional(&mut *tx)
.await?;
let config = trigger.flatten();
if config.is_none() {
return Ok(());
}
let config = config.unwrap();
let (google_resource_id, url) = super::parse_stop_channel_params(&config);
if !google_resource_id.is_empty() {
let stop_request =
StopChannelRequest { id: external_id.to_string(), resource_id: google_resource_id };
// Stop the channel (ignore errors - channel may have already expired)
let result: std::result::Result<serde_json::Value, _> = self
.http_client_request(&url, Method::POST, w_id, db, None, Some(&stop_request))
.await;
if let Err(e) = result {
tracing::warn!("Failed to stop Google channel {}: {}", external_id, e);
}
}
Ok(())
}
async fn maintain_triggers(
&self,
db: &DB,
workspace_id: &str,
triggers: &[NativeTrigger],
_oauth_data: &Self::OAuthData,
synced: &mut Vec<TriggerSyncInfo>,
errors: &mut Vec<SyncError>,
) {
renew_expiring_channels(self, db, workspace_id, triggers, synced, errors).await;
}
async fn prepare_webhook(
&self,
_db: &DB,
_w_id: &str,
headers: HashMap<String, String>,
_body: String,
_script_path: &str,
_is_flow: bool,
) -> Result<PushArgsOwned> {
// Google sends notification info in headers (same format for Drive and Calendar)
let payload = serde_json::json!({
"channel_id": headers.get("x-goog-channel-id").cloned().unwrap_or_default(),
"resource_id": headers.get("x-goog-resource-id").cloned().unwrap_or_default(),
"resource_state": headers.get("x-goog-resource-state").cloned().unwrap_or_default(),
"resource_uri": headers.get("x-goog-resource-uri").cloned().unwrap_or_default(),
"message_number": headers.get("x-goog-message-number").cloned().unwrap_or_default(),
"channel_expiration": headers.get("x-goog-channel-expiration").cloned().unwrap_or_default(),
"changed": headers.get("x-goog-changed").cloned().unwrap_or_default(),
"channel_token": headers.get("x-goog-channel-token").cloned().unwrap_or_default(),
});
let mut args: HashMap<String, Box<RawValue>> = HashMap::new();
args.insert("payload".to_string(), to_raw_value(&payload));
Ok(PushArgsOwned { extra: None, args })
}
fn external_id_and_metadata_from_response(
&self,
resp: &Self::CreateResponse,
) -> (String, Option<serde_json::Value>) {
let metadata = serde_json::json!({
"googleResourceId": resp.resource_id,
"expiration": resp.expiration,
});
(resp.id.clone(), Some(metadata))
}
fn service_config_from_create_response(
&self,
data: &NativeTriggerData<Self::ServiceConfig>,
resp: &Self::CreateResponse,
) -> Option<serde_json::Value> {
let mut config = data.service_config.clone();
config.google_resource_id = Some(resp.resource_id.clone());
config.expiration = Some(resp.expiration.clone());
serde_json::to_value(&config).ok()
}
fn additional_routes(&self) -> axum::Router {
routes::google_routes(self.clone())
}
}
// Helper methods for creating trigger type-specific watches
impl Google {
/// Build a webhook URL and watch request, then register the channel with Google.
/// Used by both `create()` (new UUID) and `update()` (reuse existing external_id).
async fn create_watch_channel(
&self,
w_id: &str,
channel_id: &str,
webhook_token: &str,
data: &NativeTriggerData<GoogleServiceConfig>,
db: &DB,
) -> Result<CreateWatchResponse> {
let base_url = &*BASE_URL.read().await;
let webhook_url = generate_webhook_service_url(
base_url,
w_id,
&data.script_path,
data.is_flow,
Some(channel_id),
ServiceName::Google,
webhook_token,
);
tracing::info!(
"Creating Google {} watch channel '{}' with webhook URL: {}",
data.service_config.trigger_type,
channel_id,
webhook_url
);
let expiration_ms = chrono::Utc::now().timestamp_millis()
+ (data.service_config.max_expiration_hours() as i64 * 3600 * 1000);
let mut watch_request = WatchRequest::new(channel_id.to_string(), webhook_url);
watch_request.expiration = Some(expiration_ms);
match data.service_config.trigger_type {
GoogleTriggerType::Drive => {
self.create_drive_watch(w_id, &data.service_config, &watch_request, db)
.await
}
GoogleTriggerType::Calendar => {
self.create_calendar_watch(w_id, &data.service_config, &watch_request, db)
.await
}
}
}
async fn create_drive_watch(
&self,
w_id: &str,
config: &GoogleServiceConfig,
watch_request: &WatchRequest,
db: &DB,
) -> Result<CreateWatchResponse> {
match config.resource_id.as_deref().filter(|s| !s.is_empty()) {
Some(resource_id) => {
// Specific file: use files.watch
let url = format!("{}/files/{}/watch", endpoints::DRIVE_API_BASE, resource_id);
self.http_client_request(&url, Method::POST, w_id, db, None, Some(watch_request))
.await
}
None => {
// All changes: use changes.watch
let token_url = format!("{}/changes/startPageToken", endpoints::DRIVE_API_BASE);
let token_response: serde_json::Value = self
.http_client_request::<_, ()>(&token_url, Method::GET, w_id, db, None, None)
.await?;
let start_page_token = token_response
.get("startPageToken")
.and_then(|v| v.as_str())
.ok_or_else(|| {
Error::InternalErr("Failed to get startPageToken".to_string())
})?;
let watch_body = serde_json::to_value(watch_request)?;
let watch_url = format!(
"{}/changes/watch?pageToken={}",
endpoints::DRIVE_API_BASE,
start_page_token
);
self.http_client_request(
&watch_url,
Method::POST,
w_id,
db,
None,
Some(&watch_body),
)
.await
}
}
}
async fn create_calendar_watch(
&self,
w_id: &str,
config: &GoogleServiceConfig,
watch_request: &WatchRequest,
db: &DB,
) -> Result<CreateWatchResponse> {
let calendar_id = config.calendar_id.as_ref().ok_or_else(|| {
Error::BadRequest("calendar_id is required for Calendar triggers".into())
})?;
let url = format!(
"{}/calendars/{}/events/watch",
endpoints::CALENDAR_API_BASE,
urlencoding::encode(calendar_id)
);
self.http_client_request(&url, Method::POST, w_id, db, None, Some(watch_request))
.await
}
/// Renew an expiring Google watch channel.
/// Stops the old channel and creates a new one with the same channel ID.
/// Returns the updated service_config with new expiration.
pub async fn renew_channel(
&self,
w_id: &str,
trigger: &NativeTrigger,
db: &DB,
) -> Result<serde_json::Value> {
let config: GoogleServiceConfig = trigger
.service_config
.as_ref()
.map(|v| serde_json::from_value(v.clone()))
.transpose()?
.ok_or_else(|| Error::InternalErr("Missing service config".to_string()))?;
let webhook_token = get_token_by_prefix(db, &trigger.webhook_token_prefix)
.await?
.ok_or_else(|| Error::InternalErr("Webhook token not found".to_string()))?;
let base_url = &*BASE_URL.read().await;
// Reuse the same channel ID so external_id stays permanent
let channel_id = trigger.external_id.clone();
let webhook_url = generate_webhook_service_url(
base_url,
w_id,
&trigger.script_path,
trigger.is_flow,
Some(&channel_id),
ServiceName::Google,
&webhook_token,
);
tracing::info!(
"Renewing Google {} watch channel '{}' with webhook URL: {}",
config.trigger_type,
channel_id,
webhook_url
);
let expiration_ms = chrono::Utc::now().timestamp_millis()
+ (config.max_expiration_hours() as i64 * 3600 * 1000);
let mut watch_request = WatchRequest::new(channel_id.clone(), webhook_url);
watch_request.expiration = Some(expiration_ms);
// Best-effort stop old channel before creating a new one
let old_google_resource_id = trigger
.service_config
.as_ref()
.and_then(|c| c.get("googleResourceId"))
.and_then(|r| r.as_str())
.unwrap_or_default();
if !old_google_resource_id.is_empty() {
let stop_request = StopChannelRequest {
id: channel_id.clone(),
resource_id: old_google_resource_id.to_string(),
};
let url = match config.trigger_type {
GoogleTriggerType::Calendar => {
format!("{}/channels/stop", endpoints::CALENDAR_API_BASE)
}
GoogleTriggerType::Drive => {
format!("{}/channels/stop", endpoints::DRIVE_API_BASE)
}
};
let result: std::result::Result<serde_json::Value, _> = self
.http_client_request(&url, Method::POST, w_id, db, None, Some(&stop_request))
.await;
if let Err(e) = result {
tracing::warn!(
"Failed to stop old Google channel {} during renewal: {}",
channel_id,
e
);
}
}
// Create new watch channel with the same channel ID
let resp = match config.trigger_type {
GoogleTriggerType::Drive => {
self.create_drive_watch(w_id, &config, &watch_request, db)
.await?
}
GoogleTriggerType::Calendar => {
self.create_calendar_watch(w_id, &config, &watch_request, db)
.await?
}
};
// Build the updated service_config with new expiration
let mut new_config = config;
new_config.google_resource_id = Some(resp.resource_id);
new_config.expiration = Some(resp.expiration);
serde_json::to_value(&new_config)
.map_err(|e| Error::internal_err(format!("Failed to serialize config: {}", e)))
}
}
/// Renewal window: renew Drive channels with <1 hour remaining, Calendar with <1 day remaining.
pub fn should_renew_channel(service_config: &serde_json::Value) -> bool {
let expiration_ms = service_config
.get("expiration")
.and_then(|v| v.as_str())
.and_then(|s| s.parse::<i64>().ok())
.unwrap_or(0);
if expiration_ms == 0 {
return false;
}
let now_ms = chrono::Utc::now().timestamp_millis();
let remaining_ms = expiration_ms - now_ms;
let trigger_type = service_config
.get("triggerType")
.and_then(|t| t.as_str())
.unwrap_or("drive");
let renewal_window_ms: i64 = match trigger_type {
"calendar" => 24 * 60 * 60 * 1000, // 1 day for Calendar (7 day expiry)
_ => 60 * 60 * 1000, // 1 hour for Drive (24h expiry)
};
remaining_ms < renewal_window_ms
}
async fn renew_expiring_channels(
handler: &Google,
db: &DB,
workspace_id: &str,
triggers: &[NativeTrigger],
synced: &mut Vec<TriggerSyncInfo>,
errors: &mut Vec<SyncError>,
) {
for trigger in triggers {
let Some(config) = &trigger.service_config else {
continue;
};
if !should_renew_channel(config) {
continue;
}
tracing::info!(
"Renewing expiring Google channel {} for script_path '{}' in workspace '{}'",
trigger.external_id,
trigger.script_path,
workspace_id
);
match handler.renew_channel(workspace_id, trigger, db).await {
Ok(new_config) => {
match update_native_trigger_service_config(
db,
workspace_id,
ServiceName::Google,
&trigger.external_id,
&new_config,
)
.await
{
Ok(()) => {
tracing::info!(
"Renewed Google channel {} for '{}'",
trigger.external_id,
trigger.script_path
);
synced.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ConfigUpdated,
});
}
Err(e) => {
tracing::error!(
"Failed to update DB after renewing Google channel {}: {}",
trigger.external_id,
e
);
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to update DB after channel renewal for {}: {}",
trigger.external_id, e
),
error_type: "channel_renewal_error".to_string(),
});
}
}
}
Err(e) => {
tracing::error!(
"Failed to renew Google channel {} for '{}': {}",
trigger.external_id,
trigger.script_path,
e
);
let _ = update_native_trigger_error(
db,
workspace_id,
ServiceName::Google,
&trigger.external_id,
Some(&format!("Channel renewal failed: {}", e)),
)
.await;
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Channel renewal failed for {}: {}",
trigger.external_id, e
),
error_type: "channel_renewal_error".to_string(),
});
}
}
}
}

View File

@@ -0,0 +1,241 @@
//! Google Native Trigger Module
//!
//! This module provides integration with Google services (Drive, Calendar)
//! push notification system to trigger Windmill scripts/flows when changes occur.
//!
//! ## Unified Architecture
//! A single "google" native trigger service handles both Drive and Calendar triggers.
//! The `trigger_type` field in `GoogleServiceConfig` determines which service to use.
//!
//! ## How it works:
//! 1. User configures a trigger with trigger_type (drive/calendar) and service-specific settings
//! 2. Windmill creates a "watch channel" via the appropriate Google API
//! 3. Google sends push notifications to Windmill's webhook when changes occur
//! 4. The webhook triggers the configured script/flow
//!
//! ## Important notes:
//! - Drive watch channels expire after max 24 hours
//! - Calendar watch channels expire after max 7 days
//! - Background sync job renews channels before expiration
use serde::{Deserialize, Serialize};
pub mod external;
pub mod routes;
pub use external::should_renew_channel;
/// Extracts `(google_resource_id, stop_url)` from a native trigger's service_config JSON.
/// Used by the `delete` method and tested independently.
pub fn parse_stop_channel_params(config: &serde_json::Value) -> (String, String) {
let google_resource_id = config
.get("googleResourceId")
.and_then(|r| r.as_str())
.map(String::from)
.unwrap_or_default();
let trigger_type = config
.get("triggerType")
.and_then(|t| t.as_str())
.unwrap_or("drive");
let stop_url = match trigger_type {
"calendar" => format!("{}/channels/stop", endpoints::CALENDAR_API_BASE),
_ => format!("{}/channels/stop", endpoints::DRIVE_API_BASE),
};
(google_resource_id, stop_url)
}
/// Handler struct for Google triggers (stateless, used for routing)
#[derive(Copy, Clone)]
pub struct Google;
/// Type of Google trigger
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GoogleTriggerType {
Drive,
Calendar,
}
impl std::fmt::Display for GoogleTriggerType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
GoogleTriggerType::Drive => write!(f, "drive"),
GoogleTriggerType::Calendar => write!(f, "calendar"),
}
}
}
/// User-provided configuration for a Google trigger.
/// The trigger_type determines which service-specific config is used.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct GoogleServiceConfig {
/// The type of trigger (drive or calendar)
pub trigger_type: GoogleTriggerType,
// Drive-specific fields (only used when trigger_type = drive)
/// The file ID to watch, or None for all changes (Drive only)
#[serde(skip_serializing_if = "Option::is_none")]
pub resource_id: Option<String>,
/// Human-readable name/path for display purposes (Drive only)
#[serde(skip_serializing_if = "Option::is_none")]
pub resource_name: Option<String>,
// Calendar-specific fields (only used when trigger_type = calendar)
/// The calendar ID to watch (Calendar only, e.g., "primary")
#[serde(skip_serializing_if = "Option::is_none")]
pub calendar_id: Option<String>,
/// Human-readable calendar name (Calendar only)
#[serde(skip_serializing_if = "Option::is_none")]
pub calendar_name: Option<String>,
// Metadata from Google watch channel (set after creation, used for renewal/deletion)
/// The resource ID assigned by Google for the watch channel
#[serde(skip_serializing_if = "Option::is_none")]
pub google_resource_id: Option<String>,
/// Channel expiration time (Unix timestamp in milliseconds, as string)
#[serde(skip_serializing_if = "Option::is_none")]
pub expiration: Option<String>,
}
impl GoogleServiceConfig {
/// Returns the expiration duration for this trigger type in hours
pub fn max_expiration_hours(&self) -> u64 {
match self.trigger_type {
GoogleTriggerType::Drive => 24, // Google Drive: max 24 hours
GoogleTriggerType::Calendar => 168, // Google Calendar: max 7 days
}
}
}
/// OAuth data structure shared by all Google services.
/// Stored encrypted in workspace_integrations table with service_name = 'google'.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct GoogleOAuthData {
/// The OAuth access token for API requests
pub access_token: String,
/// The OAuth refresh token for obtaining new access tokens
pub refresh_token: Option<String>,
/// When the access token expires
pub token_expires_at: Option<chrono::DateTime<chrono::Utc>>,
}
/// Google API endpoints
pub mod endpoints {
/// Google Drive API v3 base URL
pub const DRIVE_API_BASE: &str = "https://www.googleapis.com/drive/v3";
/// Google Calendar API v3 base URL
pub const CALENDAR_API_BASE: &str = "https://www.googleapis.com/calendar/v3";
/// Google OAuth2 token endpoint
pub const TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
/// Google OAuth2 authorization endpoint
pub const AUTH_ENDPOINT: &str = "https://accounts.google.com/o/oauth2/v2/auth";
}
/// OAuth scopes for Google services
pub mod scopes {
/// Read-only access to Google Drive files
pub const DRIVE_READONLY: &str = "https://www.googleapis.com/auth/drive.readonly";
/// Read-only access to Google Calendar
pub const CALENDAR_READONLY: &str = "https://www.googleapis.com/auth/calendar.readonly";
/// Events access to Google Calendar
pub const CALENDAR_EVENTS: &str = "https://www.googleapis.com/auth/calendar.events";
/// Returns all scopes needed for Google triggers (both Drive and Calendar)
pub fn all_scopes() -> Vec<&'static str> {
vec![DRIVE_READONLY, CALENDAR_READONLY, CALENDAR_EVENTS]
}
}
/// Common response wrapper for Google API errors
#[derive(Debug, Deserialize)]
pub struct GoogleApiError {
pub error: GoogleErrorDetails,
}
#[derive(Debug, Deserialize)]
pub struct GoogleErrorDetails {
pub code: i32,
pub message: String,
#[serde(default)]
pub errors: Vec<GoogleErrorItem>,
}
#[derive(Debug, Deserialize)]
pub struct GoogleErrorItem {
pub domain: Option<String>,
pub reason: Option<String>,
pub message: Option<String>,
}
/// Google Watch Channel response (used by Drive and Calendar push notifications)
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct WatchChannel {
/// Unique channel ID (we generate this as UUID)
pub id: String,
/// Resource ID assigned by Google
pub resource_id: String,
/// Resource URI being watched
pub resource_uri: Option<String>,
/// Channel expiration time (Unix timestamp in milliseconds)
pub expiration: i64,
/// Token for validation (optional)
pub token: Option<String>,
}
/// Response from Google API when creating a watch channel
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CreateWatchResponse {
/// The channel ID we provided
pub id: String,
/// Resource ID assigned by Google
pub resource_id: String,
/// Resource URI being watched
pub resource_uri: Option<String>,
/// Channel expiration (Unix timestamp in milliseconds)
pub expiration: String,
}
/// Request body for creating a watch channel
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct WatchRequest {
/// Unique channel ID (UUID)
pub id: String,
/// Type of delivery mechanism (always "web_hook")
#[serde(rename = "type")]
pub channel_type: String,
/// The URL to receive notifications
pub address: String,
/// Optional token for validation
#[serde(skip_serializing_if = "Option::is_none")]
pub token: Option<String>,
/// Optional expiration time in milliseconds (Google may adjust this)
#[serde(skip_serializing_if = "Option::is_none")]
pub expiration: Option<i64>,
}
impl WatchRequest {
pub fn new(channel_id: String, webhook_url: String) -> Self {
Self {
id: channel_id,
channel_type: "web_hook".to_string(),
address: webhook_url,
token: None,
expiration: None,
}
}
}
/// Request body for stopping a watch channel
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StopChannelRequest {
pub id: String,
pub resource_id: String,
}

View File

@@ -0,0 +1,220 @@
use std::sync::Arc;
use axum::{
extract::{Path, Query},
routing::get,
Extension, Json, Router,
};
use http::Method;
use serde::{Deserialize, Serialize};
use windmill_common::{error::JsonResult, DB};
use crate::{get_workspace_integration, External, ServiceName};
use super::Google;
fn escape_drive_query(s: &str) -> String {
s.replace('\\', "\\\\").replace('\'', "\\'")
}
#[derive(Debug, Serialize, Deserialize)]
pub struct GoogleCalendarEntry {
pub id: String,
pub summary: String,
#[serde(default)]
pub primary: bool,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct GoogleCalendarListResponse {
#[serde(default)]
items: Vec<GoogleCalendarListItem>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct GoogleCalendarListItem {
id: String,
#[serde(default)]
summary: String,
#[serde(default)]
primary: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct GoogleDriveFile {
pub id: String,
pub name: String,
pub mime_type: String,
#[serde(default)]
pub is_folder: bool,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct GoogleDriveFilesResponse {
pub files: Vec<GoogleDriveFile>,
#[serde(skip_serializing_if = "Option::is_none")]
pub next_page_token: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct DriveApiResponse {
#[serde(default)]
files: Vec<DriveApiFile>,
next_page_token: Option<String>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct DriveApiFile {
id: String,
name: String,
mime_type: String,
}
#[derive(Debug, Deserialize)]
pub struct DriveFilesQuery {
pub q: Option<String>,
pub parent_id: Option<String>,
pub page_token: Option<String>,
#[serde(default)]
pub shared_with_me: bool,
}
async fn list_calendars(
Extension(handler): Extension<Arc<Google>>,
Extension(db): Extension<DB>,
Path(workspace_id): Path<String>,
) -> JsonResult<Vec<GoogleCalendarEntry>> {
get_workspace_integration(&db, &workspace_id, ServiceName::Google).await?;
let url = format!(
"{}/users/me/calendarList",
super::endpoints::CALENDAR_API_BASE
);
let response: GoogleCalendarListResponse = handler
.http_client_request::<_, ()>(&url, Method::GET, &workspace_id, &db, None, None)
.await?;
let calendars = response
.items
.into_iter()
.map(|item| GoogleCalendarEntry {
id: item.id,
summary: item.summary,
primary: item.primary,
})
.collect();
Ok(Json(calendars))
}
async fn list_drive_files(
Extension(handler): Extension<Arc<Google>>,
Extension(db): Extension<DB>,
Path(workspace_id): Path<String>,
Query(query): Query<DriveFilesQuery>,
) -> JsonResult<GoogleDriveFilesResponse> {
get_workspace_integration(&db, &workspace_id, ServiceName::Google).await?;
let drive_query = if query.shared_with_me {
"sharedWithMe = true and trashed = false".to_string()
} else if let Some(ref parent_id) = query.parent_id {
format!(
"'{}' in parents and trashed = false",
escape_drive_query(parent_id)
)
} else if let Some(ref search) = query.q {
format!(
"name contains '{}' and trashed = false",
escape_drive_query(search)
)
} else {
"'root' in parents and trashed = false".to_string()
};
let mut url = format!(
"{}/files?q={}&fields=files(id,name,mimeType),nextPageToken&pageSize=50&orderBy=folder,name&supportsAllDrives=true&includeItemsFromAllDrives=true",
super::endpoints::DRIVE_API_BASE,
urlencoding::encode(&drive_query)
);
if let Some(ref page_token) = query.page_token {
url.push_str(&format!("&pageToken={}", urlencoding::encode(page_token)));
}
let response: DriveApiResponse = handler
.http_client_request::<_, ()>(&url, Method::GET, &workspace_id, &db, None, None)
.await?;
let files = response
.files
.into_iter()
.map(|f| {
let is_folder = f.mime_type == "application/vnd.google-apps.folder";
GoogleDriveFile { id: f.id, name: f.name, mime_type: f.mime_type, is_folder }
})
.collect();
Ok(Json(GoogleDriveFilesResponse {
files,
next_page_token: response.next_page_token,
}))
}
#[derive(Debug, Serialize)]
pub struct SharedDriveEntry {
pub id: String,
pub name: String,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct SharedDrivesApiResponse {
#[serde(default)]
drives: Vec<SharedDriveApiEntry>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct SharedDriveApiEntry {
id: String,
name: String,
}
async fn list_shared_drives(
Extension(handler): Extension<Arc<Google>>,
Extension(db): Extension<DB>,
Path(workspace_id): Path<String>,
) -> JsonResult<Vec<SharedDriveEntry>> {
get_workspace_integration(&db, &workspace_id, ServiceName::Google).await?;
let url = format!(
"{}/drives?pageSize=100&fields=drives(id,name)",
super::endpoints::DRIVE_API_BASE
);
let response: SharedDrivesApiResponse = handler
.http_client_request::<_, ()>(&url, Method::GET, &workspace_id, &db, None, None)
.await?;
let drives = response
.drives
.into_iter()
.map(|d| SharedDriveEntry { id: d.id, name: d.name })
.collect();
Ok(Json(drives))
}
pub fn google_routes(service: Google) -> Router {
let service = Arc::new(service);
Router::new()
.route("/calendars", get(list_calendars))
.route("/drive/files", get(list_drive_files))
.route("/drive/shared_drives", get(list_shared_drives))
.layer(Extension(service))
}

View File

@@ -1,8 +1,7 @@
use crate::{
delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix,
get_workspace_integration, list_native_triggers, store_native_trigger,
update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, NativeTriggerData,
ServiceName,
decrypt_oauth_data, delete_native_trigger, delete_token_by_prefix, get_native_trigger,
get_token_by_prefix, list_native_triggers, store_native_trigger, update_native_trigger_error,
External, NativeTrigger, NativeTriggerConfig, NativeTriggerData, ServiceName,
};
use axum::{
extract::{Path, Query},
@@ -65,7 +64,7 @@ pub struct ListQuery {
pub struct FullTriggerResponse<T: Serialize> {
#[serde(flatten)]
pub windmill_data: NativeTrigger,
pub external_data: T,
pub external_data: Option<T>,
}
#[derive(Debug, Serialize)]
@@ -132,15 +131,9 @@ async fn create_native_trigger<T: External>(
)
.await?;
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
let integration_service = service_name.integration_service();
let oauth_data: T::OAuthData =
decrypt_oauth_data(&db, &workspace_id, integration_service).await?;
let resp = handler
.create(
@@ -155,24 +148,25 @@ async fn create_native_trigger<T: External>(
let (external_id, _) = handler.external_id_and_metadata_from_response(&resp);
// update the created external trigger with a new uri containing the external_id
handler
.update(
&workspace_id,
&oauth_data,
&external_id,
&webhook_token,
&data,
&db,
&mut tx,
)
.await?;
// Fetch the updated trigger data from the external service and extract service_config
let trigger_data = handler
.get(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await?;
let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?;
// Some services (e.g. Google) can build service_config directly from the create response,
// while others (e.g. Nextcloud) need an update+get cycle to correct the webhook URL
// with the external_id assigned by the remote service.
let service_config =
if let Some(config) = handler.service_config_from_create_response(&data, &resp) {
config
} else {
handler
.update(
&workspace_id,
&oauth_data,
&external_id,
&webhook_token,
&data,
&db,
&mut tx,
)
.await?
};
let config = NativeTriggerConfig {
script_path: data.script_path.clone(),
@@ -227,22 +221,32 @@ async fn update_native_trigger_handler<T: External>(
)
.await?;
let mut tx = user_db.begin(&authed).await?;
let integration_service = service_name.integration_service();
let oauth_data: T::OAuthData =
decrypt_oauth_data(&db, &workspace_id, integration_service).await?;
let mut tx = user_db.clone().begin(&authed).await?;
let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id)
.await?
.ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?;
// Look up the full token using the stored prefix (use db, not tx, for token table)
let runnable_changed =
existing.script_path != data.script_path || existing.is_flow != data.is_flow;
let webhook_token = match get_token_by_prefix(&db, &existing.webhook_token_prefix).await? {
Some(token) => token,
None => {
tracing::warn!(
"Webhook token not found for trigger {} (prefix: {}), recreating token",
external_id,
existing.webhook_token_prefix
);
new_webhook_token(
Some(token) if !runnable_changed => token,
existing_token => {
if let Some(_) = existing_token {
delete_token_by_prefix(&db, &existing.webhook_token_prefix).await?;
} else {
tracing::warn!(
"Webhook token not found for trigger {} (prefix: {}), recreating token",
external_id,
existing.webhook_token_prefix
);
}
let token = new_webhook_token(
&mut *tx,
&db,
&authed,
@@ -251,21 +255,14 @@ async fn update_native_trigger_handler<T: External>(
&workspace_id,
service_name,
)
.await?
.await?;
tx.commit().await?;
tx = user_db.begin(&authed).await?;
token
}
};
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
handler
let service_config = handler
.update(
&workspace_id,
&oauth_data,
@@ -277,12 +274,6 @@ async fn update_native_trigger_handler<T: External>(
)
.await?;
// Fetch the updated trigger data from the external service and extract service_config
let trigger_data = handler
.get(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await?;
let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?;
let config = NativeTriggerConfig {
script_path: data.script_path.clone(),
is_flow: data.is_flow,
@@ -341,22 +332,16 @@ async fn get_native_trigger_handler<T: External>(
)
.await?;
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
let integration_service = service_name.integration_service();
let oauth_data: T::OAuthData =
decrypt_oauth_data(&db, &workspace_id, integration_service).await?;
let native_trigger = handler
.get(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await;
let native_trigger_config = match native_trigger {
Ok(native_cfg) => {
let external_data = match native_trigger {
Ok(Some(native_cfg)) => {
// Clear error if it was set
if windmill_trigger.error.is_some() {
update_native_trigger_error(
@@ -368,8 +353,9 @@ async fn get_native_trigger_handler<T: External>(
)
.await?;
}
native_cfg
Some(native_cfg)
}
Ok(None) => None,
Err(Error::NotFound(_)) => {
let error_msg = "Trigger no longer exists on external service".to_string();
tracing::warn!(
@@ -396,10 +382,7 @@ async fn get_native_trigger_handler<T: External>(
Err(e) => return Err(e),
};
let full_resp = Json(FullTriggerResponse {
windmill_data: windmill_trigger,
external_data: native_trigger_config,
});
let full_resp = Json(FullTriggerResponse { windmill_data: windmill_trigger, external_data });
Ok(full_resp)
}
@@ -430,15 +413,9 @@ async fn delete_native_trigger_handler<T: External>(
)
.await?;
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
let integration_service = service_name.integration_service();
let oauth_data: T::OAuthData =
decrypt_oauth_data(&db, &workspace_id, integration_service).await?;
handler
.delete(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
@@ -476,34 +453,6 @@ async fn delete_native_trigger_handler<T: External>(
Ok(format!("Native trigger deleted"))
}
async fn exists_native_trigger_handler<T: External>(
Extension(service_name): Extension<ServiceName>,
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((workspace_id, external_id)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists = sqlx::query_scalar!(
r#"
SELECT EXISTS(
SELECT 1
FROM native_trigger
WHERE
workspace_id = $1 AND
service_name = $2 AND
external_id = $3
)
"#,
workspace_id,
service_name as ServiceName,
external_id
)
.fetch_one(&db)
.await?
.unwrap_or(false);
Ok(Json(exists))
}
async fn list_native_triggers_handler<T: External>(
Extension(service_name): Extension<ServiceName>,
authed: ApiAuthed,
@@ -543,10 +492,6 @@ pub fn service_routes<T: External + 'static>(handler: T) -> Router {
.route(
"/delete/:external_id",
delete(delete_native_trigger_handler::<T>),
)
.route(
"/exists/:external_id",
get(exists_native_trigger_handler::<T>),
);
standard_routes
@@ -562,15 +507,12 @@ pub fn generate_native_trigger_routers() -> Router {
#[cfg(feature = "native_trigger")]
{
use crate::google::Google;
use crate::nextcloud::NextCloud;
// Register all service routes here
// When adding a new service:
// 1. Import the handler: use crate::newservice::NewServiceHandler;
// 2. Add the route: .nest("/newservice", service_routes(NewServiceHandler))
return router.nest("/nextcloud", service_routes(NextCloud));
// Add new services here:
// .nest("/newservice", service_routes(NewServiceHandler))
return router
.nest("/nextcloud", service_routes(NextCloud))
.nest("/google", service_routes(Google));
}
#[cfg(not(feature = "native_trigger"))]

View File

@@ -36,7 +36,7 @@ use async_trait::async_trait;
use chrono::{DateTime, Utc};
use http::StatusCode;
use itertools::Itertools;
use reqwest::{Client, Method};
use reqwest::Method;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::json;
use serde_json::value::RawValue;
@@ -47,6 +47,7 @@ use tokio::task;
use windmill_common::{
error::{to_anyhow, Error, Result},
triggers::TriggerKind,
utils::HTTP_CLIENT,
variables::{build_crypt, decrypt, encrypt},
DB,
};
@@ -62,9 +63,9 @@ pub mod workspace_integrations;
// Service modules - add new services here:
#[cfg(feature = "native_trigger")]
pub mod google;
#[cfg(feature = "native_trigger")]
pub mod nextcloud;
// #[cfg(feature = "native_trigger")]
// pub mod newservice;
/// Enum of all supported native trigger services.
/// When adding a new service, add a variant here (e.g., `NewService`).
@@ -73,17 +74,15 @@ pub mod nextcloud;
#[serde(rename_all = "lowercase")]
pub enum ServiceName {
Nextcloud,
// Add new services here:
// NewService,
Google,
}
impl TryFrom<String> for ServiceName {
type Error = Error;
fn try_from(value: String) -> std::result::Result<Self, Self::Error> {
// Add new service match arms here:
let service = match value.as_str() {
"nextcloud" => ServiceName::Nextcloud,
// "newservice" => ServiceName::NewService,
"google" => ServiceName::Google,
_ => {
return Err(anyhow::anyhow!(
"Unknown service, currently supported services are: [{}]",
@@ -99,49 +98,73 @@ impl TryFrom<String> for ServiceName {
impl ServiceName {
/// Returns the lowercase string identifier for this service.
/// Add new service match arms here.
pub fn as_str(&self) -> &'static str {
match self {
ServiceName::Nextcloud => "nextcloud",
// ServiceName::NewService => "newservice",
ServiceName::Google => "google",
}
}
/// Returns the corresponding TriggerKind for this service.
/// Requires adding the variant to TriggerKind in windmill_common.
pub fn as_trigger_kind(&self) -> TriggerKind {
match self {
ServiceName::Nextcloud => TriggerKind::Nextcloud,
// ServiceName::NewService => TriggerKind::NewService,
ServiceName::Google => TriggerKind::Google,
}
}
/// Returns the corresponding JobTriggerKind for this service.
/// Requires adding the variant to JobTriggerKind in windmill_common.
pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind {
match self {
ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud,
// ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService,
ServiceName::Google => windmill_common::jobs::JobTriggerKind::Google,
}
}
/// Returns the OAuth token endpoint path for this service.
/// Used for building OAuth clients dynamically.
pub fn token_endpoint(&self) -> &'static str {
match self {
ServiceName::Nextcloud => "/apps/oauth2/api/v1/token",
// ServiceName::NewService => "/oauth/token",
ServiceName::Google => "https://oauth2.googleapis.com/token",
}
}
/// Returns the OAuth authorization endpoint path for this service.
/// Used for building OAuth authorization URLs.
pub fn auth_endpoint(&self) -> &'static str {
match self {
ServiceName::Nextcloud => "/apps/oauth2/authorize",
// ServiceName::NewService => "/oauth/authorize",
ServiceName::Google => "https://accounts.google.com/o/oauth2/v2/auth",
}
}
/// Returns the OAuth scopes for this service's authorization flow.
pub fn oauth_scopes(&self) -> &'static str {
match self {
ServiceName::Nextcloud => "read write",
ServiceName::Google => "https://www.googleapis.com/auth/drive.readonly https://www.googleapis.com/auth/calendar.readonly https://www.googleapis.com/auth/calendar.events",
}
}
/// Returns the resource type used for storing OAuth tokens.
pub fn resource_type(&self) -> &'static str {
match self {
ServiceName::Nextcloud => "nextcloud",
ServiceName::Google => "gworkspace",
}
}
/// Returns extra OAuth authorization parameters required by this service.
pub fn extra_auth_params(&self) -> &[(&'static str, &'static str)] {
match self {
ServiceName::Google => &[("access_type", "offline"), ("prompt", "consent")],
ServiceName::Nextcloud => &[],
}
}
/// Returns the integration service name for workspace_integrations lookup.
pub fn integration_service(&self) -> ServiceName {
*self
}
}
impl std::fmt::Display for ServiceName {
@@ -150,6 +173,16 @@ impl std::fmt::Display for ServiceName {
}
}
/// Resolves an endpoint URL. If the endpoint is already an absolute URL (starts with http),
/// returns it as-is. Otherwise, prepends the base_url.
pub fn resolve_endpoint(base_url: &str, endpoint: &str) -> String {
if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
endpoint.to_string()
} else {
format!("{}{}", base_url, endpoint)
}
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct NativeTrigger {
pub external_id: String,
@@ -183,6 +216,7 @@ pub struct WorkspaceIntegration {
pub workspace_id: String,
pub service_name: ServiceName,
pub oauth_data: serde_json::Value,
pub resource_path: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub created_by: String,
@@ -200,6 +234,7 @@ pub trait External: Send + Sync + 'static {
const DISPLAY_NAME: &'static str;
const TOKEN_ENDPOINT: &'static str;
const REFRESH_ENDPOINT: &'static str;
const AUTH_ENDPOINT: &'static str;
async fn create(
&self,
@@ -211,6 +246,10 @@ pub trait External: Send + Sync + 'static {
tx: &mut PgConnection,
) -> Result<Self::CreateResponse>;
/// Update a trigger on the external service and return the resolved service_config to store.
/// Each service is responsible for resolving the final config:
/// - Services that re-create the resource (e.g. Google) build config from request data + response metadata.
/// - Services that modify in-place (e.g. Nextcloud) fetch back the updated state and extract config.
async fn update(
&self,
w_id: &str,
@@ -220,16 +259,21 @@ pub trait External: Send + Sync + 'static {
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<()>;
) -> Result<serde_json::Value>;
/// Fetch the trigger's state from the external service.
/// Returns `Ok(None)` (default) when the service has no "get" API (e.g. Google).
/// Services that can fetch state (e.g. Nextcloud) override to return `Ok(Some(data))`.
async fn get(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<Self::TriggerData>;
_w_id: &str,
_oauth_data: &Self::OAuthData,
_external_id: &str,
_db: &DB,
_tx: &mut PgConnection,
) -> Result<Option<Self::TriggerData>> {
Ok(None)
}
async fn delete(
&self,
@@ -240,23 +284,19 @@ pub trait External: Send + Sync + 'static {
tx: &mut PgConnection,
) -> Result<()>;
#[allow(unused)]
async fn exists(
/// Periodic background maintenance for triggers in a workspace.
/// Each service implements its own logic:
/// - Nextcloud: lists external triggers and reconciles with DB state
/// - Google: renews expiring watch channels
async fn maintain_triggers(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<bool>;
async fn list_all(
&self,
w_id: &str,
workspace_id: &str,
triggers: &[NativeTrigger],
oauth_data: &Self::OAuthData,
db: &DB,
tx: &mut PgConnection,
) -> Result<Vec<Self::TriggerData>>;
synced: &mut Vec<crate::sync::TriggerSyncInfo>,
errors: &mut Vec<crate::sync::SyncError>,
);
async fn prepare_webhook(
&self,
@@ -275,19 +315,18 @@ pub trait External: Send + Sync + 'static {
resp: &Self::CreateResponse,
) -> (String, Option<serde_json::Value>);
fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String;
/// Extracts the service-specific config from trigger data (from external service).
/// Used for comparison during sync to detect config drift.
/// Default implementation converts the trigger data to a JSON value
/// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method.
fn extract_service_config_from_trigger_data(
/// Build the service_config directly from the create response and input data,
/// skipping the update+get cycle after creation.
/// Return `None` (default) to use the update+get pattern (e.g. Nextcloud needs to
/// correct the webhook URL with the external_id assigned by the remote service).
/// Return `Some(config)` to skip update+get entirely (e.g. Google already includes
/// the channel_id in the webhook URL from the start).
fn service_config_from_create_response(
&self,
data: &Self::TriggerData,
) -> Result<serde_json::Value> {
serde_json::to_value(data).map_err(|e| {
Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e))
})
_data: &NativeTriggerData<Self::ServiceConfig>,
_resp: &Self::CreateResponse,
) -> Option<serde_json::Value> {
None
}
fn additional_routes(&self) -> axum::Router {
@@ -299,13 +338,12 @@ pub trait External: Send + Sync + 'static {
url: &str,
method: Method,
workspace_id: &str,
tx: &mut PgConnection,
db: &DB,
headers: Option<HashMap<String, String>>,
body: Option<&B>,
) -> Result<T> {
let oauth_config: OAuthConfig =
decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?;
decrypt_oauth_data(db, workspace_id, Self::SERVICE_NAME).await?;
let result = make_http_request(
url,
@@ -327,19 +365,26 @@ pub trait External: Send + Sync + 'static {
err.status().unwrap()
);
let refreshed_oauth_config =
refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?;
let refreshed_oauth_config = refresh_oauth_tokens(
&oauth_config,
Self::REFRESH_ENDPOINT,
Self::AUTH_ENDPOINT,
)
.await?;
task::spawn({
let db_clone = db.clone();
let workspace_id_clone = workspace_id.to_string();
let refreshed_json = oauth_config_to_json(&refreshed_oauth_config);
let service_name = Self::SERVICE_NAME;
let new_access_token = refreshed_oauth_config.access_token.clone();
let new_refresh_token = refreshed_oauth_config.refresh_token.clone();
async move {
update_workspace_integration_tokens_helper(
db_clone,
workspace_id_clone,
Self::SERVICE_NAME,
refreshed_json,
update_oauth_token_resource(
&db_clone,
&workspace_id_clone,
service_name,
&new_access_token,
new_refresh_token.as_deref(),
)
.await;
}
@@ -376,8 +421,8 @@ pub async fn make_http_request<T: DeserializeOwned + Send, B: Serialize>(
headers: Option<HashMap<String, String>>,
body: Option<&B>,
access_token: &str,
) -> std::result::Result<T, reqwest::Error> {
let client = Client::new();
) -> std::result::Result<T, HttpRequestError> {
let client = &*HTTP_CLIENT;
let mut request = client.request(method, url);
request = request
@@ -400,91 +445,148 @@ pub async fn make_http_request<T: DeserializeOwned + Send, B: Serialize>(
let response = request.send().await?.error_for_status()?;
let response_json = response.json().await?;
Ok(response_json)
// Handle empty responses (e.g. 204 No Content from Google channels/stop)
let bytes = response.bytes().await?;
if bytes.is_empty() {
serde_json::from_str("null").map_err(HttpRequestError::Json)
} else {
serde_json::from_slice(&bytes).map_err(HttpRequestError::Json)
}
}
pub async fn decrypt_oauth_data<
'c,
E: sqlx::Executor<'c, Database = Postgres>,
T: DeserializeOwned,
>(
tx: E,
#[derive(Debug)]
pub enum HttpRequestError {
Reqwest(reqwest::Error),
Json(serde_json::Error),
}
impl std::fmt::Display for HttpRequestError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HttpRequestError::Reqwest(e) => write!(f, "{}", e),
HttpRequestError::Json(e) => write!(f, "JSON decode error: {}", e),
}
}
}
impl std::error::Error for HttpRequestError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
HttpRequestError::Reqwest(e) => Some(e),
HttpRequestError::Json(e) => Some(e),
}
}
}
impl From<reqwest::Error> for HttpRequestError {
fn from(e: reqwest::Error) -> Self {
HttpRequestError::Reqwest(e)
}
}
impl HttpRequestError {
pub fn status(&self) -> Option<StatusCode> {
match self {
HttpRequestError::Reqwest(e) => e.status(),
HttpRequestError::Json(_) => None,
}
}
}
/// Read OAuth client_id and client_secret from instance-level global settings.
/// Used when a workspace integration has `instance_shared: true`.
async fn get_instance_oauth_credentials(
db: &DB,
service_name: ServiceName,
) -> Result<(String, String)> {
windmill_common::global_settings::get_instance_oauth_credentials(
db,
service_name.resource_type(),
)
.await
}
pub async fn decrypt_oauth_data<T: DeserializeOwned>(
db: &DB,
workspace_id: &str,
service_name: ServiceName,
) -> Result<T> {
let integration = get_workspace_integration(tx, workspace_id, service_name).await?;
let integration = get_workspace_integration(db, workspace_id, service_name).await?;
let oauth_data = integration.oauth_data;
let resource_path = integration.resource_path.as_deref().ok_or_else(|| {
Error::InternalErr(format!(
"No resource_path in {} integration config. Please reconnect the integration.",
service_name
))
})?;
let mc = build_crypt(db, workspace_id).await?;
let mut oauth_data: serde_json::Value = integration.oauth_data;
if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) {
let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string())
.map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?;
oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token);
}
let var_row = sqlx::query!(
"SELECT value, account FROM variable WHERE workspace_id = $1 AND path = $2",
workspace_id,
resource_path,
)
.fetch_optional(db)
.await?
.ok_or_else(|| {
Error::InternalErr(format!(
"Variable at {} not found for {} integration",
resource_path, service_name
))
})?;
if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str())
let access_token = decrypt(&mc, var_row.value)
.map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?;
let refresh_token = if let Some(account_id) = var_row.account {
sqlx::query_scalar!(
"SELECT refresh_token FROM account WHERE workspace_id = $1 AND id = $2",
workspace_id,
account_id,
)
.fetch_optional(db)
.await?
} else {
None
};
let (client_id, client_secret) = if oauth_data
.get("instance_shared")
.and_then(|v| v.as_bool())
.unwrap_or(false)
{
let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string())
.map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?;
oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token);
}
// Read credentials from instance-level global settings instead of workspace_integrations
let (id, secret) = get_instance_oauth_credentials(db, service_name)
.await
.map_err(|e| {
Error::InternalErr(format!(
"Failed to read instance OAuth credentials for {}: {}",
service_name, e
))
})?;
(id, secret)
} else {
(
oauth_data["client_id"].as_str().unwrap_or("").to_string(),
oauth_data["client_secret"]
.as_str()
.unwrap_or("")
.to_string(),
)
};
serde_json::from_value(oauth_data)
.map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e)))
}
#[allow(unused)]
pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result<OAuthConfig> {
let base_url = oauth_data
.get("base_url")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))?
.to_string();
let access_token = oauth_data
.get("access_token")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))?
.to_string();
let refresh_token = oauth_data
.get("refresh_token")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let client_id = oauth_data
.get("client_id")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))?
.to_string();
let client_secret = oauth_data
.get("client_secret")
.and_then(|v| v.as_str())
.ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))?
.to_string();
Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret })
}
#[inline]
pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value {
let mut json = json!({
"base_url": config.base_url,
"access_token": config.access_token,
"client_id": config.client_id,
"client_secret": config.client_secret,
let assembled = json!({
"base_url": oauth_data["base_url"].as_str().unwrap_or(""),
"access_token": access_token,
"refresh_token": refresh_token,
"client_id": client_id,
"client_secret": client_secret,
});
if let Some(refresh_token) = &config.refresh_token {
json["refresh_token"] = serde_json::Value::String(refresh_token.clone());
}
json
serde_json::from_value(assembled)
.map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e)))
}
/// Token refresh response
@@ -500,6 +602,7 @@ struct RefreshTokenResponse {
pub async fn refresh_oauth_tokens(
oauth_config: &OAuthConfig,
refresh_endpoint: &str,
auth_endpoint: &str,
) -> Result<OAuthConfig> {
let refresh_token_str = oauth_config
.refresh_token
@@ -508,9 +611,9 @@ pub async fn refresh_oauth_tokens(
// Build OAuth client for token refresh
// Auth URL is not used for refresh, but required by the client constructor
let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url))
let auth_url = Url::parse(&resolve_endpoint(&oauth_config.base_url, auth_endpoint))
.map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?;
let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint))
let token_url = Url::parse(&resolve_endpoint(&oauth_config.base_url, refresh_endpoint))
.map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?;
let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url);
@@ -539,62 +642,80 @@ pub async fn refresh_oauth_tokens(
pub async fn refresh_oauth_tokens(
_oauth_config: &OAuthConfig,
_refresh_endpoint: &str,
_auth_endpoint: &str,
) -> Result<OAuthConfig> {
Err(Error::InternalErr(
"Native triggers feature is not enabled".to_string(),
))
}
async fn update_workspace_integration_tokens_helper(
db: DB,
workspace_id: String,
async fn update_oauth_token_resource(
db: &DB,
workspace_id: &str,
service_name: ServiceName,
oauth_data: serde_json::Value,
new_access_token: &str,
new_refresh_token: Option<&str>,
) {
let result = async {
let mut tx = db.begin().await?;
let mc = build_crypt(&db, &workspace_id).await?;
let mut encrypted_oauth_data = oauth_data;
let integration = get_workspace_integration(db, workspace_id, service_name).await?;
let resource_path = integration.resource_path.ok_or_else(|| {
Error::InternalErr(format!(
"No resource_path in {} integration config",
service_name
))
})?;
if let Some(access_token) = encrypted_oauth_data
.get("access_token")
.and_then(|v| v.as_str())
{
let encrypted_access_token = encrypt(&mc, access_token);
encrypted_oauth_data["access_token"] =
serde_json::Value::String(encrypted_access_token);
}
if let Some(refresh_token) = encrypted_oauth_data
.get("refresh_token")
.and_then(|v| v.as_str())
{
let encrypted_refresh_token = encrypt(&mc, refresh_token);
encrypted_oauth_data["refresh_token"] =
serde_json::Value::String(encrypted_refresh_token);
}
let mc = build_crypt(db, workspace_id).await?;
let encrypted_token = encrypt(&mc, new_access_token);
sqlx::query!(
r#"
UPDATE workspace_integrations
SET oauth_data = $1, updated_at = now()
WHERE workspace_id = $2 AND service_name = $3
"#,
encrypted_oauth_data,
"UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3",
encrypted_token,
workspace_id,
service_name as ServiceName,
resource_path,
)
.execute(&mut *tx)
.execute(db)
.await?;
tx.commit().await?;
if let Some(refresh_token) = new_refresh_token {
sqlx::query!(
"UPDATE account SET
refresh_token = $1,
expires_at = now() + interval '1 hour',
refresh_error = NULL
WHERE workspace_id = $2 AND client = $3 AND is_workspace_integration = true",
refresh_token,
workspace_id,
service_name.as_str(),
)
.execute(db)
.await?;
} else {
// Even without a new refresh token, update expires_at to prevent
// the background refresh from re-refreshing immediately
sqlx::query!(
"UPDATE account SET
expires_at = now() + interval '1 hour',
refresh_error = NULL
WHERE workspace_id = $1 AND client = $2 AND is_workspace_integration = true",
workspace_id,
service_name.as_str(),
)
.execute(db)
.await?;
}
Ok::<(), Error>(())
}
.await;
if let Err(e) = result {
tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}",
service_name, workspace_id, e);
tracing::error!(
"Failed to update OAuth tokens for {} in workspace {}: {}",
service_name,
workspace_id,
e
);
}
}
@@ -939,6 +1060,7 @@ pub async fn store_workspace_integration(
workspace_id: &str,
service_name: ServiceName,
oauth_data: serde_json::Value,
resource_path: Option<&str>,
) -> Result<()> {
sqlx::query!(
r#"
@@ -946,20 +1068,23 @@ pub async fn store_workspace_integration(
workspace_id,
service_name,
oauth_data,
resource_path,
created_by,
created_at,
updated_at
) VALUES (
$1, $2, $3, $4, now(), now()
$1, $2, $3, $4, $5, now(), now()
)
ON CONFLICT (workspace_id, service_name)
DO UPDATE SET
oauth_data = $3,
resource_path = $4,
updated_at = now()
"#,
workspace_id,
service_name as ServiceName,
oauth_data,
resource_path,
authed.username,
)
.execute(&mut *tx)
@@ -980,6 +1105,7 @@ pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Post
workspace_id,
service_name AS "service_name!: ServiceName",
oauth_data,
resource_path,
created_at,
updated_at,
created_by
@@ -1051,3 +1177,43 @@ pub fn generate_webhook_service_url(
url
}
/// Process incoming webhook request for a native trigger service.
/// Dispatches to the service-specific `prepare_webhook` to transform headers/body into args.
/// Returns `None` if the service doesn't need special processing (standard body parsing is used).
#[cfg(feature = "native_trigger")]
pub async fn prepare_native_trigger_args(
service_name: ServiceName,
db: &DB,
w_id: &str,
headers: &http::HeaderMap,
body: String,
) -> Result<Option<PushArgsOwned>> {
let headers_map: HashMap<String, String> = headers
.iter()
.filter_map(|(k, v)| v.to_str().ok().map(|v| (k.to_string(), v.to_string())))
.collect();
match service_name {
ServiceName::Google => {
let handler = google::Google;
let args = handler
.prepare_webhook(db, w_id, headers_map, body, "", false)
.await?;
Ok(Some(args))
}
ServiceName::Nextcloud => Ok(None),
}
}
/// Fallback when native_trigger feature is disabled
#[cfg(not(feature = "native_trigger"))]
pub async fn prepare_native_trigger_args(
_service_name: ServiceName,
_db: &DB,
_w_id: &str,
_headers: &http::HeaderMap,
_body: String,
) -> Result<Option<PushArgsOwned>> {
Ok(None)
}

View File

@@ -102,6 +102,7 @@ impl External for NextCloud {
const SUPPORT_WEBHOOK: bool = true;
const TOKEN_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token";
const REFRESH_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token";
const AUTH_ENDPOINT: &'static str = "/oauth/authorize";
async fn create(
&self,
@@ -110,7 +111,7 @@ impl External for NextCloud {
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
_tx: &mut PgConnection,
) -> Result<Self::CreateResponse> {
// During create, we don't have external_id yet (it comes from NextCloud's response)
let full_nextcloud_payload =
@@ -129,7 +130,6 @@ impl External for NextCloud {
&url,
Method::POST,
w_id,
tx,
db,
Some(headers),
Some(&full_nextcloud_payload),
@@ -148,7 +148,7 @@ impl External for NextCloud {
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<()> {
) -> Result<serde_json::Value> {
// During update, we have the external_id so include it in the webhook URL
let full_nextcloud_payload =
FullNextcloudPayload::new(w_id, Some(external_id), webhook_token, data).await;
@@ -166,14 +166,25 @@ impl External for NextCloud {
&url,
Method::POST,
w_id,
tx,
db,
Some(headers),
Some(&full_nextcloud_payload),
)
.await?;
Ok(())
// Fetch back the updated state and convert to JSON config
let trigger_data = self
.get(w_id, oauth_data, external_id, db, tx)
.await?
.ok_or_else(|| {
Error::InternalErr(format!(
"Failed to fetch back trigger {} after update",
external_id
))
})?;
serde_json::to_value(&trigger_data).map_err(|e| {
Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e))
})
}
async fn get(
@@ -182,8 +193,8 @@ impl External for NextCloud {
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<Self::TriggerData> {
_tx: &mut PgConnection,
) -> Result<Option<Self::TriggerData>> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
oauth_data.base_url, external_id
@@ -193,10 +204,10 @@ impl External for NextCloud {
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let ocs_response: OcsResponse<NextCloudTriggerData> = self
.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, Some(headers), None)
.http_client_request::<_, ()>(&url, Method::GET, w_id, db, Some(headers), None)
.await?;
Ok(ocs_response.ocs.data)
Ok(Some(ocs_response.ocs.data))
}
async fn delete(
@@ -205,7 +216,7 @@ impl External for NextCloud {
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
_tx: &mut PgConnection,
) -> Result<()> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
@@ -216,7 +227,7 @@ impl External for NextCloud {
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let _: serde_json::Value = self
.http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, Some(headers), None)
.http_client_request::<_, ()>(&url, Method::DELETE, w_id, db, Some(headers), None)
.await
.or_else(|e| match &e {
Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null),
@@ -226,44 +237,72 @@ impl External for NextCloud {
Ok(())
}
async fn exists(
async fn maintain_triggers(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<bool> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
oauth_data.base_url, external_id
);
workspace_id: &str,
triggers: &[crate::NativeTrigger],
oauth_data: &Self::OAuthData,
synced: &mut Vec<crate::sync::TriggerSyncInfo>,
errors: &mut Vec<crate::sync::SyncError>,
) {
let external_triggers = match self.list_all(workspace_id, oauth_data, db).await {
Ok(triggers) => triggers,
Err(e) => {
tracing::error!(
"Failed to fetch external triggers for {}: {}",
workspace_id,
e
);
errors.push(crate::sync::SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to fetch external triggers: {}", e),
error_type: "external_service_error".to_string(),
});
return;
}
};
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
// Convert to (external_id, config_json) pairs for reconciliation
let external_pairs: Vec<(String, serde_json::Value)> = external_triggers
.iter()
.filter_map(|data| {
let config = serde_json::to_value(data).ok()?;
Some((data.id.to_string(), config))
})
.collect();
let _ = self
.http_client_request::<serde_json::Value, ()>(
&url,
Method::GET,
w_id,
tx,
db,
Some(headers),
None,
)
.await?;
Ok(true)
crate::sync::reconcile_with_external_state(
db,
workspace_id,
ServiceName::Nextcloud,
triggers,
&external_pairs,
synced,
errors,
)
.await;
}
fn external_id_and_metadata_from_response(
&self,
resp: &Self::CreateResponse,
) -> (String, Option<serde_json::Value>) {
(resp.id.to_string(), None)
}
fn additional_routes(&self) -> axum::Router {
routes::nextcloud_routes(self.clone())
}
}
impl NextCloud {
async fn list_all(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
oauth_data: &NextCloudOAuthData,
db: &DB,
tx: &mut PgConnection,
) -> Result<Vec<Self::TriggerData>> {
) -> Result<Vec<NextCloudTriggerData>> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks",
oauth_data.base_url
@@ -277,7 +316,6 @@ impl External for NextCloud {
&url,
Method::GET,
w_id,
tx,
db,
Some(headers),
None,
@@ -286,19 +324,4 @@ impl External for NextCloud {
Ok(ocs_response.ocs.data)
}
fn external_id_and_metadata_from_response(
&self,
resp: &Self::CreateResponse,
) -> (String, Option<serde_json::Value>) {
(resp.id.to_string(), None)
}
fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String {
data.id.to_string()
}
fn additional_routes(&self) -> axum::Router {
routes::nextcloud_routes(self.clone())
}
}

View File

@@ -3,7 +3,6 @@ use std::{collections::HashMap, sync::Arc};
use axum::{extract::Path, routing::get, Extension, Json, Router};
use http::Method;
use windmill_common::{
db::UserDB,
error::{Error, JsonResult},
DB,
};
@@ -11,27 +10,25 @@ use windmill_common::{
use crate::{
get_workspace_integration,
nextcloud::{NextCloudEventType, OcsResponse},
External, OAuthConfig, ServiceName,
External, ServiceName,
};
use windmill_api_auth::ApiAuthed;
async fn list_available_events<T: External>(
authed: ApiAuthed,
Extension(handler): Extension<Arc<T>>,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(workspace_id): Path<String>,
) -> JsonResult<Vec<NextCloudEventType>> {
let mut tx = user_db.clone().begin(&authed).await?;
let integration =
get_workspace_integration(&mut *tx, &workspace_id, ServiceName::Nextcloud).await?;
let integration = get_workspace_integration(&db, &workspace_id, ServiceName::Nextcloud).await?;
let auth = serde_json::from_value::<OAuthConfig>(integration.oauth_data)
.map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud OAuth data: {}", e)))?;
let base_url = integration
.oauth_data
.get("base_url")
.and_then(|v| v.as_str())
.unwrap_or("");
let url = format!(
"{}/ocs/v2.php/apps/integration_windmill/api/v1/list/events",
&auth.base_url,
base_url,
);
let mut headers = HashMap::new();
@@ -42,13 +39,11 @@ async fn list_available_events<T: External>(
&url,
Method::GET,
&workspace_id,
&mut *tx,
&db,
Some(headers),
None,
)
.await?;
tx.commit().await?;
let events = serde_json::from_str(&ocs_response.ocs.data)
.map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud events data: {}", e)))?;

View File

@@ -9,7 +9,7 @@ use crate::ServiceName;
#[cfg(feature = "native_trigger")]
use crate::{
decrypt_oauth_data, list_native_triggers, update_native_trigger_error,
update_native_trigger_service_config, External,
update_native_trigger_service_config, External, NativeTrigger,
};
#[derive(Debug, Serialize)]
@@ -60,19 +60,20 @@ pub async fn sync_all_triggers(db: &DB) -> Result<BackgroundSyncResult> {
// Each service only syncs workspaces that have the corresponding integration configured
#[cfg(feature = "native_trigger")]
{
use crate::google::Google;
use crate::nextcloud::NextCloud;
// Nextcloud sync
let (service_name, result) = sync_service_triggers(db, NextCloud).await;
total_synced += result.synced_triggers.len();
total_errors += result.errors.len();
service_results.insert(service_name, result);
// Add new services here:
// use crate::newservice::NewService;
// let (service_name, result) = sync_service_triggers(db, NewService).await;
// total_synced += result.synced_triggers.len();
// total_errors += result.errors.len();
// service_results.insert(service_name, result);
// Google sync (handles both Drive and Calendar triggers)
let (service_name, result) = sync_service_triggers(db, Google).await;
total_synced += result.synced_triggers.len();
total_errors += result.errors.len();
service_results.insert(service_name, result);
}
// Count unique workspaces processed across all services
@@ -105,6 +106,9 @@ async fn sync_service_triggers<T: External>(
let mut all_synced_triggers = Vec::new();
let mut all_errors = Vec::new();
// Use the integration service for lookup (e.g., GoogleDrive/GoogleCalendar -> Google)
let integration_service = T::SERVICE_NAME.integration_service();
// Only sync workspaces that have the corresponding integration configured
let workspaces_with_integration = match sqlx::query_scalar!(
r#"
@@ -115,7 +119,7 @@ async fn sync_service_triggers<T: External>(
AND wi.oauth_data IS NOT NULL
AND w.deleted = false
"#,
T::SERVICE_NAME as ServiceName
integration_service as ServiceName
)
.fetch_all(db)
.await
@@ -210,11 +214,14 @@ pub async fn sync_workspace_triggers<T: External>(
return Ok((Vec::new(), Vec::new()));
}
let mut all_synced_triggers = Vec::new();
let mut all_sync_errors = Vec::new();
let mut synced = Vec::new();
let mut errors = Vec::new();
// Use the integration service for OAuth lookup (e.g., GoogleDrive/GoogleCalendar -> Google)
let integration_service = T::SERVICE_NAME.integration_service();
let oauth_data = {
match decrypt_oauth_data(db, db, workspace_id, T::SERVICE_NAME).await {
match decrypt_oauth_data(db, workspace_id, integration_service).await {
Ok(oauth_data) => oauth_data,
Err(e) => {
tracing::error!(
@@ -222,46 +229,58 @@ pub async fn sync_workspace_triggers<T: External>(
workspace_id,
e
);
all_sync_errors.push(SyncError {
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to get workspace integration OAuth data: {}", e),
error_type: "oauth_error".to_string(),
});
return Ok((Vec::new(), all_sync_errors));
return Ok((Vec::new(), errors));
}
}
};
let mut tx = db.begin().await?;
let external_triggers = match handler
.list_all(workspace_id, &oauth_data, db, &mut tx)
.await
{
Ok(triggers) => triggers,
Err(e) => {
tracing::error!(
"Failed to fetch external triggers for {}: {}",
workspace_id,
e
);
all_sync_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to fetch external triggers: {}", e),
error_type: "external_service_error".to_string(),
});
return Ok((Vec::new(), all_sync_errors));
}
};
tx.commit().await?;
handler
.maintain_triggers(
db,
workspace_id,
&windmill_triggers,
&oauth_data,
&mut synced,
&mut errors,
)
.await;
// Build a map of external trigger IDs to their data
let mut external_trigger_map: HashMap<String, &T::TriggerData> = HashMap::new();
for external_trigger in &external_triggers {
let external_id = handler.get_external_id_from_trigger_data(external_trigger);
external_trigger_map.insert(external_id, external_trigger);
tracing::info!(
"Sync completed for {} in workspace '{}'. Updated: {}, Errors: {}",
T::SERVICE_NAME.as_str(),
workspace_id,
synced.len(),
errors.len()
);
Ok((synced, errors))
}
/// Reusable reconciliation logic for services with real external state (e.g. Nextcloud).
/// Compares external triggers with DB triggers: sets errors for missing ones,
/// clears errors and updates config for existing ones.
#[cfg(feature = "native_trigger")]
pub async fn reconcile_with_external_state(
db: &DB,
workspace_id: &str,
service_name: ServiceName,
windmill_triggers: &[NativeTrigger],
external_triggers: &[(String, serde_json::Value)],
synced: &mut Vec<TriggerSyncInfo>,
errors: &mut Vec<SyncError>,
) {
// Build a map of external trigger IDs to their config
let mut external_trigger_map: HashMap<String, &serde_json::Value> = HashMap::new();
for (external_id, config) in external_triggers {
external_trigger_map.insert(external_id.clone(), config);
}
for trigger in &windmill_triggers {
for trigger in windmill_triggers {
if !external_trigger_map.contains_key(&trigger.external_id) {
// Trigger no longer exists on external service - set error
let error_msg = "Trigger no longer exists on external service".to_string();
@@ -276,14 +295,14 @@ pub async fn sync_workspace_triggers<T: External>(
match update_native_trigger_error(
db,
workspace_id,
T::SERVICE_NAME,
service_name,
&trigger.external_id,
Some(&error_msg),
)
.await
{
Ok(()) => {
all_synced_triggers.push(TriggerSyncInfo {
synced.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ErrorSet(error_msg),
@@ -295,7 +314,7 @@ pub async fn sync_workspace_triggers<T: External>(
trigger.external_id,
e
);
all_sync_errors.push(SyncError {
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to update error for trigger (external_id: '{}'): {}",
@@ -308,7 +327,7 @@ pub async fn sync_workspace_triggers<T: External>(
}
} else {
// Trigger exists on external service
let external_trigger_data = external_trigger_map.get(&trigger.external_id).unwrap();
let external_service_config = external_trigger_map.get(&trigger.external_id).unwrap();
// Clear error if it was set
if trigger.error.is_some() {
@@ -321,14 +340,14 @@ pub async fn sync_workspace_triggers<T: External>(
match update_native_trigger_error(
db,
workspace_id,
T::SERVICE_NAME,
service_name,
&trigger.external_id,
None,
)
.await
{
Ok(()) => {
all_synced_triggers.push(TriggerSyncInfo {
synced.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ErrorCleared,
@@ -340,7 +359,7 @@ pub async fn sync_workspace_triggers<T: External>(
trigger.external_id,
e
);
all_sync_errors.push(SyncError {
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to clear error for trigger (external_id: '{}'): {}",
@@ -353,14 +372,12 @@ pub async fn sync_workspace_triggers<T: External>(
}
// Compare service_config and update if different
let external_service_config =
handler.extract_service_config_from_trigger_data(external_trigger_data)?;
let stored_service_config = trigger
.service_config
.clone()
.unwrap_or(serde_json::Value::Null);
if external_service_config != stored_service_config {
if **external_service_config != stored_service_config {
tracing::info!(
"Trigger (external_id: '{}', script_path: '{}') config differs from external service, updating local config",
trigger.external_id,
@@ -370,14 +387,14 @@ pub async fn sync_workspace_triggers<T: External>(
match update_native_trigger_service_config(
db,
workspace_id,
T::SERVICE_NAME,
service_name,
&trigger.external_id,
&external_service_config,
external_service_config,
)
.await
{
Ok(()) => {
all_synced_triggers.push(TriggerSyncInfo {
synced.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ConfigUpdated,
@@ -389,7 +406,7 @@ pub async fn sync_workspace_triggers<T: External>(
trigger.external_id,
e
);
all_sync_errors.push(SyncError {
errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to update config for trigger (external_id: '{}'): {}",
@@ -408,14 +425,4 @@ pub async fn sync_workspace_triggers<T: External>(
}
}
}
tracing::info!(
"Sync completed for {} in workspace '{}'. Updated: {}, Errors: {}",
T::SERVICE_NAME.as_str(),
workspace_id,
all_synced_triggers.len(),
all_sync_errors.len()
);
Ok((all_synced_triggers, all_sync_errors))
}

View File

@@ -23,7 +23,8 @@ use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::{
db::UserDB,
error::{Error, JsonResult, Result},
utils::require_admin,
global_settings::{load_value_from_global_settings, OAUTH_SETTING},
utils::{require_admin, HTTP_CLIENT},
variables::{build_crypt, encrypt},
DB,
};
@@ -32,10 +33,10 @@ use windmill_common::{
use windmill_api_auth::ApiAuthed;
#[cfg(feature = "native_trigger")]
use crate::ServiceName;
#[cfg(feature = "native_trigger")]
use crate::{delete_workspace_integration, store_workspace_integration};
use crate::{
decrypt_oauth_data, delete_token_by_prefix, delete_workspace_integration, resolve_endpoint,
store_workspace_integration, ServiceName,
};
#[cfg(feature = "native_trigger")]
use windmill_oauth::{OClient, Url, OAUTH_HTTP_CLIENT};
@@ -45,6 +46,8 @@ use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
#[cfg(feature = "native_trigger")]
use hmac::{Hmac, Mac};
#[cfg(feature = "native_trigger")]
use serde_json::json;
#[cfg(feature = "native_trigger")]
use sha2::Sha256;
#[cfg(feature = "native_trigger")]
@@ -173,10 +176,14 @@ pub struct ConnectIntegrationResponse {
#[cfg(feature = "native_trigger")]
#[derive(FromRow, Debug, Clone, Serialize, Deserialize)]
pub struct WorkspaceOAuthConfig {
#[serde(default)]
pub client_id: String,
#[serde(default)]
pub client_secret: String,
#[serde(default)]
pub base_url: String,
pub access_token: Option<String>,
#[serde(default)]
pub instance_shared: bool,
}
#[cfg(feature = "native_trigger")]
@@ -201,20 +208,112 @@ async fn generate_connect_url(
// Generate a signed state that is cluster-safe
let state = generate_signed_state(&db, &workspace_id, service_name).await?;
let auth_url = build_authorization_url(&oauth_config, &state, &redirect_uri);
let auth_url = build_authorization_url(&oauth_config, service_name, &state, &redirect_uri);
Ok(Json(auth_url))
}
#[cfg(feature = "native_trigger")]
#[derive(Debug, Deserialize)]
struct BasicOAuthData {
base_url: String,
access_token: String,
}
#[cfg(feature = "native_trigger")]
async fn try_delete_nextcloud_webhook(base_url: &str, access_token: &str, external_id: &str) {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
base_url, external_id
);
let _ = HTTP_CLIENT
.delete(&url)
.bearer_auth(access_token)
.header("OCS-APIRequest", "true")
.send()
.await;
}
/// Delete all native triggers for a workspace+service, including remote webhook cleanup.
/// This is best-effort: errors during remote cleanup or token deletion are logged but ignored.
#[cfg(feature = "native_trigger")]
async fn delete_triggers_for_service(db: &DB, workspace_id: &str, service_name: ServiceName) {
let triggers = sqlx::query!(
"SELECT external_id, webhook_token_prefix FROM native_trigger WHERE workspace_id = $1 AND service_name = $2",
workspace_id,
service_name as ServiceName
)
.fetch_all(db)
.await;
let triggers = match triggers {
Ok(t) => t,
Err(e) => {
tracing::error!("Failed to fetch native triggers for service {service_name:?} in workspace {workspace_id}: {e}");
return;
}
};
if triggers.is_empty() {
return;
}
// For Nextcloud: try to delete webhooks on the remote instance (best-effort)
if service_name == ServiceName::Nextcloud {
if let Ok(oauth_data) =
decrypt_oauth_data::<BasicOAuthData>(db, workspace_id, service_name).await
{
for trigger in &triggers {
try_delete_nextcloud_webhook(
&oauth_data.base_url,
&oauth_data.access_token,
&trigger.external_id,
)
.await;
}
}
}
// For Google: skip remote cleanup (watch channels expire naturally)
// Bulk delete all triggers
if let Err(e) = sqlx::query!(
"DELETE FROM native_trigger WHERE workspace_id = $1 AND service_name = $2",
workspace_id,
service_name as ServiceName
)
.execute(db)
.await
{
tracing::error!("Failed to delete native triggers for service {service_name:?} in workspace {workspace_id}: {e}");
}
// Delete all associated webhook tokens
for trigger in &triggers {
if let Err(e) = delete_token_by_prefix(db, &trigger.webhook_token_prefix).await {
tracing::error!(
"Failed to delete webhook token with prefix {}: {e}",
trigger.webhook_token_prefix
);
}
}
}
#[cfg(feature = "native_trigger")]
async fn delete_integration(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
) -> JsonResult<String> {
require_admin(authed.is_admin, &workspace_id)?;
// Delete triggers first (needs OAuth data that cleanup_oauth_resource will remove)
delete_triggers_for_service(&db, &workspace_id, service_name).await;
let mut tx = user_db.begin(&authed).await?;
// Clean up account+variable+resource
cleanup_oauth_resource(&mut *tx, &workspace_id, service_name).await;
let deleted = delete_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
if !deleted {
@@ -248,6 +347,7 @@ async fn delete_integration(
struct WorkspaceIntegrations {
service_name: ServiceName,
oauth_data: Option<sqlx::types::Json<WorkspaceOAuthConfig>>,
resource_path: Option<String>,
}
#[cfg(feature = "native_trigger")]
@@ -263,8 +363,9 @@ async fn list_integrations(
WorkspaceIntegrations,
r#"
SELECT
oauth_data as "oauth_data!: sqlx::types::Json<WorkspaceOAuthConfig>",
service_name as "service_name!: ServiceName"
oauth_data as "oauth_data: sqlx::types::Json<WorkspaceOAuthConfig>",
service_name as "service_name!: ServiceName",
resource_path
FROM
workspace_integrations
WHERE
@@ -277,14 +378,23 @@ async fn list_integrations(
let key_value = integrations
.into_iter()
.map(|integration| (integration.service_name, integration.oauth_data))
.map(|integration| {
(
integration.service_name,
(integration.oauth_data, integration.resource_path),
)
})
.collect::<std::collections::HashMap<_, _>>();
use strum::IntoEnumIterator;
let integrations = ServiceName::iter()
.map(|service_name| WorkspaceIntegrations {
service_name: service_name,
oauth_data: key_value.get(&service_name).cloned().flatten(),
.map(|service_name| {
let (oauth_data, resource_path) = key_value
.get(&service_name)
.cloned()
.map(|(od, rp)| (od, rp))
.unwrap_or((None, None));
WorkspaceIntegrations { service_name, oauth_data, resource_path }
})
.collect::<Vec<_>>();
@@ -304,10 +414,10 @@ async fn integration_exist(
r#"
SELECT EXISTS (
SELECT 1
FROM workspace_integrations
WHERE workspace_id = $1
AND service_name = $2
AND oauth_data IS NOT NULL
FROM workspace_integrations wi
WHERE wi.workspace_id = $1
AND wi.service_name = $2
AND wi.oauth_data IS NOT NULL
)
"#,
workspace_id,
@@ -326,18 +436,26 @@ struct RedirectUri {
redirect_uri: String,
}
#[cfg(feature = "native_trigger")]
#[derive(Debug, Deserialize)]
struct OAuthCallbackBody {
redirect_uri: String,
code: String,
state: String,
resource_path: Option<String>,
}
#[cfg(feature = "native_trigger")]
async fn oauth_callback(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name, code, state)): Path<(String, ServiceName, String, String)>,
Json(RedirectUri { redirect_uri }): Json<RedirectUri>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
Json(body): Json<OAuthCallbackBody>,
) -> JsonResult<String> {
require_admin(authed.is_admin, &workspace_id)?;
// Validate the signed state (cluster-safe, no DB storage needed)
let state_was_valid = validate_signed_state(&db, &state, &workspace_id).await?;
let state_was_valid = validate_signed_state(&db, &body.state, &workspace_id).await?;
if !state_was_valid {
return Err(Error::BadRequest(
@@ -345,31 +463,122 @@ async fn oauth_callback(
));
}
let oauth_config =
get_workspace_oauth_config::<WorkspaceOAuthConfig>(&db, &workspace_id, service_name)
.await?;
// Check if this integration uses instance-shared credentials
let existing_oauth_data = sqlx::query_scalar!(
r#"SELECT oauth_data FROM workspace_integrations
WHERE workspace_id = $1 AND service_name = $2"#,
workspace_id,
service_name as ServiceName
)
.fetch_optional(&db)
.await?
.flatten();
let is_instance_shared = existing_oauth_data
.as_ref()
.and_then(|v| v.get("instance_shared"))
.and_then(|v| v.as_bool())
.unwrap_or(false);
let oauth_config = if is_instance_shared {
get_instance_oauth_config(&db, service_name).await?
} else {
get_workspace_oauth_config::<WorkspaceOAuthConfig>(&db, &workspace_id, service_name).await?
};
let token_response =
exchange_code_for_token(&oauth_config, service_name, &code, &redirect_uri).await?;
exchange_code_for_token(&oauth_config, service_name, &body.code, &body.redirect_uri)
.await?;
let resource_path = body
.resource_path
.filter(|p| !p.is_empty())
.unwrap_or_else(|| {
format!(
"u/{}/native_{}",
authed.username,
service_name.resource_type()
)
});
let expires_in = token_response.expires_in.unwrap_or(3600);
let mut tx = user_db.begin(&authed).await?;
// Clean up any previous account+variable+resource for this integration
cleanup_oauth_resource(&mut *tx, &workspace_id, service_name).await;
// 1. Create account record for token refresh
let account_id = sqlx::query_scalar!(
"INSERT INTO account (workspace_id, client, expires_at, refresh_token, is_workspace_integration)
VALUES ($1, $2, now() + ($3 || ' seconds')::interval, $4, true)
RETURNING id",
workspace_id,
service_name.as_str(),
expires_in.to_string(),
token_response.refresh_token.as_deref().unwrap_or(""),
)
.fetch_one(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("Failed to create account: {}", e)))?;
// 2. Create variable with encrypted access token
let mc = build_crypt(&db, &workspace_id).await?;
let mut oauth_data = serde_json::to_value(oauth_config).unwrap();
let encrypted_access_token = encrypt(&mc, &token_response.access_token);
oauth_data["access_token"] = serde_json::Value::String(encrypted_access_token);
if let Some(refresh_token) = token_response.refresh_token {
let encrypted_refresh_token = encrypt(&mc, &refresh_token);
oauth_data["refresh_token"] = serde_json::Value::String(encrypted_refresh_token);
}
if let Some(expires_in) = token_response.expires_in {
let expires_at = chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64);
oauth_data["token_expires_at"] = serde_json::Value::String(expires_at.to_rfc3339());
}
sqlx::query!(
"INSERT INTO variable (workspace_id, path, value, is_secret, description, account, is_oauth)
VALUES ($1, $2, $3, true, $4, $5, true)
ON CONFLICT (workspace_id, path) DO UPDATE
SET value = EXCLUDED.value, account = EXCLUDED.account",
workspace_id,
resource_path,
encrypted_access_token,
format!("OAuth token for {} workspace integration", service_name),
account_id,
)
.execute(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("Failed to create variable: {}", e)))?;
store_workspace_integration(&mut *tx, &authed, &workspace_id, service_name, oauth_data).await?;
// 3. Create resource pointing to the variable
let resource_value = json!({ "token": format!("$var:{}", resource_path) });
sqlx::query!(
"INSERT INTO resource (workspace_id, path, value, resource_type, description, created_by)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (workspace_id, path) DO UPDATE
SET value = EXCLUDED.value, resource_type = EXCLUDED.resource_type",
workspace_id,
resource_path,
resource_value,
service_name.resource_type(),
format!("{} workspace integration", service_name),
authed.username,
)
.execute(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("Failed to create resource: {}", e)))?;
// 4. Store config + resource_path in workspace_integrations (no tokens).
// For instance-shared integrations, store the flag instead of credentials.
let stored_data = if is_instance_shared {
json!({
"instance_shared": true,
"base_url": "",
})
} else {
to_value(&oauth_config).unwrap()
};
store_workspace_integration(
&mut *tx,
&authed,
&workspace_id,
service_name,
stored_data,
Some(&resource_path),
)
.await?;
audit_log(
&mut *tx,
@@ -407,16 +616,14 @@ fn build_native_oauth_client(
service_name: ServiceName,
redirect_uri: &str,
) -> Result<OClient> {
let auth_url = Url::parse(&format!(
"{}{}",
config.base_url,
service_name.auth_endpoint()
let auth_url = Url::parse(&resolve_endpoint(
&config.base_url,
service_name.auth_endpoint(),
))
.map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?;
let token_url = Url::parse(&format!(
"{}{}",
config.base_url,
service_name.token_endpoint()
let token_url = Url::parse(&resolve_endpoint(
&config.base_url,
service_name.token_endpoint(),
))
.map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?;
let redirect = Url::parse(redirect_uri).map_err(|e| {
@@ -459,7 +666,7 @@ async fn get_workspace_oauth_config<T: DeserializeOwned>(
workspace_id: &str,
service_name: ServiceName,
) -> Result<T> {
let oauth_configs = sqlx::query_scalar!(
let oauth_data = sqlx::query_scalar!(
r#"
SELECT
oauth_data
@@ -474,15 +681,14 @@ async fn get_workspace_oauth_config<T: DeserializeOwned>(
)
.fetch_optional(db)
.await?
.flatten()
.ok_or(Error::NotFound(format!(
"Integration for service {} not found",
service_name.as_str()
)))?;
let config = serde_json::from_value::<T>(oauth_configs)
.map_err(|e| Error::InternalErr(format!("Failed to parse OAuth config: {}", e)))?;
Ok(config)
serde_json::from_value::<T>(oauth_data)
.map_err(|e| Error::InternalErr(format!("Failed to parse OAuth config: {}", e)))
}
#[cfg(feature = "native_trigger")]
@@ -502,6 +708,7 @@ pub async fn create_workspace_integration(
&workspace_id,
service_name,
to_value(oauth_data).unwrap(),
None,
)
.await?;
@@ -523,24 +730,193 @@ async fn get_workspace_oauth_config_as_oauth_config(
#[cfg(feature = "native_trigger")]
fn build_authorization_url(
config: &WorkspaceOAuthConfig,
service_name: ServiceName,
state: &str,
redirect_uri: &str,
) -> String {
let params = [
let base_auth_url = resolve_endpoint(&config.base_url, service_name.auth_endpoint());
let mut params = vec![
("response_type", "code"),
("client_id", &config.client_id),
("client_id", config.client_id.as_str()),
("redirect_uri", redirect_uri),
("state", state),
("scope", "read write"),
("scope", service_name.oauth_scopes()),
];
for &(key, value) in service_name.extra_auth_params() {
params.push((key, value));
}
let query_string = params
.iter()
.map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(v)))
.collect::<Vec<_>>()
.join("&");
format!("{}/apps/oauth2/authorize?{}", config.base_url, query_string)
format!("{}?{}", base_auth_url, query_string)
}
#[cfg(feature = "native_trigger")]
pub async fn cleanup_oauth_resource(
tx: &mut sqlx::PgConnection,
workspace_id: &str,
service_name: ServiceName,
) {
// Look up the stored resource_path from workspace_integrations
let stored_resource_path: Option<String> = sqlx::query_scalar!(
r#"SELECT resource_path FROM workspace_integrations WHERE workspace_id = $1 AND service_name = $2"#,
workspace_id,
service_name as ServiceName,
)
.fetch_optional(&mut *tx)
.await
.ok()
.flatten()
.flatten();
// Find and delete any existing account+variable+resource for this integration
let account_ids: Vec<i32> = sqlx::query_scalar!(
"DELETE FROM account WHERE workspace_id = $1 AND client = $2 AND is_workspace_integration = true RETURNING id",
workspace_id,
service_name.as_str(),
)
.fetch_all(&mut *tx)
.await
.unwrap_or_default();
if !account_ids.is_empty() {
// Delete variables linked to these accounts
let _ = sqlx::query!(
"DELETE FROM variable WHERE workspace_id = $1 AND account = ANY($2)",
workspace_id,
&account_ids,
)
.execute(&mut *tx)
.await;
}
// Delete resource by exact stored path, or fall back to legacy pattern
let resource_type = service_name.resource_type();
if let Some(ref path) = stored_resource_path {
let _ = sqlx::query!(
"DELETE FROM resource WHERE workspace_id = $1 AND path = $2",
workspace_id,
path,
)
.execute(&mut *tx)
.await;
} else {
// Legacy fallback for integrations created before user-chosen paths
let _ = sqlx::query!(
"DELETE FROM resource WHERE workspace_id = $1 AND resource_type = $2 AND path LIKE 'u/%/native_%'",
workspace_id,
resource_type,
)
.execute(&mut *tx)
.await;
}
}
/// Check if the instance admin has enabled sharing of OAuth credentials for a given service.
/// Currently only supported for Google (gworkspace).
#[cfg(feature = "native_trigger")]
async fn is_instance_sharing_enabled(db: &DB, service_name: ServiceName) -> Result<bool> {
// Only Google supports instance sharing for now
if service_name != ServiceName::Google {
return Ok(false);
}
let oauths_value = match load_value_from_global_settings(db, OAUTH_SETTING).await? {
Some(v) => v,
None => return Ok(false),
};
let key = service_name.resource_type(); // "gworkspace"
let entry = match oauths_value.get(key) {
Some(v) => v,
None => return Ok(false),
};
let id = entry.get("id").and_then(|v| v.as_str()).unwrap_or("");
let secret = entry.get("secret").and_then(|v| v.as_str()).unwrap_or("");
let share = entry
.get("share_with_workspaces")
.and_then(|v| v.as_bool())
.unwrap_or(false);
Ok(!id.is_empty() && !secret.is_empty() && share)
}
/// Read instance-level OAuth credentials for a service (when sharing is enabled).
#[cfg(feature = "native_trigger")]
async fn get_instance_oauth_config(
db: &DB,
service_name: ServiceName,
) -> Result<WorkspaceOAuthConfig> {
if !is_instance_sharing_enabled(db, service_name).await? {
return Err(Error::BadRequest(
"Instance credential sharing is not enabled for this service".to_string(),
));
}
let (client_id, client_secret) =
windmill_common::global_settings::get_instance_oauth_credentials(
db,
service_name.resource_type(),
)
.await?;
Ok(WorkspaceOAuthConfig {
client_id,
client_secret,
base_url: String::new(), // Google uses absolute URLs
instance_shared: false,
})
}
#[cfg(feature = "native_trigger")]
async fn check_instance_sharing_available(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
) -> JsonResult<bool> {
require_admin(authed.is_admin, &workspace_id)?;
let available = is_instance_sharing_enabled(&db, service_name).await?;
Ok(Json(available))
}
#[cfg(feature = "native_trigger")]
async fn generate_instance_connect_url(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
Json(RedirectUri { redirect_uri }): Json<RedirectUri>,
) -> JsonResult<String> {
require_admin(authed.is_admin, &workspace_id)?;
let instance_config = get_instance_oauth_config(&db, service_name).await?;
// Store a marker in workspace_integrations — NOT the actual credentials.
// The callback and token refresh will read credentials from global settings
// when they see instance_shared=true.
let mut tx = user_db.begin(&authed).await?;
crate::store_workspace_integration(
&mut tx,
&authed,
&workspace_id,
service_name,
json!({ "instance_shared": true, "base_url": "" }),
None,
)
.await?;
tx.commit().await?;
// Generate signed state and build authorization URL
let state = generate_signed_state(&db, &workspace_id, service_name).await?;
let auth_url = build_authorization_url(&instance_config, service_name, &state, &redirect_uri);
Ok(Json(auth_url))
}
#[cfg(feature = "native_trigger")]
@@ -553,8 +929,16 @@ pub fn workspaced_service() -> Router {
"/:service_name/generate_connect_url",
post(generate_connect_url),
)
.route(
"/:service_name/instance_sharing_available",
get(check_instance_sharing_available),
)
.route(
"/:service_name/generate_instance_connect_url",
post(generate_instance_connect_url),
)
.route("/:service_name/delete", delete(delete_integration))
.route("/:service_name/callback/:code/:state", post(oauth_callback));
.route("/:service_name/callback", post(oauth_callback));
Router::new().nest("/integrations", router)
}

View File

@@ -0,0 +1,30 @@
[package]
name = "windmill-operator"
version.workspace = true
authors.workspace = true
edition.workspace = true
[lib]
name = "windmill_operator"
path = "./src/lib.rs"
[features]
enterprise = ["windmill-common/enterprise"]
private = []
default = []
[dependencies]
serde.workspace = true
serde_json.workspace = true
sqlx.workspace = true
tracing.workspace = true
windmill-common = { workspace = true, default-features = false, features = ["instance_config_schema"] }
anyhow.workspace = true
thiserror.workspace = true
kube.workspace = true
k8s-openapi.workspace = true
tokio.workspace = true
futures.workspace = true
chrono.workspace = true
schemars = "0.8"
serde_yml.workspace = true

View File

@@ -0,0 +1,703 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: windmillinstances.windmill.dev
spec:
group: windmill.dev
names:
categories: []
kind: WindmillInstance
plural: windmillinstances
shortNames:
- wmi
singular: windmillinstance
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: '.status.synced'
name: Synced
type: string
- jsonPath: '.status.lastSyncedAt'
name: Last Synced
type: date
- jsonPath: '.metadata.creationTimestamp'
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: Auto-generated derived type for WindmillInstanceSpec via `CustomResource`
properties:
spec:
description: |-
WindmillInstance CRD spec.
Declares the desired state for instance-level configuration: - `global_settings` maps directly to the `global_settings` table - `worker_configs` maps to the `config` table with a `worker__` prefix
properties:
global_settings:
default: {}
description: Global settings to sync to the `global_settings` table.
properties:
app_workspaced_route:
nullable: true
type: boolean
base_url:
nullable: true
type: string
bunfig_install_scopes:
nullable: true
type: string
critical_alert_mute_ui:
nullable: true
type: boolean
critical_alerts_on_db_oversize:
description: Configuration for critical alerts when the database exceeds a size threshold.
nullable: true
properties:
enabled:
default: false
type: boolean
value:
default: 0.0
format: float
type: number
type: object
critical_error_channels:
items:
anyOf:
- required:
- email
- required:
- slack_channel
- required:
- teams_channel
description: A channel for delivering critical error alerts.
properties:
email:
type: string
slack_channel:
type: string
teams_channel:
description: Microsoft Teams channel reference.
properties:
channel_id:
type: string
channel_name:
type: string
team_id:
type: string
team_name:
type: string
required:
- channel_id
- channel_name
- team_id
- team_name
type: object
type: object
nullable: true
type: array
custom_instance_pg_databases:
description: Custom PostgreSQL databases managed by the instance.
nullable: true
properties:
databases:
additionalProperties:
description: Status of a single custom instance database.
properties:
error:
nullable: true
type: string
logs:
default:
super_admin: ''
description: Setup log entries for a custom instance database.
properties:
created_database:
type: string
database_credentials:
type: string
db_connect:
type: string
grant_permissions:
type: string
super_admin:
default: ''
type: string
valid_dbname:
type: string
type: object
success:
default: false
type: boolean
tag:
nullable: true
type: string
type: object
type: object
user_pwd:
nullable: true
type: string
type: object
custom_tags:
items:
type: string
nullable: true
type: array
default_tags_per_workspace:
nullable: true
type: boolean
default_tags_workspaces:
items:
type: string
nullable: true
type: array
dev_instance:
nullable: true
type: boolean
disable_stats:
nullable: true
type: boolean
ducklake_settings:
description: DuckLake catalog database settings.
nullable: true
properties:
ducklakes:
additionalProperties:
description: A single DuckLake instance configuration.
properties:
catalog:
description: DuckLake catalog backend reference.
properties:
resource_path:
type: string
resource_type:
description: The type of database backing a DuckLake catalog.
enum:
- postgresql
- mysql
- instance
type: string
required:
- resource_path
- resource_type
type: object
extra_args:
nullable: true
type: string
storage:
description: DuckLake storage location.
properties:
path:
type: string
storage:
nullable: true
type: string
required:
- path
type: object
required:
- catalog
- storage
type: object
type: object
required:
- ducklakes
type: object
email_domain:
nullable: true
type: string
expose_debug_metrics:
nullable: true
type: boolean
expose_metrics:
nullable: true
type: boolean
hub_accessible_url:
nullable: true
type: string
hub_api_secret:
nullable: true
type: string
hub_base_url:
nullable: true
type: string
indexer_settings:
description: Full-text search indexer configuration.
nullable: true
properties:
commit_job_max_batch_size:
format: uint64
minimum: 0.0
nullable: true
type: integer
commit_log_max_batch_size:
format: uint64
minimum: 0.0
nullable: true
type: integer
max_indexed_job_log_size:
format: uint64
minimum: 0.0
nullable: true
type: integer
refresh_index_period:
format: uint64
minimum: 0.0
nullable: true
type: integer
refresh_log_index_period:
format: uint64
minimum: 0.0
nullable: true
type: integer
should_clear_job_index:
nullable: true
type: boolean
should_clear_log_index:
nullable: true
type: boolean
writer_memory_budget:
format: uint64
minimum: 0.0
nullable: true
type: integer
type: object
instance_python_version:
nullable: true
type: string
job_default_timeout:
format: int64
nullable: true
type: integer
jwt_secret:
nullable: true
type: string
keep_job_dir:
nullable: true
type: boolean
license_key:
nullable: true
type: string
maven_repos:
nullable: true
type: string
min_keep_alive_version:
nullable: true
type: string
monitor_logs_on_s3:
nullable: true
type: boolean
no_default_maven:
nullable: true
type: boolean
npm_config_registry:
nullable: true
type: string
nuget_config:
nullable: true
type: string
oauths:
additionalProperties:
description: OAuth client configuration for a single provider.
properties:
allowed_domains:
items:
type: string
nullable: true
type: array
connect_config:
description: OAuth provider endpoint configuration.
nullable: true
properties:
auth_url:
type: string
extra_params:
additionalProperties:
type: string
nullable: true
type: object
extra_params_callback:
additionalProperties:
type: string
nullable: true
type: object
req_body_auth:
nullable: true
type: boolean
scopes:
items:
type: string
nullable: true
type: array
token_url:
type: string
userinfo_url:
nullable: true
type: string
required:
- auth_url
- token_url
type: object
id:
type: string
login_config:
description: OAuth provider endpoint configuration.
nullable: true
properties:
auth_url:
type: string
extra_params:
additionalProperties:
type: string
nullable: true
type: object
extra_params_callback:
additionalProperties:
type: string
nullable: true
type: object
req_body_auth:
nullable: true
type: boolean
scopes:
items:
type: string
nullable: true
type: array
token_url:
type: string
userinfo_url:
nullable: true
type: string
required:
- auth_url
- token_url
type: object
secret:
type: string
required:
- id
- secret
type: object
nullable: true
type: object
object_store_cache_config:
nullable: true
openai_azure_base_path:
nullable: true
type: string
otel:
description: OpenTelemetry exporter configuration.
nullable: true
properties:
logs_enabled:
nullable: true
type: boolean
metrics_enabled:
nullable: true
type: boolean
otel_exporter_otlp_compression:
nullable: true
type: string
otel_exporter_otlp_endpoint:
nullable: true
type: string
otel_exporter_otlp_headers:
nullable: true
type: string
otel_exporter_otlp_protocol:
nullable: true
type: string
tracing_enabled:
nullable: true
type: boolean
type: object
otel_tracing_proxy:
description: Per-language HTTP request tracing proxy configuration.
nullable: true
properties:
enabled:
default: false
type: boolean
enabled_languages:
items:
description: Script language identifier.
enum:
- python3
- deno
- go
- bash
- powershell
- postgresql
- bun
- bunnative
- mysql
- bigquery
- snowflake
- graphql
- nativets
- mssql
- oracledb
- duckdb
- php
- rust
- ansible
- csharp
- nu
- java
- ruby
type: string
type: array
type: object
pip_extra_index_url:
nullable: true
type: string
pip_index_url:
nullable: true
type: string
powershell_repo_pat:
nullable: true
type: string
powershell_repo_url:
nullable: true
type: string
request_size_limit_mb:
format: int64
nullable: true
type: integer
require_preexisting_user_for_oauth:
nullable: true
type: boolean
retention_period_secs:
format: int64
nullable: true
type: integer
ruby_repos:
nullable: true
type: string
saml_metadata:
nullable: true
type: string
scim_token:
nullable: true
type: string
secret_backend:
nullable: true
slack:
nullable: true
smtp_settings:
description: SMTP server configuration.
nullable: true
properties:
smtp_disable_tls:
nullable: true
type: boolean
smtp_from:
nullable: true
type: string
smtp_host:
nullable: true
type: string
smtp_password:
nullable: true
type: string
smtp_port:
format: uint16
minimum: 0.0
nullable: true
type: integer
smtp_tls_implicit:
nullable: true
type: boolean
smtp_username:
nullable: true
type: string
type: object
teams:
nullable: true
timeout_wait_result:
format: int64
nullable: true
type: integer
type: object
x-kubernetes-preserve-unknown-fields: true
worker_configs:
additionalProperties:
description: Worker group configuration.
properties:
additional_python_paths:
items:
type: string
nullable: true
type: array
autoscaling:
description: Worker group autoscaling configuration.
nullable: true
properties:
cooldown_seconds:
format: uint64
minimum: 0.0
nullable: true
type: integer
custom_tags:
items:
type: string
nullable: true
type: array
dec_scale_occupancy_rate:
format: uint8
minimum: 0.0
nullable: true
type: integer
enabled:
default: false
type: boolean
full_scale_cooldown_seconds:
format: uint64
minimum: 0.0
nullable: true
type: integer
full_scale_jobs_waiting:
format: uint64
minimum: 0.0
nullable: true
type: integer
inc_num_workers:
format: uint32
minimum: 0.0
nullable: true
type: integer
inc_scale_num_jobs_waiting:
format: uint64
minimum: 0.0
nullable: true
type: integer
inc_scale_occupancy_rate:
format: uint8
minimum: 0.0
nullable: true
type: integer
integration:
description: |-
Autoscaling integration backend.
The `type` field selects the backend: `"script"`, `"dryrun"`, or `"kubernetes"`. For `"script"`, `path` is required and `tag` is optional.
nullable: true
properties:
path:
nullable: true
type: string
tag:
nullable: true
type: string
type:
type: string
required:
- type
type: object
max_workers:
format: uint32
minimum: 0.0
nullable: true
type: integer
min_workers:
format: uint32
minimum: 0.0
nullable: true
type: integer
type: object
cache_clear:
format: uint32
minimum: 0.0
nullable: true
type: integer
dedicated_worker:
nullable: true
type: string
dedicated_workers:
items:
type: string
nullable: true
type: array
env_vars_allowlist:
items:
type: string
nullable: true
type: array
env_vars_static:
additionalProperties:
type: string
nullable: true
type: object
init_bash:
nullable: true
type: string
min_alive_workers_alert_threshold:
format: uint32
minimum: 0.0
nullable: true
type: integer
periodic_script_bash:
nullable: true
type: string
periodic_script_interval_seconds:
format: uint64
minimum: 0.0
nullable: true
type: integer
pip_local_dependencies:
items:
type: string
nullable: true
type: array
priority_tags:
additionalProperties:
format: uint8
minimum: 0.0
type: integer
nullable: true
type: object
worker_tags:
items:
type: string
nullable: true
type: array
type: object
x-kubernetes-preserve-unknown-fields: true
default: {}
description: Worker group configs to sync to the `config` table. Keys are worker group names (e.g. "default", "gpu"). Each key is stored in the DB as `worker__<key>`.
type: object
type: object
status:
description: Status subresource for WindmillInstance.
nullable: true
properties:
lastSyncedAt:
description: Timestamp of the last successful sync.
nullable: true
type: string
message:
default: ''
description: Human-readable status message.
type: string
observedGeneration:
default: 0
description: The `.metadata.generation` that was last observed.
format: int64
type: integer
synced:
description: Whether the last reconciliation was successful.
type: boolean
required:
- synced
type: object
required:
- spec
title: WindmillInstance
type: object
served: true
storage: true
subresources:
status: {}

View File

@@ -0,0 +1,13 @@
#[cfg(feature = "private")]
pub mod crd_ee;
#[cfg(feature = "private")]
pub mod db_sync_ee;
#[cfg(feature = "private")]
pub use db_sync_ee as db_sync;
#[cfg(feature = "private")]
pub mod reconciler_ee;
#[cfg(feature = "private")]
pub mod resolve_ee;
mod operator_oss;
pub use operator_oss::*;

View File

@@ -0,0 +1,19 @@
#[cfg(feature = "private")]
pub use crate::reconciler_ee::run;
#[cfg(feature = "private")]
pub fn print_crd_yaml() {
use kube::CustomResourceExt;
let crd = crate::crd_ee::WindmillInstance::crd();
println!("{}", serde_yml::to_string(&crd).unwrap());
}
#[cfg(not(feature = "private"))]
pub async fn run(_db: sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
anyhow::bail!("K8s operator is not available in this build")
}
#[cfg(not(feature = "private"))]
pub fn print_crd_yaml() {
eprintln!("K8s operator CRD generation is not available in this build");
}

View File

@@ -8,6 +8,8 @@
#[cfg(feature = "private")]
pub use crate::oauth_refresh_ee::_refresh_token;
#[cfg(feature = "private")]
pub use crate::oauth_refresh_ee::_refresh_workspace_integration_token;
#[cfg(not(feature = "private"))]
use sqlx::{Postgres, Transaction};
@@ -36,3 +38,156 @@ pub async fn _refresh_token<'c>(
)
.await
}
#[cfg(not(feature = "private"))]
pub async fn _refresh_workspace_integration_token<'c>(
mut tx: Transaction<'c, Postgres>,
path: &str,
w_id: &str,
account_id: i32,
db: &DB,
client_name: &str,
refresh_token: &str,
) -> error::Result<String> {
use windmill_common::global_settings::{
get_instance_oauth_credentials, workspace_integration_auth_endpoint,
workspace_integration_oauth_key, workspace_integration_token_endpoint,
};
use windmill_common::utils::now_from_db;
use windmill_common::variables::{build_crypt, encrypt};
use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT};
tracing::info!(
client = %client_name,
workspace_id = %w_id,
account_id = %account_id,
"Refreshing workspace integration OAuth token"
);
let oauth_data: serde_json::Value = sqlx::query_scalar(
"SELECT oauth_data FROM workspace_integrations \
WHERE workspace_id = $1 AND service_name::text = $2",
)
.bind(w_id)
.bind(client_name)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(|| {
error::Error::NotFound(format!(
"Workspace integration for {} not found or not configured",
client_name
))
})?;
let is_instance_shared = oauth_data
.get("instance_shared")
.and_then(|v| v.as_bool())
.unwrap_or(false);
let (client_id, client_secret, base_url);
if is_instance_shared {
let oauth_key = workspace_integration_oauth_key(client_name);
let (id, secret) = get_instance_oauth_credentials(db, oauth_key).await?;
client_id = id;
client_secret = secret;
base_url = String::new();
} else {
client_id = oauth_data["client_id"]
.as_str()
.ok_or_else(|| {
error::Error::InternalErr("Missing client_id in workspace integration".into())
})?
.to_string();
client_secret = oauth_data["client_secret"]
.as_str()
.ok_or_else(|| {
error::Error::InternalErr(
"Missing client_secret in workspace integration".into(),
)
})?
.to_string();
base_url = oauth_data["base_url"].as_str().unwrap_or("").to_string();
}
let token_endpoint = workspace_integration_token_endpoint(client_name, &base_url);
let auth_endpoint = workspace_integration_auth_endpoint(client_name, &base_url);
let auth_url = Url::parse(&auth_endpoint)
.map_err(|e| error::Error::InternalErr(format!("Invalid auth URL: {}", e)))?;
let token_url = Url::parse(&token_endpoint)
.map_err(|e| error::Error::InternalErr(format!("Invalid token URL: {}", e)))?;
let mut client = OClient::new(client_id, auth_url, token_url);
client.set_client_secret(client_secret);
let token = client
.exchange_refresh_token(&RefreshToken::from(refresh_token))
.with_client(&*OAUTH_HTTP_CLIENT)
.execute::<serde_json::Value>()
.await
.map_err(|e| {
error::Error::InternalErr(format!(
"Failed to refresh workspace integration token: {:?}",
e
))
})?;
#[derive(serde::Deserialize)]
struct WsTokenResponse {
access_token: String,
refresh_token: Option<String>,
expires_in: Option<i64>,
}
let token_result: WsTokenResponse = serde_json::from_value(token)
.map_err(|e| error::Error::InternalErr(format!("Failed to parse token response: {}", e)))?;
let expires_at = now_from_db(&mut *tx).await?
+ chrono::Duration::try_seconds(
token_result
.expires_in
.ok_or_else(|| {
error::Error::InternalErr("expires_in expected and not found".into())
})?
.try_into()
.unwrap(),
)
.unwrap_or_default();
sqlx::query(
"UPDATE account SET refresh_token = $1, expires_at = $2, refresh_error = NULL \
WHERE workspace_id = $3 AND id = $4",
)
.bind(
token_result
.refresh_token
.as_deref()
.unwrap_or(refresh_token),
)
.bind(expires_at)
.bind(w_id)
.bind(account_id)
.execute(&mut *tx)
.await?;
tx.commit().await?;
let token_str = &token_result.access_token;
let mc = build_crypt(db, w_id).await?;
let encrypted_token = encrypt(&mc, token_str);
sqlx::query("UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3")
.bind(encrypted_token)
.bind(w_id)
.bind(path)
.execute(db)
.await?;
tracing::info!(
client = %client_name,
workspace_id = %w_id,
account_id = %account_id,
"Workspace integration OAuth token refreshed successfully"
);
Ok(token_result.access_token)
}

View File

@@ -517,6 +517,7 @@ pub async fn get_resource_value_interpolated_internal<'a>(
value,
&job_id,
token_for_context,
0,
)
.await?;
if allow_cache {
@@ -535,6 +536,7 @@ pub async fn transform_json_value(
v: Value,
job_id: &Option<Uuid>,
token: Option<&str>,
depth: u8,
) -> Result<Value> {
match v {
Value::String(y) if y.starts_with("$var:") => {
@@ -563,7 +565,8 @@ pub async fn transform_json_value(
tx.commit().await?;
let v = not_found_if_none(v, "Resource", path)?;
if let Some(v) = v {
transform_json_value(db_with_opt_authed, workspace, v, job_id, token).await
transform_json_value(db_with_opt_authed, workspace, v, job_id, token, depth + 1)
.await
} else {
Ok(Value::Null)
}
@@ -636,10 +639,41 @@ pub async fn transform_json_value(
.unwrap_or_else(|| y);
Ok(serde_json::json!(value))
}
Value::Array(mut arr) if depth <= 2 && arr.len() <= 1000 => {
for i in 0..arr.len() {
let val = std::mem::take(&mut arr[i]);
arr[i] = transform_json_value(
db_with_opt_authed,
workspace,
val,
job_id,
token,
depth + 1,
)
.await?;
}
Ok(Value::Array(arr))
}
Value::Array(arr) => {
if arr.len() > 1000 {
tracing::warn!(
"Array with {} items exceeds 1000 item limit for variable/resource resolution, skipping",
arr.len()
);
}
Ok(Value::Array(arr))
}
Value::Object(mut m) => {
for (a, b) in m.clone().into_iter() {
let v =
transform_json_value(db_with_opt_authed, workspace, b, job_id, token).await?;
let v = transform_json_value(
db_with_opt_authed,
workspace,
b,
job_id,
token,
depth + 1,
)
.await?;
m.insert(a.clone(), v);
}
Ok(Value::Object(m))
@@ -1030,6 +1064,15 @@ async fn update_resource(
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE workspace_integrations SET resource_path = $1 WHERE workspace_id = $2 AND resource_path = $3",
npath,
w_id,
path
)
.execute(&mut *tx)
.await?;
}
}
@@ -1822,6 +1865,7 @@ pub async fn interpolate(
value,
&None,
None,
0,
)
.await?
{
@@ -1829,3 +1873,84 @@ pub async fn interpolate(
v => Err(anyhow::anyhow!("Expected string, got {:?}", v)),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use windmill_common::audit::AuditAuthor;
use windmill_common::db::DbWithOptAuthed;
fn test_db_with_opt_authed(db: DB) -> DbWithOptAuthed<'static, ApiAuthed> {
DbWithOptAuthed::DB {
db,
audit_author: AuditAuthor {
username: "test".to_string(),
email: "test@test.com".to_string(),
username_override: None,
token_prefix: None,
},
}
}
#[tokio::test]
async fn test_transform_array_over_1000_passthrough() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let dba = test_db_with_opt_authed(pool);
let arr: Vec<Value> = (0..1001).map(|i| json!(format!("$var:x/{i}"))).collect();
let input = Value::Array(arr.clone());
let result = transform_json_value(&dba, "test", input, &None, None, 0)
.await
.unwrap();
assert_eq!(result, Value::Array(arr));
}
#[tokio::test]
async fn test_transform_array_non_matching_strings_passthrough() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let dba = test_db_with_opt_authed(pool);
let input = json!(["hello", "world", 42, true, null, {"key": "val"}]);
let result = transform_json_value(&dba, "test", input.clone(), &None, None, 0)
.await
.unwrap();
assert_eq!(result, input);
}
#[tokio::test]
async fn test_transform_array_resolved_inside_object() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let dba = test_db_with_opt_authed(pool);
let input = json!({"urls": ["$var:u/test/nonexistent", "plain"]});
let result = transform_json_value(&dba, "test", input, &None, None, 0).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_transform_array_attempts_matching_items() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let dba = test_db_with_opt_authed(pool);
let input = json!(["$var:u/test/nonexistent", "plain"]);
let result = transform_json_value(&dba, "test", input, &None, None, 0).await;
assert!(result.is_err());
}
}

View File

@@ -803,6 +803,15 @@ async fn update_variable(
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE workspace_integrations SET resource_path = $1 WHERE workspace_id = $2 AND resource_path = $3",
npath,
w_id,
path
)
.execute(&mut *tx)
.await?;
}
}

View File

@@ -8,7 +8,6 @@ use serde_json::json;
use sqlx::{postgres::PgListener, Pool, Postgres};
use uuid::Uuid;
use windmill_api_client::types::NewScript;
#[cfg(feature = "python")]
use windmill_common::flow_status::FlowStatusModule;
use windmill_common::{
jobs::{JobKind, JobPayload, RawCode},
@@ -424,7 +423,6 @@ pub async fn listen_for_completed_jobs(db: &Pool<Postgres>) -> impl Stream<Item
listen_for_uuid_on(db, "completed").await
}
#[cfg(feature = "deno_core")]
pub async fn listen_for_queue(db: &Pool<Postgres>) -> impl Stream<Item = Uuid> + Unpin {
listen_for_uuid_on(db, "queued").await
}
@@ -486,7 +484,6 @@ pub trait StreamFind: futures::Stream + Unpin + Sized {
impl<T: futures::Stream + Unpin + Sized> StreamFind for T {}
#[cfg(feature = "python")]
pub fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
cjob.flow_status.clone().and_then(|fs| {
use windmill_common::flow_status::FlowStatus;
@@ -498,7 +495,6 @@ pub fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
})
}
#[cfg(feature = "python")]
fn find_module_in_vec(modules: Vec<FlowStatusModule>, id: &str) -> Option<FlowStatusModule> {
modules.into_iter().find(|s| s.id() == id)
}
@@ -836,14 +832,12 @@ pub async fn testing_http_connection(port: u16) -> Connection {
let agent_token = format!(
"{}{}",
windmill_common::agent_workers::AGENT_JWT_PREFIX,
windmill_common::jwt::encode_with_internal_secret(
windmill_api_agent_workers::AgentAuth {
worker_group: "testing-agent".to_owned(),
suffix: Some(suffix.clone()),
tags: vec!["flow".into(), "python3".into(), "dependency".into()],
exp: Some(usize::MAX),
}
)
windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth {
worker_group: "testing-agent".to_owned(),
suffix: Some(suffix.clone()),
tags: vec!["flow".into(), "python3".into(), "dependency".into()],
exp: Some(usize::MAX),
})
.await
.expect("JWT token to be created")
);

View File

@@ -37,6 +37,7 @@ pub enum JobTriggerKind {
Schedule,
Gcp,
Nextcloud,
Google,
}
impl std::fmt::Display for JobTriggerKind {
@@ -54,6 +55,7 @@ impl std::fmt::Display for JobTriggerKind {
JobTriggerKind::Schedule => "schedule",
JobTriggerKind::Gcp => "gcp",
JobTriggerKind::Nextcloud => "nextcloud",
JobTriggerKind::Google => "google",
};
write!(f, "{}", kind)
}

View File

@@ -20,6 +20,7 @@ pub enum TriggerKind {
Postgres,
Gcp,
Nextcloud,
Google,
}
impl TriggerKind {
@@ -37,6 +38,7 @@ impl TriggerKind {
TriggerKind::Postgres => "postgres".to_string(),
TriggerKind::Gcp => "gcp".to_string(),
TriggerKind::Nextcloud => "nextcloud".to_string(),
TriggerKind::Google => "google".to_string(),
}
}
}
@@ -56,6 +58,7 @@ impl fmt::Display for TriggerKind {
TriggerKind::Postgres => "postgres",
TriggerKind::Gcp => "gcp",
TriggerKind::Nextcloud => "nextcloud",
TriggerKind::Google => "google",
};
write!(f, "{}", s)
}

View File

@@ -20,9 +20,10 @@ use crate::{
},
get_proxy_envs_for_lang,
handle_child::handle_child,
BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH,
is_sandboxing_enabled, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, NPM_CONFIG_REGISTRY,
NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR,
BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH,
NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH,
TZ_ENV,
};
use windmill_common::{
client::AuthedClient,
@@ -168,7 +169,7 @@ pub async fn gen_bun_lockfile(
let mut empty_deps = false;
if let Some(package_json_content) = workspace_dependencies.get_bun()? {
gen_bunfig(job_dir).await?;
gen_bunfig(job_dir, job_id, w_id, db).await?;
write_file(job_dir, "package.json", package_json_content.as_str())?;
} else {
let loader = RELATIVE_BUN_LOADER
@@ -193,7 +194,7 @@ pub async fn gen_bun_lockfile(
),
)?;
gen_bunfig(job_dir).await?;
gen_bunfig(job_dir, job_id, w_id, db).await?;
let mut child_cmd = Command::new(&*BUN_PATH);
child_cmd
@@ -291,9 +292,37 @@ pub async fn gen_bun_lockfile(
}
}
async fn gen_bunfig(job_dir: &str) -> Result<()> {
let registry = NPM_CONFIG_REGISTRY.read().await.clone();
let bunfig_install_scopes = BUNFIG_INSTALL_SCOPES.read().await.clone();
async fn gen_bunfig(
job_dir: &str,
job_id: &Uuid,
w_id: &str,
db: Option<&Connection>,
) -> Result<()> {
let (registry, bunfig_install_scopes) = if let Some(conn) = db {
(
read_ee_registry(
NPM_CONFIG_REGISTRY.read().await.clone(),
"npm registry",
job_id,
w_id,
conn,
)
.await,
read_ee_registry(
BUNFIG_INSTALL_SCOPES.read().await.clone(),
"bunfig install scopes",
job_id,
w_id,
conn,
)
.await,
)
} else {
(
NPM_CONFIG_REGISTRY.read().await.clone(),
BUNFIG_INSTALL_SCOPES.read().await.clone(),
)
};
if registry.is_some() || bunfig_install_scopes.is_some() {
let (url, token_opt) = if let Some(ref s) = registry {
let url = s.trim();
@@ -372,7 +401,18 @@ pub async fn install_bun_lockfile(
};
let has_file = if npm_mode {
let registry = NPM_CONFIG_REGISTRY.read().await.clone();
let registry = if let Some(conn) = db {
read_ee_registry(
NPM_CONFIG_REGISTRY.read().await.clone(),
"npm registry",
job_id,
w_id,
conn,
)
.await
} else {
NPM_CONFIG_REGISTRY.read().await.clone()
};
if let Some(registry) = registry {
let content = registry
.trim_start_matches("https:")
@@ -407,7 +447,7 @@ pub async fn install_bun_lockfile(
let mut child_process = start_child_process(child_cmd, &*BUN_PATH, false).await?;
gen_bunfig(job_dir).await?;
gen_bunfig(job_dir, job_id, w_id, db).await?;
if let Some(db) = db {
handle_child(
job_id,

View File

@@ -170,7 +170,8 @@ pub async fn transform_json<'a>(
let value = serde_json::from_str(inner_vs).map_err(|e| {
error::Error::internal_err(format!("Error while parsing inner arg: {e:#}"))
})?;
let transformed = transform_json_value(&k, &client, workspace, value, job, db).await?;
let transformed =
transform_json_value(&k, &client, workspace, value, job, db, 0).await?;
let as_raw = serde_json::from_value(transformed).map_err(|e| {
error::Error::internal_err(format!("Error while parsing inner arg: {e:#}"))
})?;
@@ -196,7 +197,8 @@ pub async fn transform_json_as_values<'a>(
let value = serde_json::from_str(inner_vs).map_err(|e| {
error::Error::internal_err(format!("Error while parsing inner arg: {e:#}"))
})?;
let transformed = transform_json_value(&k, &client, workspace, value, job, db).await?;
let transformed =
transform_json_value(&k, &client, workspace, value, job, db, 0).await?;
let as_raw = serde_json::from_value(transformed).map_err(|e| {
error::Error::internal_err(format!("Error while parsing inner arg: {e:#}"))
})?;
@@ -237,6 +239,7 @@ pub async fn transform_json_value(
v: Value,
job: &MiniPulledJob,
conn: &Connection,
depth: u8,
) -> error::Result<Value> {
match v {
Value::String(y) if y.starts_with("$var:") => {
@@ -306,11 +309,28 @@ pub async fn transform_json_value(
.unwrap_or_else(|| y);
Ok(json!(value))
}
Value::Array(mut arr) if depth <= 2 && arr.len() <= 1000 => {
for i in 0..arr.len() {
let val = std::mem::take(&mut arr[i]);
arr[i] = transform_json_value(name, client, workspace, val, job, conn, depth + 1)
.await?;
}
Ok(Value::Array(arr))
}
Value::Array(arr) => {
if arr.len() > 1000 {
tracing::warn!(
"Array with {} items exceeds 1000 item limit for variable/resource resolution, skipping",
arr.len()
);
}
Ok(Value::Array(arr))
}
Value::Object(mut m) => {
for (a, b) in m.clone().into_iter() {
m.insert(
a.clone(),
transform_json_value(&a, client, workspace, b, job, conn).await?,
transform_json_value(&a, client, workspace, b, job, conn, depth + 1).await?,
);
}
Ok(Value::Object(m))
@@ -1408,3 +1428,128 @@ impl MaybeLock {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
use windmill_common::client::AuthedClient;
fn test_client() -> AuthedClient {
AuthedClient::new(
"http://localhost:0".to_string(),
"test".to_string(),
"test-token".to_string(),
None,
)
}
fn test_job() -> MiniPulledJob {
MiniPulledJob {
workspace_id: "test".to_string(),
id: uuid::Uuid::nil(),
args: None,
parent_job: None,
created_by: "test".to_string(),
scheduled_for: chrono::Utc::now(),
started_at: None,
runnable_path: None,
kind: windmill_common::jobs::JobKind::Noop,
runnable_id: None,
canceled_reason: None,
canceled_by: None,
permissioned_as: "test".to_string(),
permissioned_as_email: "test@test.com".to_string(),
flow_status: None,
tag: "test".to_string(),
script_lang: None,
same_worker: false,
pre_run_error: None,
concurrent_limit: None,
concurrency_time_window_s: None,
flow_innermost_root_job: None,
root_job: None,
timeout: None,
flow_step_id: None,
cache_ttl: None,
cache_ignore_s3_path: None,
priority: None,
preprocessed: None,
script_entrypoint_override: None,
trigger: None,
trigger_kind: None,
visible_to_owner: false,
permissioned_as_end_user_email: None,
runnable_settings_handle: None,
}
}
#[tokio::test]
async fn test_transform_array_over_1000_passthrough() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let conn = Connection::Sql(pool);
let client = test_client();
let job = test_job();
let arr: Vec<Value> = (0..1001).map(|i| json!(format!("$var:x/{i}"))).collect();
let input = Value::Array(arr.clone());
let result = transform_json_value("test", &client, "test", input, &job, &conn, 0)
.await
.unwrap();
assert_eq!(result, Value::Array(arr));
}
#[tokio::test]
async fn test_transform_array_non_matching_strings_passthrough() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let conn = Connection::Sql(pool);
let client = test_client();
let job = test_job();
let input = json!(["hello", "world", 42, true, null, {"key": "val"}]);
let result = transform_json_value("test", &client, "test", input.clone(), &job, &conn, 0)
.await
.unwrap();
assert_eq!(result, input);
}
#[tokio::test]
async fn test_transform_array_resolved_inside_object() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let conn = Connection::Sql(pool);
let client = test_client();
let job = test_job();
let input = json!({"urls": ["$var:u/test/nonexistent", "plain"]});
let result = transform_json_value("test", &client, "test", input, &job, &conn, 0).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_transform_array_attempts_matching_items() {
let db_url = std::env::var("DATABASE_URL")
.unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string());
let pool = sqlx::PgPool::connect(&db_url).await.unwrap();
let conn = Connection::Sql(pool);
let client = test_client();
let job = test_job();
let input = json!(["$var:u/test/nonexistent", "plain"]);
let result = transform_json_value("test", &client, "test", input, &job, &conn, 0).await;
assert!(result.is_err());
}
}

View File

@@ -32,8 +32,8 @@ use crate::{
},
get_proxy_envs_for_lang,
handle_child::handle_child,
CSHARP_CACHE_DIR, is_sandboxing_enabled, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH,
NUGET_CONFIG, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
is_sandboxing_enabled, read_ee_registry, CSHARP_CACHE_DIR, DISABLE_NUSER, DOTNET_PATH,
HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
};
#[cfg(feature = "csharp")]
use windmill_common::scripts::ScriptLang;
@@ -82,8 +82,18 @@ pub async fn generate_nuget_lockfile(
) -> error::Result<String> {
check_executor_binary_exists("dotnet", DOTNET_PATH.as_str(), "C#")?;
if let Some(nuget_config) = NUGET_CONFIG.read().await.clone() {
write_file(job_dir, "nuget.config", &nuget_config)?;
if let Some(nuget_config) = read_ee_registry(
NUGET_CONFIG.read().await.clone(),
"nuget config",
job_id,
w_id,
conn,
)
.await
{
if !nuget_config.trim().is_empty() {
write_file(job_dir, "nuget.config", &nuget_config)?;
}
}
let (reqs, lines_to_remove) = parse_csharp_reqs(code);
@@ -334,8 +344,18 @@ async fn build_cs_proj(
hash: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
if let Some(nuget_config) = NUGET_CONFIG.read().await.clone() {
write_file(job_dir, "nuget.config", &nuget_config)?;
if let Some(nuget_config) = read_ee_registry(
NUGET_CONFIG.read().await.clone(),
"nuget config",
job_id,
w_id,
conn,
)
.await
{
if !nuget_config.trim().is_empty() {
write_file(job_dir, "nuget.config", &nuget_config)?;
}
}
let mut build_cs_cmd = Command::new(DOTNET_PATH.as_str());

View File

@@ -13,7 +13,8 @@ use crate::{
},
get_proxy_envs_for_lang,
handle_child::handle_child,
is_sandboxing_enabled, DENO_CACHE_DIR, DENO_PATH, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
is_sandboxing_enabled, read_ee_registry, DENO_CACHE_DIR, DENO_PATH, HOME_ENV,
NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
};
use windmill_common::client::AuthedClient;
@@ -54,6 +55,9 @@ lazy_static::lazy_static! {
async fn get_common_deno_proc_envs(
token: &str,
base_internal_url: &str,
job_id: &Uuid,
w_id: &str,
conn: Option<&Connection>,
) -> HashMap<String, String> {
let hostname = BASE_URL.read().await.clone();
let hostname_base = hostname.split("://").last().unwrap_or("localhost");
@@ -75,7 +79,19 @@ async fn get_common_deno_proc_envs(
),
]);
if let Some(ref s) = NPM_CONFIG_REGISTRY.read().await.clone() {
let registry = if let Some(conn) = conn {
read_ee_registry(
NPM_CONFIG_REGISTRY.read().await.clone(),
"npm registry",
job_id,
w_id,
conn,
)
.await
} else {
NPM_CONFIG_REGISTRY.read().await.clone()
};
if let Some(ref s) = registry {
let (url, _token_opt) = parse_npm_config(s);
deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), url);
}
@@ -131,7 +147,7 @@ pub async fn generate_deno_lock(
write_file(job_dir, "import_map.json", &import_map)?;
write_file(job_dir, "empty.ts", "")?;
let deno_envs = get_common_deno_proc_envs("", base_internal_url).await;
let deno_envs = get_common_deno_proc_envs("", base_internal_url, job_id, w_id, db).await;
let mut child_cmd = Command::new(DENO_PATH.as_str());
child_cmd
@@ -362,8 +378,14 @@ try {{
write_import_map_f
)?;
let mut common_deno_proc_envs =
get_common_deno_proc_envs(&client.token, base_internal_url).await;
let mut common_deno_proc_envs = get_common_deno_proc_envs(
&client.token,
base_internal_url,
&job.id,
&job.workspace_id,
Some(conn),
)
.await;
if is_sandboxing_enabled() {
common_deno_proc_envs.insert("HOME".to_string(), job_dir.to_string());
}
@@ -553,7 +575,14 @@ pub async fn start_worker(
use crate::common::build_envs_map;
let _ = write_file(job_dir, "main.ts", inner_content)?;
let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await;
let common_deno_proc_envs = get_common_deno_proc_envs(
&token,
base_internal_url,
&Uuid::nil(),
w_id,
Some(&db.into()),
)
.await;
let context = variables::get_reserved_variables(
&db.into(),

View File

@@ -24,8 +24,8 @@ use crate::{
read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL,
},
handle_child::handle_child,
is_sandboxing_enabled, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV,
NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
is_sandboxing_enabled, read_ee_registry, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR,
GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
};
use windmill_common::client::AuthedClient;
@@ -392,10 +392,26 @@ func Run(req Req) (interface{{}}, error){{
})
.env("HOME", HOME_ENV.as_str());
if let Some(ref goprivate) = *GOPRIVATE {
if let Some(ref goprivate) = read_ee_registry(
GOPRIVATE.clone(),
"go private",
&job.id,
&job.workspace_id,
conn,
)
.await
{
run_go.env("GOPRIVATE", goprivate);
}
if let Some(ref goproxy) = *GOPROXY {
if let Some(ref goproxy) = read_ee_registry(
GOPROXY.clone(),
"go proxy",
&job.id,
&job.workspace_id,
conn,
)
.await
{
run_go.env("GOPROXY", goproxy);
}
@@ -570,6 +586,7 @@ pub async fn install_go_dependencies(
.env_clear()
.env("HOME", HOME_ENV.as_str())
.env("PATH", PATH_ENV.as_str())
.envs(PROXY_ENVS.clone())
.env("GOPATH", {
#[cfg(unix)]
{
@@ -584,13 +601,17 @@ pub async fn install_go_dependencies(
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if let Some(ref goprivate) = *GOPRIVATE {
if let Some(ref goprivate) =
read_ee_registry(GOPRIVATE.clone(), "go private", job_id, w_id, conn).await
{
child_cmd.env("GOPRIVATE", goprivate);
}
// TODO: Remove if no incidents reported
if !std::env::var("WMDEBUG_NO_GOPROXY_ON_TIDY").ok().is_some() {
if let Some(ref goproxy) = *GOPROXY {
if let Some(ref goproxy) =
read_ee_registry(GOPROXY.clone(), "go proxy", job_id, w_id, conn).await
{
child_cmd.env("GOPROXY", goproxy);
}
}

View File

@@ -24,10 +24,10 @@ use crate::{
build_command_with_isolation, create_args_and_out_file, get_reserved_variables,
read_result, start_child_process, OccupancyMetrics,
},
handle_child,
handle_child, is_sandboxing_enabled, read_ee_registry,
universal_pkg_installer::{par_install_language_dependencies_all_at_once, RequiredDependency},
is_sandboxing_enabled, COURSIER_CACHE_DIR, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_HOME_DIR,
JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
COURSIER_CACHE_DIR, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_HOME_DIR, JAVA_REPOSITORY_DIR,
MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
};
use windmill_common::client::AuthedClient;
@@ -224,7 +224,7 @@ pub async fn resolve<'a>(
"--cache",
COURSIER_CACHE_DIR,
])
.args(&get_repos().await)
.args(&get_repos(job_id, w_id, conn).await)
.args(&deps.split("\n").collect_vec())
.stderr(Stdio::piped());
@@ -306,7 +306,7 @@ async fn install<'a>(
"JAVA classpath: {}", &classpath
);
let (repos, no_default, trust_store_metadata) = (
get_repos().await,
get_repos(&job.id, &job.workspace_id, conn).await,
get_no_default(),
metadata(TRUST_STORE_PATH.clone()).await,
);
@@ -788,21 +788,26 @@ fn parse_proxy() -> anyhow::Result<JavaProxySettings> {
Ok(jps)
}
async fn get_repos() -> Vec<String> {
MAVEN_REPOS
.read()
.await
.as_ref()
.map(|repos| {
repos
.trim()
.split_whitespace()
.into_iter()
.map(|el| vec!["--repository".to_owned(), el.to_owned()])
.collect_vec()
})
.unwrap_or_default()
.concat()
async fn get_repos(job_id: &Uuid, w_id: &str, conn: &Connection) -> Vec<String> {
read_ee_registry(
MAVEN_REPOS.read().await.clone(),
"maven repos",
job_id,
w_id,
conn,
)
.await
.as_ref()
.map(|repos| {
repos
.trim()
.split_whitespace()
.into_iter()
.map(|el| vec!["--repository".to_owned(), el.to_owned()])
.collect_vec()
})
.unwrap_or_default()
.concat()
}
fn get_no_default() -> String {

View File

@@ -27,8 +27,9 @@ use crate::{
read_file_content, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
is_sandboxing_enabled, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR,
POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV,
is_sandboxing_enabled, read_ee_registry, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS,
TZ_ENV,
};
fn val_to_pwsh_param(v: serde_json::Value) -> String {
@@ -355,8 +356,22 @@ pub async fn handle_powershell_job(
}
if !modules_to_install.is_empty() {
let powershell_repo_url = POWERSHELL_REPO_URL.read().await.clone();
let powershell_repo_pat = POWERSHELL_REPO_PAT.read().await.clone();
let powershell_repo_url = read_ee_registry(
POWERSHELL_REPO_URL.read().await.clone(),
"powershell repo url",
&job.id,
&job.workspace_id,
db,
)
.await;
let powershell_repo_pat = read_ee_registry(
POWERSHELL_REPO_PAT.read().await.clone(),
"powershell repo pat",
&job.id,
&job.workspace_id,
db,
)
.await;
let has_private_repo = powershell_repo_url.is_some();
let has_credentials = powershell_repo_pat.is_some();

View File

@@ -132,7 +132,7 @@ use crate::{
},
get_proxy_envs_for_lang,
handle_child::handle_child,
is_sandboxing_enabled,
is_sandboxing_enabled, read_ee_registry,
worker_utils::ping_job_status,
PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR,
@@ -286,21 +286,29 @@ pub async fn uv_pip_compile(
if no_cache {
args.extend(["--no-cache"]);
}
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
.read()
.await
.clone()
.map(handle_ephemeral_token);
let pip_extra_index_url = read_ee_registry(
PIP_EXTRA_INDEX_URL.read().await.clone(),
"pip extra index url",
job_id,
w_id,
conn,
)
.await
.map(handle_ephemeral_token);
if let Some(url) = pip_extra_index_url.as_ref() {
url.split(",").for_each(|url| {
args.extend(["--extra-index-url", url]);
});
}
let pip_index_url = PIP_INDEX_URL
.read()
.await
.clone()
.map(handle_ephemeral_token);
let pip_index_url = read_ee_registry(
PIP_INDEX_URL.read().await.clone(),
"pip index url",
job_id,
w_id,
conn,
)
.await
.map(handle_ephemeral_token);
if let Some(url) = pip_index_url.as_ref() {
args.extend(["--index-url", url]);
}
@@ -1635,16 +1643,24 @@ pub async fn handle_python_reqs(
);
let pip_indexes = (
PIP_EXTRA_INDEX_URL
.read()
.await
.clone()
.map(handle_ephemeral_token),
PIP_INDEX_URL
.read()
.await
.clone()
.map(handle_ephemeral_token),
read_ee_registry(
PIP_EXTRA_INDEX_URL.read().await.clone(),
"pip extra index url",
job_id,
w_id,
conn,
)
.await
.map(handle_ephemeral_token),
read_ee_registry(
PIP_INDEX_URL.read().await.clone(),
"pip index url",
job_id,
w_id,
conn,
)
.await
.map(handle_ephemeral_token),
);
// Cached paths

View File

@@ -28,8 +28,9 @@ use crate::{
},
get_proxy_envs_for_lang,
handle_child::{self},
is_sandboxing_enabled, read_ee_registry,
universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency},
is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS,
DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS,
TRACING_PROXY_CA_CERT_PATH,
};
use windmill_common::scripts::ScriptLang;
@@ -360,7 +361,16 @@ Your Gemfile syntax will continue to work as-is."
])
.envs(RUBY_PROXY_ENVS.clone());
for repo in RUBY_REPOS.read().await.clone().unwrap_or_default() {
for repo in read_ee_registry(
RUBY_REPOS.read().await.clone(),
"ruby repos",
job_id,
w_id,
conn,
)
.await
.unwrap_or_default()
{
if let (Some(url), usr, Some(passwd)) =
(repo.domain(), repo.username(), repo.password())
{
@@ -591,7 +601,15 @@ async fn install<'a>(
let job_dir = job_dir.to_owned();
let jailed = !cfg!(windows) && is_sandboxing_enabled();
let RubyAnnotations { verbose } = RubyAnnotations::parse(&inner_content);
let repos = RUBY_REPOS.read().await.clone().unwrap_or_default();
let repos = read_ee_registry(
RUBY_REPOS.read().await.clone(),
"ruby repos",
&job.id,
&job.workspace_id,
conn,
)
.await
.unwrap_or_default();
let (envs, reserved_variables) = (
envs.clone(),
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?,

View File

@@ -1,4 +1,6 @@
use serde_json::value::RawValue;
#[cfg(not(windows))]
use std::sync::Once;
use std::{collections::HashMap, process::Stdio};
use uuid::Uuid;
use windmill_parser_rust::parse_rust_deps_into_manifest;
@@ -25,8 +27,8 @@ use crate::{
},
get_proxy_envs_for_lang,
handle_child::handle_child,
is_sandboxing_enabled, CARGO_REGISTRIES, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, RUST_CACHE_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
is_sandboxing_enabled, read_ee_registry, CARGO_REGISTRIES, DISABLE_NUSER, HOME_ENV,
NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
};
use windmill_common::client::AuthedClient;
use windmill_common::scripts::ScriptLang;
@@ -38,15 +40,42 @@ const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.co
const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str =
include_str!("../nsjail/download.rust.config.proto");
fn find_cargo_path() -> String {
if let Ok(p) = std::env::var("CARGO_PATH") {
return p;
}
let from_home = format!("{}/bin/cargo", CARGO_HOME.as_str());
if std::path::Path::new(&from_home).exists() {
return from_home;
}
for p in ["/usr/local/cargo/bin/cargo", "/usr/bin/cargo"] {
if std::path::Path::new(p).exists() {
return p.to_string();
}
}
from_home
}
#[cfg(not(windows))]
fn find_preinstalled_dir(env_var: &str, candidates: &[&str]) -> String {
if let Ok(p) = std::env::var(env_var) {
return p;
}
for c in candidates {
if std::path::Path::new(c).exists() {
return c.to_string();
}
}
candidates[0].to_string()
}
lazy_static::lazy_static! {
static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable");
static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() });
static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() });
static ref CARGO_PATH: String = std::env::var("CARGO_PATH").unwrap_or_else(|_| format!("{}/bin/cargo", CARGO_HOME.as_str()));
// static ref CARGO_SWEEP_PATH: String = std::env::var("CARGO_SWEEP_PATH").unwrap_or_else(|_| format!("{}/bin/cargo-sweep", CARGO_HOME.as_str()));
static ref CARGO_PATH: String = find_cargo_path();
static ref SWEEP_MAXSIZE: String = std::env::var("CARGO_SWEEP_MAXSIZE").unwrap_or("25GB".to_owned());
static ref NO_SHARED_BUILD_DIR: bool = std::env::var("RUST_NO_SHARED_BUILD_DIR").ok().map(|flag| flag == "true").unwrap_or(false);
}
#[cfg(windows)]
@@ -63,6 +92,72 @@ lazy_static::lazy_static! {
const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/";
#[cfg(not(windows))]
lazy_static::lazy_static! {
static ref PREINSTALLED_CARGO: String = find_preinstalled_dir(
"CARGO_PREINSTALL_DIR",
&["/usr/local/cargo", &format!("{}/.cargo", *HOME_DIR)],
);
static ref PREINSTALLED_RUSTUP: String = find_preinstalled_dir(
"RUSTUP_PREINSTALL_DIR",
&["/usr/local/rustup", &format!("{}/.rustup", *HOME_DIR)],
);
}
#[cfg(not(windows))]
static RUST_DIRS_INIT: Once = Once::new();
#[cfg(not(windows))]
fn symlink_preinstalled_entries(preinstalled: &str, target: &str) {
use std::fs;
use std::os::unix::fs as unix_fs;
use std::path::Path;
if target == preinstalled || !Path::new(preinstalled).exists() {
return;
}
let _ = fs::create_dir_all(target);
let Ok(entries) = fs::read_dir(preinstalled) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let link_path = Path::new(target).join(&name);
if !link_path.exists() {
let _ = unix_fs::symlink(entry.path(), &link_path);
}
}
}
#[cfg(not(windows))]
fn symlink_single_entry(preinstalled: &str, target: &str, name: &str) {
use std::os::unix::fs as unix_fs;
use std::path::Path;
let src = Path::new(preinstalled).join(name);
if !src.exists() {
return;
}
let _ = std::fs::create_dir_all(target);
let dst = Path::new(target).join(name);
if !dst.exists() {
let _ = unix_fs::symlink(&src, &dst);
}
}
#[cfg(not(windows))]
fn ensure_rust_runtime_dirs() {
RUST_DIRS_INIT.call_once(|| {
// Only symlink bin/ from cargo (registry/git must be writable)
symlink_single_entry(&PREINSTALLED_CARGO, CARGO_HOME.as_str(), "bin");
// Symlink all entries from rustup (toolchains, settings.toml, etc.)
symlink_preinstalled_entries(&PREINSTALLED_RUSTUP, RUSTUP_HOME.as_str());
});
}
#[cfg(windows)]
fn ensure_rust_runtime_dirs() {}
fn gen_cargo_crate(code: &str, job_dir: &str) -> anyhow::Result<()> {
let manifest = parse_rust_deps_into_manifest(code)?;
write_file(job_dir, "Cargo.toml", &manifest)?;
@@ -135,11 +230,26 @@ pub fn __WINDMILL_RUN__(_args: __WINDMILL_ARGS__) -> Result<String, Box<dyn std:
Ok(())
}
async fn write_cargo_config(job_dir: &str) -> anyhow::Result<()> {
if let Some(cargo_registries) = CARGO_REGISTRIES.read().await.clone() {
let cargo_dir = format!("{job_dir}/.cargo");
create_dir_all(&cargo_dir).await?;
write_file(&cargo_dir, "config.toml", &cargo_registries)?;
async fn write_cargo_config(
job_dir: &str,
job_id: &Uuid,
w_id: &str,
conn: &Connection,
) -> anyhow::Result<()> {
if let Some(cargo_registries) = read_ee_registry(
CARGO_REGISTRIES.read().await.clone(),
"cargo registries",
job_id,
w_id,
conn,
)
.await
{
if !cargo_registries.trim().is_empty() {
let cargo_dir = format!("{job_dir}/.cargo");
create_dir_all(&cargo_dir).await?;
write_file(&cargo_dir, "config.toml", &cargo_registries)?;
}
}
Ok(())
}
@@ -155,10 +265,11 @@ pub async fn generate_cargo_lockfile(
w_id: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
ensure_rust_runtime_dirs();
check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?;
gen_cargo_crate(code, job_dir)?;
write_cargo_config(job_dir).await?;
write_cargo_config(job_dir, job_id, w_id, conn).await?;
let mut gen_lockfile_cmd = Command::new(CARGO_PATH.as_str());
gen_lockfile_cmd
@@ -260,10 +371,11 @@ async fn get_build_dir(
if run_sweep {
// Also run sweep to make sure target isn't using too much disk
let mut sweep_cmd = Command::new(CARGO_PATH.as_str());
let sweep_path = format!("{}/bin:{}", CARGO_HOME.as_str(), PATH_ENV.as_str());
sweep_cmd
.current_dir(job_dir)
.env_clear()
.env("PATH", PATH_ENV.as_str())
.env("PATH", &sweep_path)
.env("CARGO_HOME", CARGO_HOME.as_str())
.env("HOME", HOME_ENV.as_str())
.env("CARGO_TARGET_DIR", &(bd.clone() + "/target"))
@@ -335,6 +447,7 @@ pub async fn build_rust_crate(
occupancy_metrics: &mut OccupancyMetrics,
is_preview: bool,
) -> error::Result<String> {
ensure_rust_runtime_dirs();
let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
let build_dir = get_build_dir(job, job_dir, conn, worker_name, is_preview).await?;
@@ -487,6 +600,7 @@ pub async fn handle_rust_job(
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>, Error> {
ensure_rust_runtime_dirs();
check_executor_binary_exists("cargo", CARGO_PATH.as_str(), "rust")?;
let hash = compute_rust_hash(inner_content, requirements_o);
@@ -520,7 +634,7 @@ pub async fn handle_rust_job(
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
gen_cargo_crate(inner_content, job_dir)?;
write_cargo_config(job_dir).await?;
write_cargo_config(job_dir, &job.id, &job.workspace_id, conn).await?;
if let Some(reqs) = requirements_o {
if !reqs.is_empty() {

Some files were not shown because too many files have changed in this diff Show More