Compare commits

..

6 Commits

Author SHA1 Message Date
HugoCasa
e033c73b79 chore: update ee-repo-ref to batch-pulling latest
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-06 18:07:36 +01:00
HugoCasa
8c3ac22d8d feat: add as_worker_tag() helper, benchmark results and model
- Extract bunnative→nativets tag logic into ScriptLang::as_worker_tag()
- Add benchmark results for batch pull vs direct SQL (1W and 3W)
- Add throughput model script comparing batch vs SQL at scale
- Add nativets_sleep benchmark script support

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-06 18:05:15 +01:00
HugoCasa
21398e5447 fix: use BATCH_PULL_URL env var and add JWT exp claim
- Replace BASE_INTERNAL_URL overloading with dedicated BATCH_PULL_URL
  env var for native workers' HTTP pull endpoint
- Add exp claim to JWT token (required by jsonwebtoken validation)
- Token expires in 30 days, renewed on worker restart

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 19:15:12 +01:00
HugoCasa
980cbcccf0 Merge remote-tracking branch 'origin/main' into batch-pulling 2026-03-05 17:14:53 +01:00
HugoCasa
dd422fcc5d fix: enable batch pull for worker-only mode with BASE_INTERNAL_URL
Native workers in Mode::Worker (no co-located server) can now use HTTP
batch pull when BASE_INTERNAL_URL is explicitly set pointing to the
remote server. The batch buffer itself only runs on the server side.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 16:33:20 +01:00
HugoCasa
876a9cfc8e feat: batch job pulling for native workers
Reduce DB polling overhead for native workers by batch-fetching jobs
server-side and serving them from an in-memory buffer via HTTP.

- Add batch_pull() in windmill-queue: single SELECT...FOR UPDATE SKIP LOCKED LIMIT N
- Add batch pull SQL helpers (make_batch_pull_query, format_batch_pull_query)
- OSS stubs for agent-workers accept batch_buffer parameter (4-tuple return)
- Native workers self-sign JWT and pull jobs via HTTP when co-located with server
- Add uses_batch_http_pull column to worker_ping for server-side tracking
- Worker pull loop: HTTP batch pull when client available, SQL otherwise

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 16:08:28 +01:00
678 changed files with 9354 additions and 24270 deletions

View File

@@ -1,7 +1,6 @@
name: Backend only integration tests
on:
workflow_dispatch:
push:
branches:
- "main"

View File

@@ -1,65 +0,0 @@
# Project display name in the dashboard
name: Windmill
workspace:
mainBranch: main
worktreeRoot: ../windmill__worktrees
defaultAgent: claude
startupEnvs:
CARGO_FEATURES: "quickjs"
WM_CLONE_DB: false
USE_RUST_PLUGIN: false
lifecycleHooks:
postCreate: bash ./scripts/post-create.sh
preRemove: bash ./scripts/pre-remove.sh
auto_name:
model: gemini-2.5-flash-lite
# Each service defines a port env var that webmux injects into pane and agent
# process environments when creating a worktree. Ports are auto-assigned:
# base + (slot x step).
services:
- name: backend
portEnv: BACKEND_PORT
portStart: 8000
portStep: 10
- name: frontend
portEnv: FRONTEND_PORT
portStart: 3000
portStep: 10
profiles:
default:
runtime: host
yolo: true
envPassthrough: []
systemPrompt: >
You are running inside a tmux session with other panes running services.
Pane layout (current window):
- Pane 0: this pane (claude agent)
- Pane 1: backend (cargo watch -x run)
- Pane 2: frontend (npm run dev)
To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend).
When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}.
Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check.
panes:
- id: agent
kind: agent
focus: true
- id: backend
kind: command
split: right
command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
- id: frontend
kind: command
split: bottom
command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
integrations:
github:
linkedRepos: []
linear:
enabled: true

113
.wmdev.yaml Normal file
View File

@@ -0,0 +1,113 @@
name: Windmill
startupEnvs:
CARGO_FEATURES: "quickjs"
WM_CLONE_DB: false
USE_RUST_PLUGIN: false
services:
- name: BE
portEnv: BACKEND_PORT
- name: FE
portEnv: FRONTEND_PORT
profiles:
default:
name: default
sandbox:
name: sandbox
image: windmill-sandbox
envPassthrough:
- AWS_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY
- R2_ENDPOINT
- R2_BUCKET
- R2_PUBLIC_URL
extraMounts:
- hostPath: ~/.ssh
guestPath: /root/.ssh
writable: true
- hostPath: ~/.codex
guestPath: /root/.codex
writable: true
- hostPath: ~/windmill-ee-private
writable: true
- hostPath: ~/windmill-ee-private__worktrees
writable: true
systemPrompt: >
You are running inside a sandboxed container with full permissions.
This worktree is configured with the following ports:
- Backend: port ${BACKEND_PORT}.
Start with: cd backend && PORT=${BACKEND_PORT}
DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill
cargo watch -x run
- Frontend: port ${FRONTEND_PORT}.
Start with: cd frontend && REMOTE=http://localhost:${BACKEND_PORT}
npm run dev -- --port ${FRONTEND_PORT} --host 0.0.0.0
--- Screenshots ---
You can take screenshots of the frontend UI and upload them to R2
for use in PR descriptions.
1) Take a screenshot:
bunx playwright screenshot --browser chromium
http://localhost:${FRONTEND_PORT}/path/to/page /tmp/screenshot.png
2) Upload to R2:
aws s3 cp /tmp/screenshot.png
"s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/screenshot.png"
--endpoint-url "$(printenv R2_ENDPOINT)"
3) The public URL will be:
$(printenv R2_PUBLIC_URL)/<branch>/screenshot.png
4) Include in PR descriptions using markdown image syntax.
--- Terminal Recordings (asciinema) ---
You can record terminal sessions and upload them for sharing.
asciinema is available on PATH.
1) Write a shell script with the commands to demo. Add sleep
delays for readable pacing:
- 0.5s after printing a "$ command" line (lets viewer read it)
- 1.5-2s after command output (lets viewer absorb the result)
- Set GIT_PAGER=cat and PAGER=cat to prevent pager hangs
2) Record headlessly:
asciinema rec --headless --overwrite \
-c "bash /tmp/demo.sh" \
--window-size 120x50 \
--title "Description of demo" \
/tmp/demo.cast
3) Upload to asciinema.org:
XDG_DATA_HOME=/tmp/.local/share \
asciinema upload --server-url https://asciinema.org /tmp/demo.cast
--- Mermaid Diagrams ---
You can render Mermaid diagrams to SVG using the pre-installed mmdc CLI.
The puppeteer config (no-sandbox + Chromium path) is at /root/.puppeteerrc.json.
1) Write a .mmd file with your diagram:
cat > /tmp/diagram.mmd << 'EOF'
graph TD
A[Start] --> B[End]
EOF
2) Render to SVG (the -p flag is required):
mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json
3) Upload to R2:
aws s3 cp /tmp/diagram.svg
"s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg"
--endpoint-url "$(printenv R2_ENDPOINT)"
4) The public URL will be:
$(printenv R2_PUBLIC_URL)/<branch>/diagram.svg
5) Include in PR descriptions using markdown image syntax.
IMPORTANT: Read docs/autonomous-mode.md before starting any work.
linkedRepos:
- repo: windmill-labs/windmill-ee-private
alias: ee

View File

@@ -1,69 +1,5 @@
# Changelog
## [1.654.0](https://github.com/windmill-labs/windmill/compare/v1.653.0...v1.654.0) (2026-03-10)
### Features
* add git sync support for workspace dependencies ([#8144](https://github.com/windmill-labs/windmill/issues/8144)) ([4f29e05](https://github.com/windmill-labs/windmill/commit/4f29e05e3ae725e0be7ab797f8fa2186d8c5c0a5))
* add kafka trigger offset reset and auto.offset.reset config ([#8283](https://github.com/windmill-labs/windmill/issues/8283)) ([b02f9e5](https://github.com/windmill-labs/windmill/commit/b02f9e5c2426bff2356e1aaaa18e05b18c5efc6b))
* add preprocessor support for dedicated workers and bunnative scripts ([#8284](https://github.com/windmill-labs/windmill/issues/8284)) ([dc0e59f](https://github.com/windmill-labs/windmill/commit/dc0e59f432a0e3a53606adb8ac76d2dd2d365ace))
* add Vertex AI support for Google Gemini models ([#8303](https://github.com/windmill-labs/windmill/issues/8303)) ([cb349cb](https://github.com/windmill-labs/windmill/commit/cb349cb3d1b7561fb70a8c23fa83dc1c9441821c))
* **frontend:** replace flat sugiyama with recursive compound layout for flow graph ([#8204](https://github.com/windmill-labs/windmill/issues/8204)) ([cad4436](https://github.com/windmill-labs/windmill/commit/cad44365ac17029a2257f12cef061219b0265570))
### Bug Fixes
* **cli:** fail when passing an invalid --workspace arg ([#8294](https://github.com/windmill-labs/windmill/issues/8294)) ([f291b1c](https://github.com/windmill-labs/windmill/commit/f291b1cc19689e69e7aa008c19ce747e9c56240e))
* debounce webhook arg accumulation with max_count/max_time limits ([#8307](https://github.com/windmill-labs/windmill/issues/8307)) ([83be59e](https://github.com/windmill-labs/windmill/commit/83be59e0e866ebd091f1e27c0571710a989fd2e4))
* delete debounce_key on post-preprocessing limit exceeded ([#8299](https://github.com/windmill-labs/windmill/issues/8299)) ([438f609](https://github.com/windmill-labs/windmill/commit/438f609a78325ee5c2493079ca27bf587fa0d5ff))
* explicilty fail when --base-url --token --workspace are invalid ([#8302](https://github.com/windmill-labs/windmill/issues/8302)) ([5baeb8c](https://github.com/windmill-labs/windmill/commit/5baeb8c842a392c21457b7561e30b385e02a6a48))
* handle missing schema in RunnableByPath during wmill.d.ts generation ([#8300](https://github.com/windmill-labs/windmill/issues/8300)) ([b841e0a](https://github.com/windmill-labs/windmill/commit/b841e0a0384941079f37374f8fbbe2dd7fb51897))
* optimize flow lock generation and add rt.d.ts guidance for TS resource types ([#8295](https://github.com/windmill-labs/windmill/issues/8295)) ([b40cf80](https://github.com/windmill-labs/windmill/commit/b40cf80fdd62cbc31db0872ada551ce213b9dac8))
* preserve teams oauth tenant on settings page reload ([#8308](https://github.com/windmill-labs/windmill/issues/8308)) ([dbfa271](https://github.com/windmill-labs/windmill/commit/dbfa271b8962fe7b3d2aa8bf494e9557047fc8b3))
* resync custom_instance_user password on startup ([#8297](https://github.com/windmill-labs/windmill/issues/8297)) ([53ac43f](https://github.com/windmill-labs/windmill/commit/53ac43f5ee34570a9bb7b3441c73095e23690300))
* show meaningful error messages in database manager schema fetch ([#8296](https://github.com/windmill-labs/windmill/issues/8296)) ([cda8439](https://github.com/windmill-labs/windmill/commit/cda843922dcfd9a02ef9926751cbf8f544d2d4b6))
* skip loading flow preview history for new flows ([#8293](https://github.com/windmill-labs/windmill/issues/8293)) ([ac8c668](https://github.com/windmill-labs/windmill/commit/ac8c668cb93e56bc2a247bbdbbec14e5608125d2))
* teams selection not sticking in workspace settings ([#8309](https://github.com/windmill-labs/windmill/issues/8309)) ([fefc8c6](https://github.com/windmill-labs/windmill/commit/fefc8c62a00fe7a39f3104091e08087cd7c37afb))
## [1.653.0](https://github.com/windmill-labs/windmill/compare/v1.652.0...v1.653.0) (2026-03-10)
### Features
* add indexer time window setting (default 7 days) ([#8290](https://github.com/windmill-labs/windmill/issues/8290)) ([0c4d72c](https://github.com/windmill-labs/windmill/commit/0c4d72cfe38d61cf3f6e9bc31056005f1adb494d))
* add slack connection fields to workspace settings export/import ([#8287](https://github.com/windmill-labs/windmill/issues/8287)) ([39e77ec](https://github.com/windmill-labs/windmill/commit/39e77ecd002b41630fa8d146ee0f15369656acda))
### Performance Improvements
* optimize job_stats storage for timestamps and zero-memory jobs ([#8289](https://github.com/windmill-labs/windmill/issues/8289)) ([2d8335d](https://github.com/windmill-labs/windmill/commit/2d8335dc43a7cb182eb5a058119d8b0be067cdfd))
## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09)
### Features
* add secretKeyRef support for package registry and storage credentials ([#8275](https://github.com/windmill-labs/windmill/issues/8275)) ([73d27e9](https://github.com/windmill-labs/windmill/commit/73d27e92dd6ced1602f6328f245fec0fa96860e1))
* expose OTEL trace context as env vars in job execution ([#8277](https://github.com/windmill-labs/windmill/issues/8277)) ([93f75ad](https://github.com/windmill-labs/windmill/commit/93f75ada5e49036f0d998e3d3d53de4dc2c2e83f))
* workflow-as-code (WAC) v2 ([#8172](https://github.com/windmill-labs/windmill/issues/8172)) ([a6d4390](https://github.com/windmill-labs/windmill/commit/a6d4390790d21d535df1e9d525bffd577c50d8dc))
### Bug Fixes
* cli: support deleting linked resources-variables without throwing ([#8248](https://github.com/windmill-labs/windmill/issues/8248)) ([7859bca](https://github.com/windmill-labs/windmill/commit/7859bca6ae80d32a73a46910960afc6812e64115))
* Database studio fixes ([#8251](https://github.com/windmill-labs/windmill/issues/8251)) ([1d78589](https://github.com/windmill-labs/windmill/commit/1d785899404e8636a206cda9a2914df32a1a5269))
* **frontend:** unsaved changes dialog when flow already saved ([#8259](https://github.com/windmill-labs/windmill/issues/8259)) ([0330993](https://github.com/windmill-labs/windmill/commit/0330993cb66cdabffcd6e552a0f85a9a3931c62d))
* gracefully handle uninitialized OTEL tracing proxy port ([#8274](https://github.com/windmill-labs/windmill/issues/8274)) ([8b1fe8f](https://github.com/windmill-labs/windmill/commit/8b1fe8f9de7b0c03655558d0c46cfff71a4b2047))
* guard iteration picker VirtualList against empty items array ([#8273](https://github.com/windmill-labs/windmill/issues/8273)) ([c97cf60](https://github.com/windmill-labs/windmill/commit/c97cf604ab4a902d89fe873b90dbeb9dabc940eb)), closes [#8272](https://github.com/windmill-labs/windmill/issues/8272)
* mask secrets in OAuth config debug/log output ([#8269](https://github.com/windmill-labs/windmill/issues/8269)) ([e75763d](https://github.com/windmill-labs/windmill/commit/e75763dbe5ffe08e6cde082203596d510c2c3b29))
* parallel branchall hang on bad stop_after_all_iters_if + results.x.length null ([#8276](https://github.com/windmill-labs/windmill/issues/8276)) ([41e523f](https://github.com/windmill-labs/windmill/commit/41e523f827c4e3d5db525a1f14e24936b0b8af46))
* redact secrets in set_global_setting log line ([#8270](https://github.com/windmill-labs/windmill/issues/8270)) ([6a0473c](https://github.com/windmill-labs/windmill/commit/6a0473c5783dc0fef2ae82dc5345a5f0596f124d))
* remove $bindable() fallback values causing props_invalid_value error in oauth settings ([#8265](https://github.com/windmill-labs/windmill/issues/8265)) ([037035e](https://github.com/windmill-labs/windmill/commit/037035e094937827305dad29bd76a495d78bc46f))
* skip down migrations in potentially_stale checksum comparison ([#8271](https://github.com/windmill-labs/windmill/issues/8271)) ([5ba4029](https://github.com/windmill-labs/windmill/commit/5ba4029d8692b2e6054fca7f45ed4cfded4738ef))
* sql input horizontal scroll missing after switching flow steps ([#8249](https://github.com/windmill-labs/windmill/issues/8249)) ([ce8ac9c](https://github.com/windmill-labs/windmill/commit/ce8ac9cf52dc17061673b9b72556279c48c26f8e))
* wmill workspace whoami output ([#8246](https://github.com/windmill-labs/windmill/issues/8246)) ([1ac391a](https://github.com/windmill-labs/windmill/commit/1ac391a795585747fe5911ac41b157556569fedb))
## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05)

View File

@@ -26,27 +26,6 @@ Open-source platform for internal tools, workflows, API integrations, background
- **Login**: `admin@windmill.dev` / `changeme`
- **Instance settings**: navigate to `/#superadmin-settings`
## Banned Patterns
### `$bindable(default_value)` on optional props
Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state.
**Bad:**
```svelte
let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props()
```
**Correct alternatives:**
1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site:
```svelte
let { my_prop = $bindable() }: { my_prop?: string } = $props()
let effective_value = $derived(my_prop ?? default_value)
```
2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value.
## Core Principles
- Search for existing code to reuse before writing new code

View File

@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Int4"
]
},
"nullable": []
},
"hash": "0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699"
}

View File

@@ -1,40 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT j.id, j.runnable_path, j.args, j.kind::text AS \"kind!\"\n FROM v2_job j\n JOIN v2_job_queue q ON j.id = q.id\n WHERE j.runnable_path = $1\n AND j.kind = 'deploymentcallback'\n ORDER BY j.created_at DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "args",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "kind!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
true,
true,
null
]
},
"hash": "0d4f28ca0c5697c96711ca7225a9a4013e6ccabb495c371471c9d1287defda8f"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Int4"
]
},
"nullable": []
},
"hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6"
}

View File

@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n auto_offset_reset = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"VarcharArray",
"JsonbArray",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "12631fecee6aa11a45cf5c8d101c0dd8de50ac9b57e68198f637038d344fdd46"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH _ AS (\n UPDATE debounce_key\n SET debounced_times = 0, -- reset debounced_times\n first_started_at = now(), -- rest\n previous_job_id = NULL\n WHERE job_id = $1\n )\n UPDATE v2_job_debounce_batch \n SET debounce_batch = nextval('debounce_batch_seq') -- move to new batch\n WHERE id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "16c96166ffa6b9aec65c6072b204b52b87e3c2f3d76e47eb173fc78721355066"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\",\n (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS \"is_wac!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE COALESCE(ji.parent_job, ji.id)\n END\n JOIN v2_job j ON j.id = q.id\n LEFT JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ",
"query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE ji.parent_job\n END\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ",
"describe": {
"columns": [
{
@@ -32,11 +32,6 @@
"ordinal": 5,
"name": "is_flow_level!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "is_wac!",
"type_info": "Bool"
}
],
"parameters": {
@@ -50,9 +45,8 @@
false,
true,
false,
null,
null
]
},
"hash": "dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353"
"hash": "1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82"
}

View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Int4"
]
},
"nullable": []
},
"hash": "1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "2c8b8ed14647332491846ae3fa8b0ab8113d52ae8ae613a810c2b452e0972d05"
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "3e8afd021088a99a24f27fa6f0a1b7f3edba3e9b834c814b464305bc2eb6ba80"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8",
"Uuid"
]
},
"nullable": []
},
"hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5"
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"TextArray",
"Float4",
"Int8",
"Int8",
"Text",
"Int8",
"Int8",
"Float4",
"Float4",
"Float4",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "6cd099d458ac380d5da27b9e69da035755496ea50f2b78fb9b1cd3a2eb7e7625"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2",
"query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
@@ -73,5 +73,5 @@
false
]
},
"hash": "9229d9a9ff389cf26e480b604b83900e2d362ee934ef27284ef39f4eed440e59"
"hash": "72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e"
}

View File

@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Float4"
]
},
"nullable": []
},
"hash": "a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO audit_partitioned\n (workspace_id, username, operation, action_kind, resource, parameters, email, span)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
"query": "INSERT INTO audit\n (workspace_id, username, operation, action_kind, resource, parameters, email, span)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
"describe": {
"columns": [],
"parameters": {
@@ -29,5 +29,5 @@
},
"nullable": []
},
"hash": "fbccafe6d34093a723b9d5a6ee8d618a80ceba6de2d39202e6293ef5207c31f6"
"hash": "ad8487a797713b3a6c10fb399c9fb8dcd940bb92e998145e250f28ccfe1c7033"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n auto_offset_reset,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12, $13, $14\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -12,7 +12,6 @@
"VarcharArray",
"JsonbArray",
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
@@ -35,5 +34,5 @@
},
"nullable": []
},
"hash": "4b5a711986017654bdd495893a16ddc6ab09c98cd8723865cbd341404bc6a02f"
"hash": "aed5439aa6dad950e505f9f8f6914fa5ca21319c501b2822c9bc751ddfc9a0a4"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "suspend",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false
]
},
"hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)\n SELECT q.workspace_id, q.id, q.started_at,\n COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,\n $2::jsonb, r.memory_peak, 'failure'::job_status, q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_runtime r ON r.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Jsonb"
]
},
"nullable": []
},
"hash": "beecb176df512e4a94771d0d73c4c597e07e53d499131b57e4d6441fd0af09cb"
}

View File

@@ -1,24 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "job_ids: serde_json::Value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT args FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "args",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "c944e384c4b4c6455b431978adc54d176f294b661675392cf92561c0e6e02e6e"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH _ AS (\n UPDATE debounce_key\n SET debounced_times = 0,\n first_started_at = now(),\n previous_job_id = NULL\n WHERE job_id = $1\n )\n UPDATE v2_job_debounce_batch\n SET debounce_batch = nextval('debounce_batch_seq')\n WHERE id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "c9530931f670eab1208c4a284a55afdc3fcbb0eb5f98fd63e2ec89442becbfaa"
}

View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Float4"
]
},
"nullable": []
},
"hash": "d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc"
}

View File

@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"VarcharArray",
"JsonbArray",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Text",
"Text",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "e2921e44c70cf6c76c55177f2b56985e84c59ecb3e1a13fcf27d5f7ae5f8d84c"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Float8"
]
},
"nullable": []
},
"hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20"
}

203
backend/Cargo.lock generated
View File

@@ -7103,7 +7103,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.3",
"socket2 0.6.2",
"system-configuration",
"tokio",
"tower-service",
@@ -7974,9 +7974,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.183"
version = "0.2.182"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
[[package]]
name = "libffi"
@@ -8102,9 +8102,9 @@ dependencies = [
[[package]]
name = "libz-sys"
version = "1.1.25"
version = "1.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1"
checksum = "4735e9cbde5aac84a5ce588f6b23a90b9b0b528f6c5a8db8a4aff300463a0839"
dependencies = [
"cc",
"libc",
@@ -10664,7 +10664,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.1",
"rustls 0.23.35",
"socket2 0.6.3",
"socket2 0.6.2",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -10673,9 +10673,9 @@ dependencies = [
[[package]]
name = "quinn-proto"
version = "0.11.14"
version = "0.11.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
dependencies = [
"aws-lc-rs",
"bytes",
@@ -10702,7 +10702,7 @@ dependencies = [
"cfg_aliases 0.2.1",
"libc",
"once_cell",
"socket2 0.6.3",
"socket2 0.6.2",
"tracing",
"windows-sys 0.60.2",
]
@@ -11987,9 +11987,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71"
[[package]]
name = "schannel"
version = "0.1.29"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
dependencies = [
"windows-sys 0.61.2",
]
@@ -12677,12 +12677,12 @@ dependencies = [
[[package]]
name = "socket2"
version = "0.6.3"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -14462,7 +14462,7 @@ dependencies = [
"indexmap 2.11.1",
"toml_datetime 0.7.0",
"toml_parser",
"winnow 0.7.15",
"winnow 0.7.14",
]
[[package]]
@@ -14471,7 +14471,7 @@ version = "1.0.9+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
dependencies = [
"winnow 0.7.15",
"winnow 0.7.14",
]
[[package]]
@@ -15741,7 +15741,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-nats",
@@ -15808,7 +15808,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15821,7 +15821,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"argon2",
@@ -15849,7 +15849,6 @@ dependencies = [
"dashmap 6.1.0",
"datafusion",
"ed25519-dalek",
"eventsource-stream",
"flate2",
"futures",
"git-version",
@@ -15961,7 +15960,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15984,7 +15983,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15997,7 +15996,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16023,7 +16022,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -16033,7 +16032,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16050,7 +16049,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"base64 0.22.1",
@@ -16073,7 +16072,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16096,7 +16095,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16112,7 +16111,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16132,7 +16131,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16152,7 +16151,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16166,7 +16165,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-nats",
@@ -16193,7 +16192,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16218,7 +16217,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"flate2",
@@ -16236,7 +16235,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16257,7 +16256,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16277,7 +16276,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16307,7 +16306,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16334,7 +16333,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"lazy_static",
"serde",
@@ -16346,7 +16345,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"argon2",
"axum 0.7.9",
@@ -16369,7 +16368,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16383,7 +16382,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16414,7 +16413,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"chrono",
"lazy_static",
@@ -16428,7 +16427,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16447,7 +16446,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"aes-gcm",
"anyhow",
@@ -16546,7 +16545,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -16565,7 +16564,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"regex",
"serde",
@@ -16580,7 +16579,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -16604,7 +16603,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"futures",
@@ -16621,7 +16620,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -16637,7 +16636,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -16658,7 +16657,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -16689,7 +16688,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-oauth2",
@@ -16713,7 +16712,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-stream",
@@ -16747,7 +16746,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"futures",
@@ -16765,7 +16764,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -16774,7 +16773,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -16786,7 +16785,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"serde_json",
@@ -16798,7 +16797,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"gosyn",
@@ -16810,7 +16809,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -16822,7 +16821,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"serde_json",
@@ -16834,7 +16833,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -16845,7 +16844,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16856,7 +16855,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16869,7 +16868,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -16893,7 +16892,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -16907,7 +16906,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16924,7 +16923,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -16939,7 +16938,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -16956,25 +16955,9 @@ dependencies = [
"windmill-parser-sql",
]
[[package]]
name = "windmill-parser-wac"
version = "1.654.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"serde",
"serde_json",
"sha2 0.10.9",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"swc_ecma_visit",
]
[[package]]
name = "windmill-parser-yaml"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"serde",
@@ -16985,7 +16968,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -17022,7 +17005,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"const_format",
@@ -17060,7 +17043,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -17071,7 +17054,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -17100,7 +17083,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -17123,7 +17106,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17156,7 +17139,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17176,7 +17159,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17210,7 +17193,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17245,7 +17228,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17268,7 +17251,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17292,7 +17275,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-nats",
@@ -17316,7 +17299,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17351,7 +17334,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17379,7 +17362,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-trait",
@@ -17402,7 +17385,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"bitflags 2.9.4",
@@ -17420,7 +17403,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"anyhow",
"async-once-cell",
@@ -17526,7 +17509,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.654.0"
version = "1.651.1"
dependencies = [
"bytes",
"futures",
@@ -18126,9 +18109,9 @@ dependencies = [
[[package]]
name = "winnow"
version = "0.7.15"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
dependencies = [
"memchr",
]
@@ -18409,18 +18392,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.42"
version = "0.8.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.42"
version = "0.8.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
dependencies = [
"proc-macro2",
"quote",

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.654.0"
version = "1.651.1"
authors.workspace = true
edition.workspace = true
@@ -68,7 +68,6 @@ members = [
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-imports",
"./parsers/windmill-parser-wac",
"./parsers/windmill-sql-datatype-parser-wasm",
"./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu",
"./windmill-worker-volumes",
@@ -78,7 +77,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.654.0"
version = "1.651.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -333,7 +332,6 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-parser-wac = { path = "./parsers/windmill-parser-wac" }
windmill-jseval = { path = "./windmill-jseval" }
windmill-runtime-nativets = { path = "./windmill-runtime-nativets" }
windmill-api-client = { path = "./windmill-api-client" }

View File

@@ -1 +1 @@
cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64
c3c543f4c60a8c4dfe0d912c79a051376fb091a9

View File

@@ -0,0 +1 @@
ALTER TABLE worker_ping DROP COLUMN IF EXISTS uses_batch_http_pull;

View File

@@ -0,0 +1 @@
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS uses_batch_http_pull BOOLEAN NOT NULL DEFAULT false;

View File

@@ -1,2 +0,0 @@
ALTER TABLE kafka_trigger DROP COLUMN auto_offset_reset;
ALTER TABLE kafka_trigger DROP COLUMN reset_offset;

View File

@@ -1,2 +0,0 @@
ALTER TABLE kafka_trigger ADD COLUMN auto_offset_reset VARCHAR(10) NOT NULL DEFAULT 'latest';
ALTER TABLE kafka_trigger ADD COLUMN reset_offset BOOLEAN NOT NULL DEFAULT FALSE;

View File

@@ -1,2 +0,0 @@
ALTER TABLE job_stats DROP COLUMN IF EXISTS timeseries_start;
ALTER TABLE job_stats DROP COLUMN IF EXISTS offsets_cs;

View File

@@ -1,5 +0,0 @@
-- Store timeseries timestamps as a start time + integer centisecond offsets
-- instead of full TIMESTAMPTZ[] arrays. Saves ~4 bytes per data point.
-- i32 centiseconds gives ~248 days of range with 10ms precision.
ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS timeseries_start TIMESTAMPTZ;
ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS offsets_cs INTEGER[];

View File

@@ -1 +0,0 @@
DROP TABLE IF EXISTS audit_partitioned CASCADE;

View File

@@ -1,58 +0,0 @@
-- Create a new daily-partitioned audit table alongside the existing one.
-- New inserts go to audit_partitioned; reads UNION ALL both tables.
-- The old audit table empties out naturally via retention cleanup.
CREATE TABLE audit_partitioned (
workspace_id VARCHAR(50) NOT NULL,
id BIGINT NOT NULL DEFAULT nextval('audit_id_seq'),
timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
username VARCHAR(255) NOT NULL,
operation VARCHAR(50) NOT NULL,
action_kind ACTION_KIND NOT NULL,
resource VARCHAR(255),
parameters JSONB,
email VARCHAR(255),
span VARCHAR(255),
PRIMARY KEY (id, timestamp)
) PARTITION BY RANGE (timestamp);
-- Create daily partitions for today + 3 days
DO $$
DECLARE
curr_date DATE := CURRENT_DATE;
end_date DATE := CURRENT_DATE + INTERVAL '3 days';
BEGIN
WHILE curr_date <= end_date LOOP
EXECUTE format(
'CREATE TABLE %I PARTITION OF audit_partitioned FOR VALUES FROM (%L) TO (%L)',
'audit_' || to_char(curr_date, 'YYYYMMDD'),
curr_date,
curr_date + INTERVAL '1 day'
);
curr_date := curr_date + INTERVAL '1 day';
END LOOP;
END $$;
-- Indexes (auto-propagated to all current and future partitions)
CREATE INDEX ix_audit_partitioned_timestamps ON audit_partitioned (timestamp DESC);
CREATE INDEX idx_audit_partitioned_workspace ON audit_partitioned (workspace_id, timestamp DESC);
CREATE INDEX idx_audit_partitioned_recent_login_activities
ON audit_partitioned (timestamp, username)
WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');
-- Grants (match the old audit table)
GRANT ALL ON audit_partitioned TO windmill_user;
GRANT ALL ON audit_partitioned TO windmill_admin;
-- RLS (match the old audit table)
ALTER TABLE audit_partitioned ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON audit_partitioned FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_own ON audit_partitioned FOR ALL TO windmill_user
USING ((username)::text = current_setting('session.user'::text));
CREATE POLICY schedule ON audit_partitioned FOR INSERT TO windmill_user
WITH CHECK ((username)::text ~~ 'schedule-%'::text);
CREATE POLICY schedule_audit ON audit_partitioned FOR INSERT TO windmill_user
WITH CHECK ((parameters ->> 'end_user'::text) ~~ 'schedule-%'::text);
CREATE POLICY webhook ON audit_partitioned FOR INSERT TO windmill_user
WITH CHECK ((username)::text ~~ 'webhook-%'::text);

View File

@@ -296,14 +296,11 @@ pub fn parse_python_signature(
// Check if main function was found
if params.is_none() {
let is_wac_v2 = (code.contains("@workflow") || code.contains("workflow("))
&& (code.contains("@task") || code.contains("task("))
&& (code.contains("import wmill") || code.contains("from wmill"));
return Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(!is_wac_v2),
no_main_func: Some(true),
has_preprocessor: Some(has_preprocessor),
});
}

View File

@@ -238,7 +238,7 @@ lazy_static::lazy_static! {
// used for `unsafe` sql interpolation
// -- %%name%% (type) = default
static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%[ \t]*([\w][\w \t\/]*)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
static ref RE_ARG_SQL_INTERPOLATION: Regex = Regex::new(r#"(?m)^--\s*%%([a-z_][a-z0-9_]*)%%\s*([\s\w\/]+)?(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
}
fn parsed_default(parsed_typ: &Typ, default: String) -> Option<serde_json::Value> {
@@ -1547,36 +1547,4 @@ SELECT $1::integer;
Ok(())
}
#[test]
fn test_parse_pgsql_safe_interpolated_args() -> anyhow::Result<()> {
// There was a bug where enum would be "angrycreative"/"bishop"/"test SELECT x"
let code = r#"
-- %%table_name%% angrycreative/bishop/test
SELECT x
"#;
assert_eq!(
parse_pgsql_sig(code)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![Arg {
otyp: Some("__sanitized_enum__".to_string()),
name: "table_name".to_string(),
typ: Typ::Str(Some(vec![
"angrycreative".to_string(),
"bishop".to_string(),
"test".to_string()
])),
default: None,
has_default: false,
oidx: None,
},],
no_main_func: None,
has_preprocessor: None
}
);
Ok(())
}
}

View File

@@ -261,9 +261,7 @@ pub fn parse_deno_signature(
for specifier in &named_export.specifiers {
if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier {
let export_name = match &spec.exported {
Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => {
ident.sym.as_ref()
}
Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(),
Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(),
None => match &spec.orig {
swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(),
@@ -317,11 +315,7 @@ pub fn parse_deno_signature(
let mut c: u16 = 0;
let is_wac_v2 = entrypoint_params.is_none()
&& code.contains("workflow(")
&& code.contains("task(")
&& code.contains("windmill-client");
let no_main_func = entrypoint_params.is_none() && !is_wac_v2;
let no_main_func = entrypoint_params.is_none();
let mut type_resolver = HashMap::new();
let r = MainArgSignature {
star_args: false,
@@ -839,9 +833,7 @@ fn tstype_to_typ(
false,
),
symbol @ _ if symbol.starts_with("DynMultiselect_") => (
Typ::DynMultiselect(
symbol.strip_prefix("DynMultiselect_").unwrap().to_string(),
),
Typ::DynMultiselect(symbol.strip_prefix("DynMultiselect_").unwrap().to_string()),
false,
),
symbol @ _ => {

View File

@@ -1,21 +0,0 @@
[package]
name = "windmill-parser-wac"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_wac"
path = "./src/lib.rs"
[dependencies]
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
swc_common.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
swc_ecma_visit.workspace = true
serde.workspace = true
serde_json.workspace = true
anyhow.workspace = true
sha2.workspace = true

View File

@@ -1,44 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct WorkflowDag {
pub nodes: Vec<DagNode>,
pub edges: Vec<DagEdge>,
pub params: Vec<Param>,
pub source_hash: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Param {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub typ: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DagNode {
pub id: String,
pub node_type: DagNodeType,
pub label: String,
pub line: usize,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(tag = "type")]
pub enum DagNodeType {
Step { name: String, script: String },
Branch { condition_source: String },
ParallelStart,
ParallelEnd,
LoopStart { iter_source: String },
LoopEnd,
Return,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DagEdge {
pub from: String,
pub to: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}

View File

@@ -1,32 +0,0 @@
pub mod dag;
pub mod python;
pub mod typescript;
pub mod validation;
use dag::WorkflowDag;
use validation::CompileError;
#[derive(Debug, serde::Serialize)]
#[serde(tag = "type")]
pub enum ParseResult {
#[serde(rename = "success")]
Success(WorkflowDag),
#[serde(rename = "error")]
Error { errors: Vec<CompileError> },
}
pub fn parse_workflow(code: &str, language: &str) -> ParseResult {
let result = match language {
"python" | "python3" | "py" => python::parse_python_workflow(code),
"typescript" | "ts" | "deno" | "bun" => typescript::parse_ts_workflow(code),
_ => Err(vec![CompileError {
message: format!("Unsupported language: {language}"),
line: 0,
}]),
};
match result {
Ok(dag) => ParseResult::Success(dag),
Err(errors) => ParseResult::Error { errors },
}
}

View File

@@ -1,717 +0,0 @@
use std::collections::HashMap;
use rustpython_parser::{
ast::{
Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
StmtTryStar, StmtWhile,
},
Parse,
};
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
use crate::validation::{self, CompileError};
struct LineIndex {
newline_offsets: Vec<usize>,
}
impl LineIndex {
fn new(source: &str) -> Self {
let mut offsets = vec![0];
for (i, c) in source.char_indices() {
if c == '\n' {
offsets.push(i + 1);
}
}
Self { newline_offsets: offsets }
}
fn line_of(&self, byte_offset: usize) -> usize {
match self.newline_offsets.binary_search(&byte_offset) {
Ok(line) => line + 1,
Err(line) => line,
}
}
}
/// Maps task function name → optional external path (from `@task(path="...")`)
type TaskFunctions = HashMap<String, Option<String>>;
/// First pass: scan top-level `@task async def foo(...)` declarations.
fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
let mut tasks = HashMap::new();
for stmt in stmts {
if let Stmt::AsyncFunctionDef(func) = stmt {
for dec in &func.decorator_list {
match dec {
// @task (bare decorator)
Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
tasks.insert(func.name.to_string(), None);
}
// @task(path="...")
Expr::Call(call) => {
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
if id.as_str() == "task" {
let path = extract_task_path_kwarg(call);
tasks.insert(func.name.to_string(), path);
}
}
}
_ => {}
}
}
}
}
tasks
}
/// Extract the `path=` keyword argument from a `@task(path="...")` call.
fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
for kw in &call.keywords {
if let Some(ref arg) = kw.arg {
if arg.as_str() == "path" {
if let Expr::Constant(c) = &kw.value {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
}
}
}
None
}
struct WacWalker {
nodes: Vec<DagNode>,
edges: Vec<DagEdge>,
errors: Vec<CompileError>,
node_counter: usize,
line_index: LineIndex,
task_functions: TaskFunctions,
in_try: bool,
in_while: bool,
in_nested_func: bool,
in_comprehension: bool,
}
impl WacWalker {
fn new(source: &str, task_functions: TaskFunctions) -> Self {
Self {
nodes: Vec::new(),
edges: Vec::new(),
errors: Vec::new(),
node_counter: 0,
line_index: LineIndex::new(source),
task_functions,
in_try: false,
in_while: false,
in_nested_func: false,
in_comprehension: false,
}
}
fn next_id(&mut self) -> String {
let id = format!("step_{}", self.node_counter);
self.node_counter += 1;
id
}
fn add_node(&mut self, node: DagNode) -> String {
let id = node.id.clone();
self.nodes.push(node);
id
}
fn add_edge(&mut self, from: &str, to: &str, label: Option<String>) {
self.edges
.push(DagEdge { from: from.to_string(), to: to.to_string(), label });
}
fn line_of_expr(&self, expr: &Expr) -> usize {
let offset = match expr {
Expr::Call(c) => c.range.start().to_usize(),
Expr::Await(a) => a.range.start().to_usize(),
Expr::Attribute(a) => a.range.start().to_usize(),
Expr::Name(n) => n.range.start().to_usize(),
_ => 0,
};
self.line_index.line_of(offset)
}
fn line_of_stmt(&self, stmt: &Stmt) -> usize {
let offset = match stmt {
Stmt::If(s) => s.range.start().to_usize(),
Stmt::For(s) => s.range.start().to_usize(),
Stmt::While(s) => s.range.start().to_usize(),
Stmt::Return(s) => s.range.start().to_usize(),
Stmt::Expr(s) => s.range.start().to_usize(),
Stmt::Try(s) => s.range.start().to_usize(),
Stmt::TryStar(s) => s.range.start().to_usize(),
Stmt::Assign(s) => s.range.start().to_usize(),
Stmt::AnnAssign(s) => s.range.start().to_usize(),
Stmt::FunctionDef(s) => s.range.start().to_usize(),
Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
_ => 0,
};
self.line_index.line_of(offset)
}
/// Check if an expression is a call to a known @task function
fn is_task_fn_call(&self, expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
return self.task_functions.contains_key(id.as_str());
}
}
false
}
/// Check if an expression is `asyncio.gather(...)` call
fn is_asyncio_gather_call(expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
call.func.as_ref()
{
if attr.as_str() == "gather" {
if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
return id.as_str() == "asyncio";
}
}
}
}
false
}
/// Extract step name and script from a task function call.
/// Name = function name, script = task_path or function name.
fn extract_step_info_from_task_call(&self, call: &ExprCall) -> Option<(String, String)> {
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
let name = id.to_string();
let script = self
.task_functions
.get(id.as_str())
.and_then(|p| p.clone())
.unwrap_or_else(|| name.clone());
Some((name, script))
} else {
None
}
}
fn expr_to_source(expr: &Expr) -> String {
match expr {
Expr::Compare(c) => {
let left = Self::expr_to_source(&c.left);
if let Some(comparator) = c.comparators.first() {
let right = Self::expr_to_source(comparator);
let op = match c.ops.first() {
Some(rustpython_parser::ast::CmpOp::Gt) => ">",
Some(rustpython_parser::ast::CmpOp::Lt) => "<",
Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
Some(rustpython_parser::ast::CmpOp::Eq) => "==",
Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
Some(rustpython_parser::ast::CmpOp::In) => "in",
Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
Some(rustpython_parser::ast::CmpOp::Is) => "is",
Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
None => "?",
};
format!("{left} {op} {right}")
} else {
left
}
}
Expr::Subscript(s) => {
let value = Self::expr_to_source(&s.value);
let slice = Self::expr_to_source(&s.slice);
format!("{value}[{slice}]")
}
Expr::Attribute(a) => {
let value = Self::expr_to_source(&a.value);
format!("{value}.{}", a.attr)
}
Expr::Name(n) => n.id.to_string(),
Expr::Constant(c) => match &c.value {
rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
rustpython_parser::ast::Constant::Int(i) => i.to_string(),
rustpython_parser::ast::Constant::Float(f) => f.to_string(),
rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
rustpython_parser::ast::Constant::None => "None".to_string(),
_ => "...".to_string(),
},
_ => "...".to_string(),
}
}
/// Check if a statement body contains any task function calls (recursively)
fn body_contains_step(&self, body: &[Stmt]) -> bool {
for stmt in body {
if self.stmt_contains_step(stmt) {
return true;
}
}
false
}
fn stmt_contains_step(&self, stmt: &Stmt) -> bool {
match stmt {
Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
Stmt::Assign(a) => self.expr_contains_step(&a.value),
Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::While(s) => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
Stmt::Try(s) => {
self.body_contains_step(&s.body)
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
})
}
Stmt::TryStar(s) => {
self.body_contains_step(&s.body)
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
})
}
Stmt::Return(_) => false,
_ => false,
}
}
fn expr_contains_step(&self, expr: &Expr) -> bool {
if self.is_task_fn_call(expr) {
return true;
}
match expr {
Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
Expr::Call(call) => {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
return true;
}
if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
return call.args.iter().any(|a| self.expr_contains_step(a));
}
false
}
_ => false,
}
}
/// Walk a list of statements, returning (first_node_id, last_node_id)
fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> {
let mut first_id: Option<String> = None;
let mut prev_id: Option<String> = None;
for stmt in body {
if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) {
if let Some(ref prev) = prev_id {
self.add_edge(prev, &stmt_first, None);
}
if first_id.is_none() {
first_id = Some(stmt_first);
}
prev_id = Some(stmt_last);
}
}
match (first_id, prev_id) {
(Some(f), Some(l)) => Some((f, l)),
_ => None,
}
}
fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> {
match stmt {
Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
Stmt::If(if_stmt) => self.walk_if(if_stmt),
Stmt::For(for_stmt) => self.walk_for(for_stmt),
Stmt::While(while_stmt) => self.walk_while(while_stmt),
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
Stmt::Return(ret) => self.walk_return(ret),
Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
if self.stmt_contains_step(stmt) {
self.errors.push(validation::error_step_in_nested_function(
self.line_of_stmt(stmt),
));
}
None
}
_ => None,
}
}
fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> {
// await task_fn(...)
if let Expr::Await(ExprAwait { value, .. }) = expr {
// await task_fn(...)
if let Expr::Call(call) = value.as_ref() {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
return self.emit_step(call, expr);
}
}
// await asyncio.gather(task_fn(...), task_fn(...), ...)
if Self::is_asyncio_gather_call(value) {
if let Expr::Call(gather_call) = value.as_ref() {
return self.emit_parallel(gather_call, expr);
}
}
}
// Bare task_fn() without await — validation error
if self.is_task_fn_call(expr) {
self.errors
.push(validation::error_missing_await(self.line_of_expr(expr)));
}
None
}
fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> {
if self.in_try {
self.errors
.push(validation::error_step_in_try(self.line_of_expr(expr)));
return None;
}
if self.in_while {
self.errors
.push(validation::error_step_in_while(self.line_of_expr(expr)));
return None;
}
if self.in_nested_func {
self.errors.push(validation::error_step_in_nested_function(
self.line_of_expr(expr),
));
return None;
}
if self.in_comprehension {
self.errors.push(validation::error_step_in_comprehension(
self.line_of_expr(expr),
));
return None;
}
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
node_type: DagNodeType::Step { name: name.clone(), script },
label: name,
line: self.line_of_expr(expr),
});
Some((node_id.clone(), node_id))
}
fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> {
if self.in_try {
self.errors
.push(validation::error_step_in_try(self.line_of_expr(expr)));
return None;
}
if self.in_while {
self.errors
.push(validation::error_step_in_while(self.line_of_expr(expr)));
return None;
}
let line = self.line_of_expr(expr);
let start_id = self.next_id();
let start_node_id = self.add_node(DagNode {
id: start_id.clone(),
node_type: DagNodeType::ParallelStart,
label: "parallel".to_string(),
line,
});
let mut step_ids = Vec::new();
for arg in &gather_call.args {
// Each arg should be task_fn(...)
if let Expr::Call(call) = arg {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
let step_id = self.next_id();
let node_id = self.add_node(DagNode {
id: step_id.clone(),
node_type: DagNodeType::Step { name: name.clone(), script },
label: name,
line: self.line_of_expr(arg),
});
self.add_edge(&start_node_id, &node_id, None);
step_ids.push(node_id);
}
}
}
let end_id = self.next_id();
let end_node_id = self.add_node(DagNode {
id: end_id.clone(),
node_type: DagNodeType::ParallelEnd,
label: "join".to_string(),
line,
});
for step_id in &step_ids {
self.add_edge(step_id, &end_node_id, None);
}
Some((start_node_id, end_node_id))
}
fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(&if_stmt.body);
let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
if !has_steps_in_body && !has_steps_in_else {
return None;
}
let line = self.line_index.line_of(if_stmt.range.start().to_usize());
let condition_source = Self::expr_to_source(&if_stmt.test);
let branch_id = self.next_id();
let branch_node_id = self.add_node(DagNode {
id: branch_id.clone(),
node_type: DagNodeType::Branch { condition_source },
label: "if".to_string(),
line,
});
let merge_id = format!("{branch_id}_merge");
let mut last_ids = Vec::new();
if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
last_ids.push(true_last);
} else {
last_ids.push(branch_node_id.clone());
}
if !if_stmt.orelse.is_empty() {
if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
last_ids.push(else_last);
} else {
last_ids.push(branch_node_id.clone());
}
}
if last_ids.len() == 1 {
Some((branch_node_id, last_ids.into_iter().next().unwrap()))
} else {
Some((branch_node_id, merge_id))
}
}
fn walk_for(&mut self, for_stmt: &StmtFor) -> Option<(String, String)> {
if !self.body_contains_step(&for_stmt.body) {
return None;
}
let line = self.line_index.line_of(for_stmt.range.start().to_usize());
let iter_source = Self::expr_to_source(&for_stmt.iter);
let start_id = self.next_id();
let start_node_id = self.add_node(DagNode {
id: start_id.clone(),
node_type: DagNodeType::LoopStart { iter_source },
label: "for".to_string(),
line,
});
if let Some((body_first, body_last)) = self.walk_body(&for_stmt.body) {
self.add_edge(&start_node_id, &body_first, None);
self.add_edge(&body_last, &start_node_id, Some("next".to_string()));
}
let end_id = self.next_id();
let end_node_id = self.add_node(DagNode {
id: end_id.clone(),
node_type: DagNodeType::LoopEnd,
label: "end for".to_string(),
line,
});
self.add_edge(&start_node_id, &end_node_id, Some("done".to_string()));
Some((start_node_id, end_node_id))
}
fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> {
if self.body_contains_step(&while_stmt.body) {
let line = self.line_index.line_of(while_stmt.range.start().to_usize());
self.errors.push(validation::error_step_in_while(line));
}
None
}
fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> {
let has_steps = self.body_contains_step(&try_stmt.body)
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
});
if has_steps {
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
self.errors.push(validation::error_step_in_try(line));
}
None
}
fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
let has_steps = self.body_contains_step(&try_stmt.body)
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
});
if has_steps {
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
self.errors.push(validation::error_step_in_try(line));
}
None
}
fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
let line = self.line_index.line_of(ret.range.start().to_usize());
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
node_type: DagNodeType::Return,
label: "return".to_string(),
line,
});
Some((node_id.clone(), node_id))
}
}
/// Extract workflow function parameters (no longer skips ctx)
fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
let mut params = Vec::new();
for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
let name = arg_with_default.def.arg.to_string();
let typ = arg_with_default
.def
.annotation
.as_ref()
.map(|ann| WacWalker::expr_to_source(ann));
params.push(Param { name, typ });
}
params
}
pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
// First pass: collect @task functions
let task_functions = collect_task_functions(&ast);
// Find the @workflow async def
let workflow_fn = ast.iter().find_map(|stmt| {
if let Stmt::AsyncFunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return Some(func);
}
}
// Also check non-async for error reporting
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return None; // Will be reported as not-async below
}
}
None
});
// Check for non-async workflow function
let non_async_workflow = ast.iter().find_map(|stmt| {
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
let line_index = LineIndex::new(code);
return Some(line_index.line_of(func.range.start().to_usize()));
}
}
None
});
if let Some(line) = non_async_workflow {
if workflow_fn.is_none() {
return Err(vec![validation::error_not_async(line)]);
}
}
let workflow_fn = workflow_fn.ok_or_else(|| {
vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
})?;
let params = extract_params(&workflow_fn.args);
let source_hash = compute_source_hash(code);
let mut walker = WacWalker::new(code, task_functions);
walker.walk_body(&workflow_fn.body);
if !walker.errors.is_empty() {
return Err(walker.errors);
}
Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash })
}
fn compute_source_hash(code: &str) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(code.as_bytes());
format!("{:x}", hasher.finalize())
}
trait ToUsize {
fn to_usize(self) -> usize;
}
impl ToUsize for rustpython_parser::text_size::TextSize {
fn to_usize(self) -> usize {
u32::from(self) as usize
}
}

View File

@@ -1,739 +0,0 @@
use std::collections::HashMap;
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
use swc_ecma_ast::*;
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax};
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
use crate::validation::{self, CompileError};
/// Maps task function name → optional external path (from `task("f/path", ...)`)
type TaskFunctions = HashMap<String, Option<String>>;
/// First pass: scan top-level `const foo = task(async (...) => {})` or
/// `const foo = task("f/path", async (...) => {})` declarations.
fn collect_task_functions(module: &Module) -> TaskFunctions {
let mut tasks = HashMap::new();
for item in &module.body {
// const foo = task(async (...) => { ... })
// const foo = task("f/path", async (...) => { ... })
if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item {
for decl in &var_decl.decls {
if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) {
if let Some(path) = extract_task_call_info(init) {
tasks.insert(name, path);
}
}
}
}
// export const foo = task(...)
if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item {
if let Decl::Var(var_decl) = &export.decl {
for decl in &var_decl.decls {
if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) {
if let Some(path) = extract_task_call_info(init) {
tasks.insert(name, path);
}
}
}
}
}
}
tasks
}
/// Extract variable name from a pattern (simple ident case)
fn extract_var_name(pat: &Pat) -> Option<String> {
if let Pat::Ident(BindingIdent { id, .. }) = pat {
Some(id.sym.to_string())
} else {
None
}
}
/// Check if expr is `task(async fn)` or `task("path", async fn)`.
/// Returns Some(optional_path) if it is a task() call.
fn extract_task_call_info(expr: &Expr) -> Option<Option<String>> {
if let Expr::Call(call) = expr {
if let Callee::Expr(callee) = &call.callee {
if let Expr::Ident(ident) = callee.as_ref() {
if ident.sym.as_ref() == "task" {
// task("f/path", async fn) or task(async fn)
if call.args.len() == 2 {
// task("f/path", async fn)
let path = extract_string_lit(&call.args[0].expr);
return Some(path);
} else if call.args.len() == 1 {
// task(async fn)
return Some(None);
}
}
}
}
}
None
}
struct TsWacWalker {
nodes: Vec<DagNode>,
edges: Vec<DagEdge>,
errors: Vec<CompileError>,
node_counter: usize,
cm: Lrc<SourceMap>,
task_functions: TaskFunctions,
in_try: bool,
in_while: bool,
in_nested_func: bool,
}
impl TsWacWalker {
fn new(cm: Lrc<SourceMap>, task_functions: TaskFunctions) -> Self {
Self {
nodes: Vec::new(),
edges: Vec::new(),
errors: Vec::new(),
node_counter: 0,
cm,
task_functions,
in_try: false,
in_while: false,
in_nested_func: false,
}
}
fn next_id(&mut self) -> String {
let id = format!("step_{}", self.node_counter);
self.node_counter += 1;
id
}
fn add_node(&mut self, node: DagNode) -> String {
let id = node.id.clone();
self.nodes.push(node);
id
}
fn add_edge(&mut self, from: &str, to: &str, label: Option<String>) {
self.edges
.push(DagEdge { from: from.to_string(), to: to.to_string(), label });
}
fn span_line(&self, span: swc_common::Span) -> usize {
let loc = self.cm.lookup_char_pos(span.lo);
loc.line
}
/// Check if expr is a call to a known task function
fn is_task_call(&self, expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Callee::Expr(callee) = &call.callee {
if let Expr::Ident(ident) = callee.as_ref() {
return self.task_functions.contains_key(ident.sym.as_ref());
}
}
}
false
}
/// Check if expr is `Promise.all([...])`
fn is_promise_all(expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Callee::Expr(callee) = &call.callee {
if let Expr::Member(MemberExpr { obj, prop: MemberProp::Ident(prop), .. }) =
callee.as_ref()
{
if prop.sym.as_ref() == "all" {
if let Expr::Ident(ident) = obj.as_ref() {
return ident.sym.as_ref() == "Promise";
}
}
}
}
}
false
}
/// Extract step name and script from a task function call.
/// Name = function name, script = task_path or function name.
fn extract_step_info_from_task_call(&self, call: &CallExpr) -> Option<(String, String)> {
if let Callee::Expr(callee) = &call.callee {
if let Expr::Ident(ident) = callee.as_ref() {
let name = ident.sym.to_string();
let script = self
.task_functions
.get(ident.sym.as_ref())
.and_then(|p| p.clone())
.unwrap_or_else(|| name.clone());
return Some((name, script));
}
}
None
}
fn expr_to_source(&self, expr: &Expr) -> String {
let span = expr.span();
self.cm
.span_to_snippet(span)
.unwrap_or_else(|_| "...".to_string())
}
fn body_contains_step(&self, stmts: &[Stmt]) -> bool {
stmts.iter().any(|s| self.stmt_contains_step(s))
}
fn stmt_contains_step(&self, stmt: &Stmt) -> bool {
match stmt {
Stmt::Expr(expr_stmt) => self.expr_contains_step(&expr_stmt.expr),
Stmt::Decl(Decl::Var(var_decl)) => var_decl.decls.iter().any(|d| {
d.init
.as_ref()
.map_or(false, |init| self.expr_contains_step(init))
}),
Stmt::If(if_stmt) => {
self.stmt_contains_step(&if_stmt.cons)
|| if_stmt
.alt
.as_ref()
.map_or(false, |alt| self.stmt_contains_step(alt))
}
Stmt::Block(block) => self.body_contains_step(&block.stmts),
Stmt::For(for_stmt) => self.stmt_contains_step(&for_stmt.body),
Stmt::ForIn(for_in) => self.stmt_contains_step(&for_in.body),
Stmt::ForOf(for_of) => self.stmt_contains_step(&for_of.body),
Stmt::While(while_stmt) => self.stmt_contains_step(&while_stmt.body),
Stmt::Try(try_stmt) => {
self.body_contains_step(&try_stmt.block.stmts)
|| try_stmt
.handler
.as_ref()
.map_or(false, |h| self.body_contains_step(&h.body.stmts))
|| try_stmt
.finalizer
.as_ref()
.map_or(false, |f| self.body_contains_step(&f.stmts))
}
Stmt::Return(ret) => ret
.arg
.as_ref()
.map_or(false, |arg| self.expr_contains_step(arg)),
_ => false,
}
}
fn expr_contains_step(&self, expr: &Expr) -> bool {
if self.is_task_call(expr) {
return true;
}
match expr {
Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg),
Expr::Call(call) => {
if Self::is_promise_all(&Expr::Call(call.clone())) {
return call.args.iter().any(|a| self.expr_contains_step(&a.expr));
}
false
}
Expr::Paren(p) => self.expr_contains_step(&p.expr),
_ => false,
}
}
fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> {
let mut first_id: Option<String> = None;
let mut prev_id: Option<String> = None;
for stmt in stmts {
if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) {
if let Some(ref prev) = prev_id {
self.add_edge(prev, &stmt_first, None);
}
if first_id.is_none() {
first_id = Some(stmt_first);
}
prev_id = Some(stmt_last);
}
}
match (first_id, prev_id) {
(Some(f), Some(l)) => Some((f, l)),
_ => None,
}
}
fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> {
match stmt {
Stmt::Expr(expr_stmt) => self.walk_expr_stmt(&expr_stmt.expr),
Stmt::Decl(Decl::Var(var_decl)) => {
// const result = await task_fn(...)
for decl in &var_decl.decls {
if let Some(init) = &decl.init {
if let Some(result) = self.walk_expr_stmt(init) {
return Some(result);
}
}
}
None
}
Stmt::If(if_stmt) => self.walk_if(if_stmt),
Stmt::For(for_stmt) => self.walk_for_stmt(for_stmt),
Stmt::ForIn(for_in) => self.walk_for_in(for_in),
Stmt::ForOf(for_of) => self.walk_for_of(for_of),
Stmt::While(while_stmt) => self.walk_while(while_stmt),
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
Stmt::Block(block) => self.walk_body(&block.stmts),
Stmt::Return(ret) => self.walk_return(ret),
Stmt::Decl(Decl::Fn(_)) => {
if self.stmt_contains_step(stmt) {
self.errors.push(validation::error_step_in_nested_function(
self.span_line(stmt.span()),
));
}
None
}
_ => None,
}
}
fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> {
// await task_fn(...)
if let Expr::Await(await_expr) = expr {
if let Expr::Call(call) = await_expr.arg.as_ref() {
if self.is_task_call(&Expr::Call(call.clone())) {
return self.emit_step(call, expr);
}
}
// await Promise.all([task_fn(...), ...])
if Self::is_promise_all(&await_expr.arg) {
if let Expr::Call(promise_call) = await_expr.arg.as_ref() {
return self.emit_parallel(promise_call, expr);
}
}
}
// Bare task_fn() without await
if self.is_task_call(expr) {
self.errors
.push(validation::error_missing_await(self.span_line(expr.span())));
}
None
}
fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> {
if self.in_try {
self.errors
.push(validation::error_step_in_catch(self.span_line(expr.span())));
return None;
}
if self.in_while {
self.errors
.push(validation::error_step_in_while(self.span_line(expr.span())));
return None;
}
if self.in_nested_func {
self.errors.push(validation::error_step_in_nested_function(
self.span_line(expr.span()),
));
return None;
}
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
node_type: DagNodeType::Step { name: name.clone(), script },
label: name,
line: self.span_line(expr.span()),
});
Some((node_id.clone(), node_id))
}
fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> {
if self.in_try {
self.errors
.push(validation::error_step_in_catch(self.span_line(expr.span())));
return None;
}
if self.in_while {
self.errors
.push(validation::error_step_in_while(self.span_line(expr.span())));
return None;
}
let line = self.span_line(expr.span());
let start_id = self.next_id();
let start_node_id = self.add_node(DagNode {
id: start_id.clone(),
node_type: DagNodeType::ParallelStart,
label: "parallel".to_string(),
line,
});
let mut step_ids = Vec::new();
// Promise.all takes an array as first argument
if let Some(first_arg) = promise_call.args.first() {
if let Expr::Array(ArrayLit { elems, .. }) = first_arg.expr.as_ref() {
for elem in elems.iter().flatten() {
if let Expr::Call(call) = elem.expr.as_ref() {
if self.is_task_call(&Expr::Call(call.clone())) {
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
let step_id = self.next_id();
let node_id = self.add_node(DagNode {
id: step_id.clone(),
node_type: DagNodeType::Step { name: name.clone(), script },
label: name,
line: self.span_line(elem.expr.span()),
});
self.add_edge(&start_node_id, &node_id, None);
step_ids.push(node_id);
}
}
}
}
}
let end_id = self.next_id();
let end_node_id = self.add_node(DagNode {
id: end_id.clone(),
node_type: DagNodeType::ParallelEnd,
label: "join".to_string(),
line,
});
for step_id in &step_ids {
self.add_edge(step_id, &end_node_id, None);
}
Some((start_node_id, end_node_id))
}
fn walk_if(&mut self, if_stmt: &IfStmt) -> Option<(String, String)> {
let has_steps_cons = self.stmt_contains_step(&if_stmt.cons);
let has_steps_alt = if_stmt
.alt
.as_ref()
.map_or(false, |a| self.stmt_contains_step(a));
if !has_steps_cons && !has_steps_alt {
return None;
}
let line = self.span_line(if_stmt.span);
let condition_source = self.expr_to_source(&if_stmt.test);
let branch_id = self.next_id();
let branch_node_id = self.add_node(DagNode {
id: branch_id.clone(),
node_type: DagNodeType::Branch { condition_source },
label: "if".to_string(),
line,
});
let mut last_ids = Vec::new();
// True branch
if let Some((true_first, true_last)) = self.walk_stmt(&if_stmt.cons) {
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
last_ids.push(true_last);
} else {
last_ids.push(branch_node_id.clone());
}
// False branch
if let Some(alt) = &if_stmt.alt {
if let Some((else_first, else_last)) = self.walk_stmt(alt) {
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
last_ids.push(else_last);
} else {
last_ids.push(branch_node_id.clone());
}
}
if last_ids.len() == 1 {
Some((branch_node_id, last_ids.into_iter().next().unwrap()))
} else {
let merge_id = format!("{branch_id}_merge");
Some((branch_node_id, merge_id))
}
}
fn walk_for_stmt(&mut self, for_stmt: &ForStmt) -> Option<(String, String)> {
if !self.stmt_contains_step(&for_stmt.body) {
return None;
}
self.walk_loop_body(&for_stmt.body, for_stmt.span, "for")
}
fn walk_for_in(&mut self, for_in: &ForInStmt) -> Option<(String, String)> {
if !self.stmt_contains_step(&for_in.body) {
return None;
}
let iter_source = self.expr_to_source(&for_in.right);
self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source)
}
fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> {
if !self.stmt_contains_step(&for_of.body) {
return None;
}
let iter_source = self.expr_to_source(&for_of.right);
self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source)
}
fn walk_loop_body(
&mut self,
body: &Stmt,
span: swc_common::Span,
_label: &str,
) -> Option<(String, String)> {
self.walk_loop_body_with_iter(body, span, "...")
}
fn walk_loop_body_with_iter(
&mut self,
body: &Stmt,
span: swc_common::Span,
iter_source: &str,
) -> Option<(String, String)> {
let line = self.span_line(span);
let start_id = self.next_id();
let start_node_id = self.add_node(DagNode {
id: start_id.clone(),
node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() },
label: "for".to_string(),
line,
});
if let Some((body_first, body_last)) = self.walk_stmt(body) {
self.add_edge(&start_node_id, &body_first, None);
self.add_edge(&body_last, &start_node_id, Some("next".to_string()));
}
let end_id = self.next_id();
let end_node_id = self.add_node(DagNode {
id: end_id.clone(),
node_type: DagNodeType::LoopEnd,
label: "end for".to_string(),
line,
});
self.add_edge(&start_node_id, &end_node_id, Some("done".to_string()));
Some((start_node_id, end_node_id))
}
fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> {
if self.stmt_contains_step(&while_stmt.body) {
self.errors.push(validation::error_step_in_while(
self.span_line(while_stmt.span),
));
}
None
}
fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> {
let has_steps = self.body_contains_step(&try_stmt.block.stmts)
|| try_stmt
.handler
.as_ref()
.map_or(false, |h| self.body_contains_step(&h.body.stmts))
|| try_stmt
.finalizer
.as_ref()
.map_or(false, |f| self.body_contains_step(&f.stmts));
if has_steps {
self.errors.push(validation::error_step_in_catch(
self.span_line(try_stmt.span),
));
}
None
}
fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> {
let line = self.span_line(ret.span);
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
node_type: DagNodeType::Return,
label: "return".to_string(),
line,
});
Some((node_id.clone(), node_id))
}
}
/// Extract workflow function params (no longer skips ctx)
fn extract_ts_params(params: &[swc_ecma_ast::Param], cm: &Lrc<SourceMap>) -> Vec<Param> {
let mut result = Vec::new();
for param in params {
let (name, typ) = match &param.pat {
Pat::Ident(BindingIdent { id, type_ann, .. }) => {
let name = id.sym.to_string();
let typ = type_ann.as_ref().map(|ann| {
cm.span_to_snippet(ann.type_ann.span())
.unwrap_or_else(|_| "unknown".to_string())
});
(name, typ)
}
_ => continue,
};
result.push(Param { name, typ });
}
result
}
pub fn parse_ts_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
let cm: Lrc<SourceMap> = Default::default();
let fm = cm.new_source_file(FileName::Custom("workflow.ts".into()).into(), code.into());
let lexer = Lexer::new(
Syntax::Typescript(TsSyntax::default()),
Default::default(),
StringInput::from(&*fm),
None,
);
let mut parser = Parser::new_from(lexer);
let module = parser
.parse_module()
.map_err(|e| vec![CompileError { message: format!("Parse error: {e:?}"), line: 0 }])?;
// First pass: collect task functions
let task_functions = collect_task_functions(&module);
// Find: export default workflow(async (...) => { ... })
// or: export default workflow(async function(...) { ... })
let mut workflow_body: Option<(&[Stmt], Vec<Param>)> = None;
for item in &module.body {
// export default workflow(async (...) => { ... })
if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) = item {
if let Some(result) = find_workflow_call(&export.expr, &cm) {
workflow_body = Some(result);
break;
}
}
// const wf = workflow(async (...) => { ... }); export default wf;
if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultDecl(export)) = item {
if let DefaultDecl::Fn(_) = &export.decl {
// `export default async function(...) { ... }` — not wrapped in workflow(), skip
}
}
if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item {
for decl in &var_decl.decls {
if let Some(init) = &decl.init {
if let Some(result) = find_workflow_call(init, &cm) {
workflow_body = Some(result);
break;
}
}
}
}
}
let (stmts, params) = workflow_body.ok_or_else(|| {
vec![CompileError {
message: "No workflow() wrapped async function found.".to_string(),
line: 0,
}]
})?;
let source_hash = compute_source_hash(code);
let mut walker = TsWacWalker::new(cm, task_functions);
walker.walk_body(stmts);
if !walker.errors.is_empty() {
return Err(walker.errors);
}
Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash })
}
/// Find workflow(async (...) => { ... }) or workflow(async function(...) { ... })
fn find_workflow_call<'a>(expr: &'a Expr, cm: &Lrc<SourceMap>) -> Option<(&'a [Stmt], Vec<Param>)> {
if let Expr::Call(call) = expr {
// Check if callee is `workflow`
let is_workflow = match &call.callee {
Callee::Expr(callee_expr) => {
if let Expr::Ident(ident) = callee_expr.as_ref() {
ident.sym.as_ref() == "workflow"
} else {
false
}
}
_ => false,
};
if is_workflow {
if let Some(first_arg) = call.args.first() {
return extract_async_fn_body(&first_arg.expr, cm);
}
}
}
None
}
fn extract_async_fn_body<'a>(
expr: &'a Expr,
cm: &Lrc<SourceMap>,
) -> Option<(&'a [Stmt], Vec<Param>)> {
match expr {
Expr::Arrow(arrow) if arrow.is_async => {
let params = extract_arrow_params(&arrow.params, cm);
match &*arrow.body {
BlockStmtOrExpr::BlockStmt(block) => Some((&block.stmts, params)),
_ => None,
}
}
Expr::Fn(fn_expr) if fn_expr.function.is_async => {
let params = extract_ts_params(&fn_expr.function.params, cm);
fn_expr
.function
.body
.as_ref()
.map(|body| (body.stmts.as_slice(), params))
}
Expr::Paren(p) => extract_async_fn_body(&p.expr, cm),
_ => None,
}
}
/// Extract arrow function params (no longer skips ctx)
fn extract_arrow_params(pats: &[Pat], cm: &Lrc<SourceMap>) -> Vec<Param> {
let mut result = Vec::new();
for pat in pats {
match pat {
Pat::Ident(BindingIdent { id, type_ann, .. }) => {
let name = id.sym.to_string();
let typ = type_ann.as_ref().map(|ann| {
cm.span_to_snippet(ann.type_ann.span())
.unwrap_or_else(|_| "unknown".to_string())
});
result.push(Param { name, typ });
}
_ => {}
}
}
result
}
fn extract_string_lit(expr: &Expr) -> Option<String> {
match expr {
Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()),
Expr::Tpl(tpl) if tpl.exprs.is_empty() && tpl.quasis.len() == 1 => {
tpl.quasis.first().map(|q| q.raw.to_string())
}
_ => None,
}
}
fn compute_source_hash(code: &str) -> String {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(code.as_bytes());
format!("{:x}", hasher.finalize())
}

View File

@@ -1,64 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct CompileError {
pub message: String,
pub line: usize,
}
impl std::fmt::Display for CompileError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "line {}: {}", self.line, self.message)
}
}
pub fn error_step_in_try(line: usize) -> CompileError {
CompileError {
message:
"Task calls inside try/except are not allowed. Steps have built-in error handling."
.to_string(),
line,
}
}
pub fn error_step_in_while(line: usize) -> CompileError {
CompileError {
message: "Task calls inside while loops are not allowed. Use for loops instead."
.to_string(),
line,
}
}
pub fn error_step_in_nested_function(line: usize) -> CompileError {
CompileError {
message: "Task calls inside nested functions, closures, or lambdas are not allowed."
.to_string(),
line,
}
}
pub fn error_step_in_comprehension(line: usize) -> CompileError {
CompileError { message: "Task calls inside comprehensions are not allowed.".to_string(), line }
}
pub fn error_not_async(line: usize) -> CompileError {
CompileError { message: "Workflow function must be async.".to_string(), line }
}
pub fn error_missing_await(line: usize) -> CompileError {
CompileError {
message:
"Task calls must be awaited directly or used inside asyncio.gather()/Promise.all()."
.to_string(),
line,
}
}
pub fn error_step_in_catch(line: usize) -> CompileError {
CompileError {
message:
"Task calls inside catch blocks are not allowed. Steps have built-in error handling."
.to_string(),
line,
}
}

View File

@@ -1,266 +0,0 @@
use windmill_parser_wac::dag::DagNodeType;
use windmill_parser_wac::python::parse_python_workflow;
#[test]
fn test_simple_sequential_workflow() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def extract_data(url: str): ...
@task
async def load_data(data: list): ...
@workflow
async def my_etl(url: str):
raw = await extract_data(url=url)
await load_data(data=raw)
return {"status": "done"}
"#;
let dag = parse_python_workflow(code).expect("should parse");
assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return
assert_eq!(dag.edges.len(), 2); // step0->step1, step1->return
// Check params (url — no ctx to skip)
assert_eq!(dag.params.len(), 1);
assert_eq!(dag.params[0].name, "url");
assert_eq!(dag.params[0].typ.as_deref(), Some("str"));
// Check first step
match &dag.nodes[0].node_type {
DagNodeType::Step { name, script } => {
assert_eq!(name, "extract_data");
assert_eq!(script, "extract_data");
}
_ => panic!("expected Step node"),
}
// Check second step
match &dag.nodes[1].node_type {
DagNodeType::Step { name, script } => {
assert_eq!(name, "load_data");
assert_eq!(script, "load_data");
}
_ => panic!("expected Step node"),
}
// Check return
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return));
// Check source hash is non-empty
assert!(!dag.source_hash.is_empty());
}
#[test]
fn test_parallel_workflow() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def extract_data(url: str): ...
@task
async def clean_data(data: list): ...
@task
async def compute_stats(data: list): ...
@task
async def load_to_warehouse(rows: list): ...
@workflow
async def my_etl(url: str):
raw = await extract_data(url=url)
cleaned, stats = await asyncio.gather(
clean_data(data=raw),
compute_stats(data=raw),
)
await load_to_warehouse(rows=cleaned)
return {"status": "done"}
"#;
let dag = parse_python_workflow(code).expect("should parse");
// extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7
assert_eq!(dag.nodes.len(), 7);
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart));
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd));
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return));
}
#[test]
fn test_conditional_workflow() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def send_alert(msg: str): ...
@task
async def load_data(): ...
@workflow
async def my_etl(count: int):
if count > 100:
await send_alert(msg="large")
await load_data()
return {"done": True}
"#;
let dag = parse_python_workflow(code).expect("should parse");
// Branch, notify step, load step, return = 4
assert_eq!(dag.nodes.len(), 4);
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. }));
assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. }));
}
#[test]
fn test_for_loop_workflow() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def process_item(item: str): ...
@workflow
async def my_etl(items: list):
for item in items:
await process_item(item=item)
return {"done": True}
"#;
let dag = parse_python_workflow(code).expect("should parse");
// LoopStart, step, LoopEnd, return = 4
assert_eq!(dag.nodes.len(), 4);
assert!(matches!(
dag.nodes[0].node_type,
DagNodeType::LoopStart { .. }
));
assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd));
}
#[test]
fn test_reject_step_in_try() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def extract_data(): ...
@workflow
async def my_etl():
try:
await extract_data()
except Exception:
pass
"#;
let result = parse_python_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("try/except"));
}
#[test]
fn test_reject_step_in_while() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def extract_data(): ...
@workflow
async def my_etl():
while True:
await extract_data()
"#;
let result = parse_python_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("while"));
}
#[test]
fn test_reject_non_async() {
let code = r#"
from wmill import workflow
@workflow
def my_etl():
pass
"#;
let result = parse_python_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("async"));
}
#[test]
fn test_reject_missing_await() {
let code = r#"
import asyncio
from wmill import workflow, task
@task
async def extract_data(): ...
@workflow
async def my_etl():
extract_data()
"#;
let result = parse_python_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("awaited"));
}
#[test]
fn test_no_workflow_function() {
let code = r#"
async def my_func():
pass
"#;
let result = parse_python_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("No @workflow"));
}
#[test]
fn test_task_with_external_path() {
let code = r#"
import asyncio
from wmill import workflow, task
@task(path="f/external_script")
async def run_external(x: int): ...
@workflow
async def my_wf(x: int):
result = await run_external(x=x)
return result
"#;
let dag = parse_python_workflow(code).expect("should parse");
assert_eq!(dag.nodes.len(), 2); // 1 step + 1 return (bare `return` is not a step node but walk_return creates one)
match &dag.nodes[0].node_type {
DagNodeType::Step { name, script } => {
assert_eq!(name, "run_external");
assert_eq!(script, "f/external_script");
}
_ => panic!("expected Step node"),
}
}

View File

@@ -1,245 +0,0 @@
use windmill_parser_wac::dag::DagNodeType;
use windmill_parser_wac::typescript::parse_ts_workflow;
#[test]
fn test_simple_sequential_ts_workflow() {
let code = r#"
import { workflow, task } from "windmill-client";
const extract_data = task(async (url: string) => {});
const load_data = task(async (data: any) => {});
export default workflow(async (url: string) => {
const raw = await extract_data(url);
await load_data(raw);
return { status: "done" };
});
"#;
let dag = parse_ts_workflow(code).expect("should parse");
assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return
assert_eq!(dag.edges.len(), 2);
// Check params (url — no ctx to skip)
assert_eq!(dag.params.len(), 1);
assert_eq!(dag.params[0].name, "url");
assert_eq!(dag.params[0].typ.as_deref(), Some("string"));
match &dag.nodes[0].node_type {
DagNodeType::Step { name, script } => {
assert_eq!(name, "extract_data");
assert_eq!(script, "extract_data");
}
_ => panic!("expected Step node"),
}
match &dag.nodes[1].node_type {
DagNodeType::Step { name, script } => {
assert_eq!(name, "load_data");
assert_eq!(script, "load_data");
}
_ => panic!("expected Step node"),
}
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return));
assert!(!dag.source_hash.is_empty());
}
#[test]
fn test_parallel_ts_workflow() {
let code = r#"
import { workflow, task } from "windmill-client";
const extract_data = task(async (url: string) => {});
const clean_data = task(async (data: any) => {});
const compute_stats = task(async (data: any) => {});
const load_to_warehouse = task(async (rows: any) => {});
export default workflow(async (url: string) => {
const raw = await extract_data(url);
const [cleaned, stats] = await Promise.all([
clean_data(raw),
compute_stats(raw),
]);
await load_to_warehouse(cleaned);
return { status: "done", rows: stats.rowCount };
});
"#;
let dag = parse_ts_workflow(code).expect("should parse");
// extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7
assert_eq!(dag.nodes.len(), 7);
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart));
assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd));
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. }));
assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return));
}
#[test]
fn test_conditional_ts_workflow() {
let code = r#"
import { workflow, task } from "windmill-client";
const send_alert = task(async (msg: string) => {});
const load_data = task(async () => {});
export default workflow(async (count: number) => {
if (count > 100) {
await send_alert("large");
}
await load_data();
return { done: true };
});
"#;
let dag = parse_ts_workflow(code).expect("should parse");
// Branch, notify, load, return = 4
assert_eq!(dag.nodes.len(), 4);
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. }));
}
#[test]
fn test_for_of_ts_workflow() {
let code = r#"
import { workflow, task } from "windmill-client";
const process_item = task(async (item: string) => {});
export default workflow(async (items: string[]) => {
for (const item of items) {
await process_item(item);
}
return { done: true };
});
"#;
let dag = parse_ts_workflow(code).expect("should parse");
// LoopStart, step, LoopEnd, return = 4
assert_eq!(dag.nodes.len(), 4);
assert!(matches!(
dag.nodes[0].node_type,
DagNodeType::LoopStart { .. }
));
}
#[test]
fn test_reject_step_in_try_catch() {
let code = r#"
import { workflow, task } from "windmill-client";
const extract_data = task(async () => {});
export default workflow(async () => {
try {
await extract_data();
} catch (e) {
console.log(e);
}
});
"#;
let result = parse_ts_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("catch"));
}
#[test]
fn test_reject_step_in_while_ts() {
let code = r#"
import { workflow, task } from "windmill-client";
const extract_data = task(async () => {});
export default workflow(async () => {
while (true) {
await extract_data();
}
});
"#;
let result = parse_ts_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("while"));
}
#[test]
fn test_reject_missing_await_ts() {
let code = r#"
import { workflow, task } from "windmill-client";
const extract_data = task(async () => {});
export default workflow(async () => {
extract_data();
});
"#;
let result = parse_ts_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("awaited"));
}
#[test]
fn test_no_workflow_wrapper() {
let code = r#"
export default async function main(ctx: any) {
return {};
}
"#;
let result = parse_ts_workflow(code);
assert!(result.is_err());
let errors = result.unwrap_err();
assert!(errors[0].message.contains("No workflow()"));
}
#[test]
fn test_variable_declaration_with_step() {
let code = r#"
import { workflow, task } from "windmill-client";
const compute = task(async () => {});
export default workflow(async () => {
const result = await compute();
return result;
});
"#;
let dag = parse_ts_workflow(code).expect("should parse");
assert_eq!(dag.nodes.len(), 2); // step + return
assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. }));
}
#[test]
fn test_task_with_external_path() {
let code = r#"
import { workflow, task } from "windmill-client";
const run_external = task("f/external_script", async (x: number) => {});
export default workflow(async (x: number) => {
const result = await run_external(x);
return result;
});
"#;
let dag = parse_ts_workflow(code).expect("should parse");
assert_eq!(dag.nodes.len(), 2); // step + return
match &dag.nodes[0].node_type {
DagNodeType::Step { name, script } => {
assert_eq!(name, "run_external");
assert_eq!(script, "f/external_script");
}
_ => panic!("expected Step node"),
}
}

View File

@@ -38,7 +38,6 @@ csharp-parser = [ "dep:windmill-parser-csharp"]
nu-parser = [ "dep:windmill-parser-nu"]
java-parser = [ "dep:windmill-parser-java"]
ruby-parser = [ "dep:windmill-parser-ruby"]
wac-parser = [ "dep:windmill-parser-wac"]
[dependencies]
anyhow.workspace = true
@@ -56,7 +55,6 @@ windmill-parser-csharp = { workspace = true, optional = true }
windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
windmill-parser-ruby = { workspace = true, optional = true }
windmill-parser-wac = { workspace = true, optional = true }
wasm-bindgen.workspace = true
serde_json.workspace = true

View File

@@ -56,12 +56,6 @@ const targets = [
features: "ruby-parser",
env: "tree-sitter",
},
{
ident: "wac",
desc: "Workflow-as-Code",
features: "wac-parser",
env: "default",
},
# ^^^ Add new entry here ^^^
];
# NOTE: This is legacy command for building all, but it is not more used

View File

@@ -223,11 +223,4 @@ pub fn parse_assets_ansible(code: &str) -> String {
}
}
#[cfg(feature = "wac-parser")]
#[wasm_bindgen]
pub fn parse_workflow_as_code(code: &str, language: &str) -> String {
let result = windmill_parser_wac::parse_workflow(code, language);
serde_json::to_string(&result).unwrap_or_else(|_| "{\"type\": \"error\"}".to_string())
}
// for related places search: ADD_NEW_LANG

View File

@@ -105,7 +105,6 @@ pub async fn connect(
let mut pool_options = sqlx::postgres::PgPoolOptions::new()
.min_connections((max_connections / 5).clamp(1, max_connections))
.max_connections(max_connections)
.acquire_timeout(Duration::from_secs(5))
.max_lifetime(Duration::from_secs(30 * 60)); // 30 mins
if worker_mode {
pool_options = pool_options.idle_timeout(Duration::from_secs(60));

View File

@@ -37,13 +37,12 @@ use windmill_common::ee_oss::{
use windmill_common::{
agent_workers::AgentConfig,
global_settings::{
APP_WORKSPACED_ROUTE_SETTING, AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING,
BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
APP_WORKSPACED_ROUTE_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING,
ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
@@ -62,8 +61,9 @@ use windmill_common::{
MODE_AND_ADDONS,
},
worker::{
is_native_mode_from_env, reload_custom_tags_setting, Connection, HUB_CACHE_DIR,
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, WINDMILL_DIR, WORKER_GROUP,
is_native_mode_from_env, reload_custom_tags_setting, Connection, HttpClient, HUB_CACHE_DIR,
HUB_RT_CACHE_DIR, NATIVE_MODE_RESOLVED, TMP_LOGS_DIR, USES_BATCH_HTTP_PULL, WINDMILL_DIR,
WORKER_GROUP,
},
KillpillSender, DEFAULT_HUB_BASE_URL, METRICS_ENABLED,
};
@@ -98,13 +98,12 @@ use windmill_worker::{
use crate::monitor::{
initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db,
reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting,
reload_base_url_setting, reload_bunfig_install_scopes_setting,
reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting,
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting,
reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key,
reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
reload_app_workspaced_route_setting, reload_base_url_setting,
reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting,
reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting,
reload_extra_pip_index_url_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting,
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration,
};
@@ -519,51 +518,6 @@ fn print_help() {
println!("- At startup, Windmill logs currently set configuration keys for visibility.");
}
async fn resync_custom_instance_user_pwd_if_needed(db: &Pool<Postgres>) {
use windmill_common::utils::get_custom_pg_instance_password;
use windmill_common::{get_database_url, PgDatabase};
let user_pwd = match get_custom_pg_instance_password(db).await {
Ok(pwd) => pwd,
Err(_) => {
// Setting doesn't exist yet (fresh install or pre-migration), skip check
return;
}
};
let mut pg_creds = match get_database_url().await {
Ok(url) => match PgDatabase::parse_uri(&url.as_str().await) {
Ok(creds) => creds,
Err(e) => {
tracing::warn!("Failed to parse database URL for custom_instance_user check: {e}");
return;
}
},
Err(e) => {
tracing::warn!("Failed to get database URL for custom_instance_user check: {e}");
return;
}
};
pg_creds.user = Some("custom_instance_user".to_string());
pg_creds.password = Some(user_pwd);
match pg_creds.connect().await {
Ok(_) => {
tracing::info!("custom_instance_user password is in sync");
}
Err(e) => {
tracing::warn!("custom_instance_user password is out of sync ({e}), refreshing...");
if let Err(e) = windmill_api_settings::refresh_custom_instance_user_pwd_inner(db).await
{
tracing::error!("Failed to refresh custom_instance_user password: {e}");
} else {
tracing::info!("Successfully refreshed custom_instance_user password");
}
}
}
}
async fn windmill_main() -> anyhow::Result<()> {
let (killpill_tx, mut killpill_rx) = KillpillSender::new(2);
let mut monitor_killpill_rx = killpill_tx.subscribe();
@@ -885,11 +839,6 @@ async fn windmill_main() -> anyhow::Result<()> {
// NOTE: Variable/resource cache initialization moved to API server in windmill-api
// Check if custom_instance_user password is in sync
if server_mode {
resync_custom_instance_user_pwd_if_needed(&db).await;
}
Connection::Sql(db)
};
@@ -972,6 +921,20 @@ Windmill Community Edition {GIT_VERSION}
default_base_internal_url.clone()
};
// BATCH_PULL_URL: explicit URL for native workers to pull jobs via HTTP.
// In standalone mode (server_mode=true), defaults to the local server.
let batch_pull_url: Option<String> = if is_native_mode_from_env() {
if let Ok(url) = std::env::var("BATCH_PULL_URL") {
Some(url)
} else if server_mode {
Some(default_base_internal_url.clone())
} else {
None
}
} else {
None
};
initial_load(
&conn,
killpill_tx.clone(),
@@ -1182,6 +1145,30 @@ Windmill Community Edition {GIT_VERSION}
)?;
let mut workers = vec![];
// For native workers, create a self-signed JWT for batch pulling via HTTP.
// Enabled when BATCH_PULL_URL is set (explicitly or auto-detected in standalone mode).
let batch_pull_client = if let Some(ref pull_url) = batch_pull_url {
match create_native_batch_pull_client(pull_url).await {
Ok(client) => {
tracing::info!(
"Native batch pull client created for HTTP pull at {}",
pull_url
);
USES_BATCH_HTTP_PULL
.store(true, std::sync::atomic::Ordering::Relaxed);
Some(client)
}
Err(e) => {
tracing::warn!(
"Failed to create native batch pull client, falling back to SQL pull: {e:#}"
);
None
}
}
} else {
None
};
for i in 0..num_workers {
let suffix = if i == 0 && first_suffix.is_some() {
first_suffix.as_ref().unwrap().clone()
@@ -1205,6 +1192,7 @@ Windmill Community Edition {GIT_VERSION}
WORKER_GROUP.as_str(),
&suffix,
),
batch_pull_client: batch_pull_client.clone(),
};
workers.push(worker_conn);
}
@@ -1666,9 +1654,6 @@ async fn process_notify_event(
}
TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await,
RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await,
AUDIT_LOG_RETENTION_DAYS_SETTING => {
reload_audit_log_retention_days_setting(conn).await
}
MONITOR_LOGS_ON_OBJECT_STORE_SETTING => {
reload_delete_logs_periodically_setting(conn).await
}
@@ -1821,6 +1806,7 @@ fn display_config(envs: &[&str]) {
pub struct WorkerConn {
conn: Connection,
worker_name: String,
batch_pull_client: Option<HttpClient>,
}
pub async fn run_workers(
@@ -1891,6 +1877,7 @@ pub async fn run_workers(
let wk_conf = &workers[i as usize - 1];
let conn1 = wk_conf.conn.clone();
let worker_name = wk_conf.worker_name.clone();
let batch_pull_client = wk_conf.batch_pull_client.clone();
WORKERS_NAMES.write().await.push(worker_name.clone());
let ip = ip.clone();
let rx = killpill_rxs.pop().unwrap();
@@ -1913,6 +1900,7 @@ pub async fn run_workers(
rx,
tx,
&base_internal_url,
batch_pull_client.as_ref(),
);
// #[cfg(tokio_unstable)]
@@ -1931,6 +1919,41 @@ pub async fn run_workers(
Ok(())
}
/// Create an HTTP client for native workers to pull jobs from the local server's batch buffer.
/// Self-signs a JWT with native_mode=true using the same JWT secret the server uses.
async fn create_native_batch_pull_client(base_internal_url: &str) -> anyhow::Result<HttpClient> {
use windmill_common::agent_workers::{build_agent_http_client, AGENT_JWT_PREFIX};
use windmill_common::jwt::encode_with_internal_secret;
#[derive(serde::Serialize)]
struct NativeAgentAuth {
worker_group: String,
tags: Vec<String>,
native_mode: Option<bool>,
exp: usize,
}
let worker_config = windmill_common::worker::WORKER_CONFIG.read().await;
let tags = worker_config.worker_tags.clone();
drop(worker_config);
// Token expires in 30 days — renewed on restart
let exp = (chrono::Utc::now() + chrono::Duration::days(30)).timestamp() as usize;
let claims = NativeAgentAuth {
worker_group: WORKER_GROUP.to_string(),
tags,
native_mode: Some(true),
exp,
};
let jwt = encode_with_internal_secret(claims).await?;
let token = format!("{}{}", AGENT_JWT_PREFIX, jwt);
let suffix = create_default_worker_suffix(&HOSTNAME);
Ok(build_agent_http_client(&suffix, &token, base_internal_url))
}
async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
if max_delay_secs == 0 {
max_delay_secs = 1;

View File

@@ -48,17 +48,16 @@ use windmill_common::{
error,
flow_status::{FlowStatus, FlowStatusModule},
global_settings::{
AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING,
INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING,
NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
UV_INDEX_STRATEGY_SETTING,
@@ -78,9 +77,9 @@ use windmill_common::{
DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR,
WORKER_CONFIG, WORKER_GROUP,
},
KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB,
DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY,
CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS,
};
@@ -324,15 +323,9 @@ pub async fn initial_load(
if server_mode {
reload_retention_period_setting(&conn).await;
reload_audit_log_retention_days_setting(&conn).await;
reload_request_size(&conn).await;
reload_saml_metadata_setting(&conn).await;
reload_scim_token_setting(&conn).await;
// Ensure audit partitions exist before any requests arrive
if let Some(db) = conn.as_sql() {
manage_audit_partitions(&db, audit_log_retention_days().await).await;
}
}
if worker_mode {
@@ -1034,10 +1027,12 @@ pub async fn delete_expired_items(db: &DB) -> () {
Err(e) => tracing::error!("Error deleting log file: {:?}", e),
}
let audit_retention_days = audit_log_retention_days().await;
let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24;
#[cfg(not(feature = "enterprise"))]
let audit_retention_secs = 1 * 60 * 60 * 24 * 14;
#[cfg(feature = "enterprise")]
let audit_retention_secs = 1 * 60 * 60 * 24 * 365;
// Clean up old (non-partitioned) audit table — will eventually be empty and dropped
if let Err(e) = sqlx::query_scalar!(
"DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval",
audit_retention_secs,
@@ -1045,7 +1040,7 @@ pub async fn delete_expired_items(db: &DB) -> () {
.fetch_all(db)
.await
{
tracing::error!("Error deleting audit log: {:?}", e);
tracing::error!("Error deleting audit log on CE: {:?}", e);
}
if let Err(e) = sqlx::query_scalar!(
@@ -1570,22 +1565,6 @@ pub async fn reload_retention_period_setting(conn: &Connection) {
tracing::error!("Error reloading retention period: {:?}", e)
}
}
pub async fn reload_audit_log_retention_days_setting(conn: &Connection) {
if let Err(e) = reload_setting(
conn,
AUDIT_LOG_RETENTION_DAYS_SETTING,
"AUDIT_LOG_RETENTION_DAYS",
0, // 0 means use default: 365 for EE, 14 for CE
AUDIT_LOG_RETENTION_DAYS.clone(),
|x| x,
)
.await
{
tracing::error!("Error reloading audit log retention days: {:?}", e)
}
}
pub async fn reload_delete_logs_periodically_setting(conn: &Connection) {
if let Err(e) = reload_setting(
conn,
@@ -2203,15 +2182,6 @@ pub async fn monitor_db(
}
};
// run every hour (120 iterations * 30s = 3600s)
let manage_audit_partitions_f = async {
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
if let Some(db) = conn.as_sql() {
manage_audit_partitions(&db, audit_log_retention_days().await).await;
}
}
};
join!(
expired_items_f,
zombie_jobs_f,
@@ -2234,7 +2204,6 @@ pub async fn monitor_db(
native_triggers_sync_f,
cleanup_notify_events_f,
check_expiring_tokens_f,
manage_audit_partitions_f,
);
}
@@ -2856,7 +2825,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
&windmill_queue::MiniCompletedJob::from(job),
memory_peak,
None,
error::Error::ExecutionErr(error_message.clone()),
error::Error::ExecutionErr(error_message),
matches!(error_kind, ErrorMessage::SameWorker), // unrecoverable if the job is a same worker zombie
Some(&same_worker_tx_never_used),
"",
@@ -2867,74 +2836,10 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, node_n
&mut windmill_common::bench::BenchmarkIter::new(),
)
.await;
// If handle_job_error failed (e.g. schedule push failure rolled back the tx),
// the job is still in the queue. Force-complete it to prevent infinite zombie loops.
if let Err(e) = force_complete_zombie_job(db, &job_id, &error_message).await {
tracing::error!("Failed to force-complete zombie job {}: {e:#}", job_id);
}
}
}
}
/// Force-complete a zombie job that handle_job_error failed to complete.
/// This is a minimal fallback: it inserts a failed completed job and deletes
/// from the queue in a single transaction, without schedule pushing or
/// error handler logic that could cause the completion to fail.
async fn force_complete_zombie_job(
db: &Pool<Postgres>,
job_id: &Uuid,
error_message: &str,
) -> error::Result<()> {
let still_queued = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1)",
job_id
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !still_queued {
return Ok(());
}
tracing::error!(
"Zombie job {job_id} was not completed by handle_job_error, force-completing it"
);
let error_value = serde_json::json!({
"message": error_message,
"name": "ExecutionErr",
});
let mut tx = db.begin().await?;
sqlx::query!(
"INSERT INTO v2_job_completed
(workspace_id, id, started_at, duration_ms, result, memory_peak, status, worker)
SELECT q.workspace_id, q.id, q.started_at,
COALESCE((EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000, 0)::bigint,
$2::jsonb, r.memory_peak, 'failure'::job_status, q.worker
FROM v2_job_queue q
LEFT JOIN v2_job_runtime r ON r.id = q.id
WHERE q.id = $1
ON CONFLICT (id) DO UPDATE SET status = 'failure', result = $2::jsonb",
job_id,
error_value,
)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job_id)
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("Force-completed zombie job {job_id}");
Ok(())
}
async fn cleanup_concurrency_counters_orphaned_keys(db: &DB) -> error::Result<()> {
let result = sqlx::query!(
"
@@ -3463,72 +3368,3 @@ RETURNING job_id
}
Ok(())
}
async fn audit_log_retention_days() -> i64 {
let v = *AUDIT_LOG_RETENTION_DAYS.read().await;
if v > 0 {
v
} else if cfg!(feature = "enterprise") {
365
} else {
14
}
}
async fn manage_audit_partitions(db: &DB, retention_days: i64) {
let today = chrono::Utc::now().date_naive();
// Create partitions for today and the next 3 days
for days_ahead in 0..=3i64 {
let date = today + chrono::Duration::days(days_ahead);
let next_date = date + chrono::Duration::days(1);
let partition_name = format!("audit_{}", date.format("%Y%m%d"));
let quoted_name = format!("\"{}\"", partition_name.replace('"', "\"\""));
let sql = format!(
"CREATE TABLE IF NOT EXISTS {quoted_name} PARTITION OF audit_partitioned \
FOR VALUES FROM ('{date}') TO ('{next_date}')"
);
if let Err(e) = sqlx::query(&sql).execute(db).await {
if !e.to_string().contains("already exists") {
tracing::error!("Error creating audit partition {partition_name}: {e:?}");
}
}
}
// Drop expired partitions
let cutoff_date = today - chrono::Duration::days(retention_days);
let partitions = sqlx::query_scalar::<_, String>(
"SELECT c.relname::text \
FROM pg_inherits i \
JOIN pg_class c ON c.oid = i.inhrelid \
WHERE i.inhparent = 'audit_partitioned'::regclass",
)
.fetch_all(db)
.await;
match partitions {
Ok(partitions) => {
for partition_name in partitions {
if let Some(date_str) = partition_name.strip_prefix("audit_") {
if let Ok(date) = chrono::NaiveDate::parse_from_str(date_str, "%Y%m%d") {
if date < cutoff_date {
let quoted_name =
format!("\"{}\"", partition_name.replace('"', "\"\""));
let sql = format!("DROP TABLE IF EXISTS {quoted_name}");
match sqlx::query(&sql).execute(db).await {
Ok(_) => tracing::info!(
"Dropped expired audit partition {partition_name}"
),
Err(e) => tracing::error!(
"Error dropping audit partition {partition_name}: {e:?}"
),
}
}
}
}
}
}
Err(e) => tracing::error!("Error listing audit partitions: {e:?}"),
}
}

View File

@@ -174,7 +174,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
windmill_migrations: name(text), created_at(ts)
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
FK: (workspace_id) -> workspace(id)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), native_mode(bool), uses_batch_http_pull(bool)
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char)
FK: (parent_workspace_id) -> workspace(id)
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)

View File

@@ -1,157 +0,0 @@
#!/usr/bin/env bash
# E2E test for WAC v2 workflow-as-code suspend/resume lifecycle
set -euo pipefail
BASE_URL="${BASE_URL:-http://localhost:8070}"
TOKEN="${WM_TOKEN:-}"
WORKSPACE="dev"
TIMEOUT=60 # seconds
# Get auth token if not set
if [ -z "$TOKEN" ]; then
TOKEN=$(curl -s "${BASE_URL}/api/auth/login" \
-H "Content-Type: application/json" \
-d '{"email":"admin@windmill.dev","password":"changeme"}' | tr -d '"')
fi
echo "=== WAC v2 E2E Test ==="
echo "Base URL: $BASE_URL"
echo ""
WAC_CODE='import { task, workflow } from "windmill-client@1.999.19";
const double = task(async (x: number): Promise<number> => {
console.log("[double] START at " + new Date().toISOString());
await new Promise(r => setTimeout(r, 2000));
console.log("[double] END at " + new Date().toISOString());
return x * 2;
});
const increment = task(async (x: number): Promise<number> => {
console.log("[increment] START at " + new Date().toISOString());
await new Promise(r => setTimeout(r, 2000));
console.log("[increment] END at " + new Date().toISOString());
return x + 1;
});
export const main = workflow(async (x: number = 10) => {
const [doubled, incremented] = await Promise.all([
double(x),
increment(x),
]);
const final_result = await double(incremented);
return { doubled, incremented, final_result };
});'
echo "Step 1: Submitting preview job..."
JOB_ID=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs/run/preview" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $TOKEN" \
-d "$(jq -n --arg code "$WAC_CODE" '{
content: $code,
language: "bun",
args: {"x": 10}
}')" | tr -d '"')
echo "Job ID: $JOB_ID"
if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then
echo "FAIL: Could not create job"
exit 1
fi
echo ""
echo "Step 2: Polling for completion (timeout: ${TIMEOUT}s)..."
START=$SECONDS
LAST_STATUS=""
while true; do
ELAPSED=$((SECONDS - START))
if [ $ELAPSED -gt $TIMEOUT ]; then
echo "FAIL: Timed out after ${TIMEOUT}s"
# Dump job state for debugging
echo ""
echo "=== Debug info ==="
echo "Parent job queue state:"
source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local
psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until, canceled_by FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null
echo "Child jobs:"
psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at" 2>/dev/null
echo "Completed children:"
psql "$DATABASE_URL" -c "SELECT id FROM completed_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null
echo "Checkpoint:"
psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null
echo "Total child count:"
psql "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null
exit 1
fi
# Check completed job
RESULT=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \
-H "Authorization: Bearer $TOKEN" 2>/dev/null)
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \
-H "Authorization: Bearer $TOKEN" 2>/dev/null)
if [ "$HTTP_CODE" = "200" ]; then
echo "Job completed in ${ELAPSED}s!"
echo ""
echo "Step 3: Checking result..."
echo "Result: $RESULT"
# Validate
DOUBLED=$(echo "$RESULT" | jq -r '.doubled // empty')
INCREMENTED=$(echo "$RESULT" | jq -r '.incremented // empty')
FINAL=$(echo "$RESULT" | jq -r '.final_result // empty')
PASS=true
if [ "$DOUBLED" != "20" ]; then
echo "FAIL: doubled = $DOUBLED, expected 20"
PASS=false
fi
if [ "$INCREMENTED" != "11" ]; then
echo "FAIL: incremented = $INCREMENTED, expected 11"
PASS=false
fi
if [ "$FINAL" != "22" ]; then
echo "FAIL: final_result = $FINAL, expected 22"
PASS=false
fi
if $PASS; then
echo "PASS: All values correct!"
# Check no excessive child jobs
source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null
CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null)
echo "Total child jobs created: $CHILD_COUNT (expected: 3)"
if [ "$CHILD_COUNT" -gt "3" ]; then
echo "WARN: More children than expected ($CHILD_COUNT > 3)"
fi
exit 0
else
exit 1
fi
fi
# Show progress
STATUS=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/get/${JOB_ID}" \
-H "Authorization: Bearer $TOKEN" 2>/dev/null | jq -r '.type // empty')
if [ "$STATUS" != "$LAST_STATUS" ]; then
echo " [${ELAPSED}s] Status: $STATUS"
LAST_STATUS="$STATUS"
fi
# Check for runaway child creation
source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null
CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null)
if [ "$CHILD_COUNT" -gt "10" ]; then
echo "FAIL: Runaway child creation detected! $CHILD_COUNT children (expected 3)"
echo ""
echo "=== Debug info ==="
psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null
psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at LIMIT 20" 2>/dev/null
psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null
exit 1
fi
sleep 1
done

View File

@@ -888,7 +888,7 @@ mod dedicated_worker_protocol {
if bundle_for_node {
// For Node.js: bundle to JavaScript first (like production's build_loader with LoaderMode::Node)
let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None, None);
let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None);
std::fs::write(dir.join("wrapper.mjs"), wrapper).unwrap();
// Use the exact same build_loader function as production
@@ -925,7 +925,7 @@ mod dedicated_worker_protocol {
output_path
} else {
// For Bun: use TypeScript directly (like production)
let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None, None);
let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None);
let wrapper_path = dir.join("wrapper.mjs");
std::fs::write(&wrapper_path, wrapper).unwrap();
wrapper_path

View File

@@ -30,89 +30,6 @@ export async function main(foo: string, bar: string) {
'',
'f/system/hello_with_preprocessor', 123413, 'deno', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'system',
'
export async function preprocessor(foo: string, bar: string) {
return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" };
}
export async function main(foo: string, bar: string) {
return "Hello " + foo + " " + bar;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
'',
'',
'f/system/hello_preprocessor_dedicated_bun', 123414, 'bun', E'{}\n//bun.lock\n{}');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'system',
'
def preprocessor(foo: str, bar: str):
return {"foo": foo + "_preprocessed", "bar": bar + "_preprocessed"}
def main(foo: str, bar: str):
return "Hello " + foo + " " + bar
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
'',
'',
'f/system/hello_preprocessor_dedicated_python', 123415, 'python3', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'system',
'
export async function preprocessor(foo: string, bar: string) {
return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" };
}
export async function main(foo: string, bar: string) {
return "Hello " + foo + " " + bar;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
'',
'',
'f/system/hello_preprocessor_dedicated_deno', 123416, 'deno', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'system',
'//native
export async function preprocessor(foo: string, bar: string) {
return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" };
}
export async function main(foo: string, bar: string) {
return "Hello " + foo + " " + bar;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
'',
'',
'f/system/hello_preprocessor_bunnative', 123417, 'bunnative', E'{}\n//bun.lock\n{}');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'system',
'//native
export async function preprocessor(foo: string, bar: string) {
return { foo: foo + "_preprocessed", bar: bar + "_preprocessed" };
}
export async function main(foo: string, bar: string) {
return "Hello " + foo + " " + bar;
}
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
'',
'',
'f/system/hello_preprocessor_dedicated_bunnative', 123418, 'bunnative', E'{}\n//bun.lock\n{}');
INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
'test-workspace',
'',

View File

@@ -1,16 +1,16 @@
mod job_payload {
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_common::flow_status::RestartedFrom;
use windmill_common::flows::{FlowModule, FlowModuleValue, FlowValue};
use windmill_common::jobs::JobPayload;
use windmill_common::scripts::{ScriptHash, ScriptLang};
use windmill_common::flow_status::RestartedFrom;
use windmill_test_utils::*;
use windmill_common::min_version::{
MIN_VERSION, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432,
MIN_VERSION_IS_AT_LEAST_1_440,
};
use windmill_test_utils::*;
pub async fn initialize_tracing() {
use std::sync::Once;
@@ -305,7 +305,7 @@ mod job_payload {
path: "f/system/hello_with_nodes_flow".to_string(),
dedicated_worker: None,
version: 1443253234253454,
debouncing_settings: Default::default(),
debouncing_settings: Default::default(),
})
.run_until_complete(&db, false, port)
.await
@@ -768,256 +768,4 @@ mod job_payload {
.await;
Ok(())
}
#[sqlx::test(fixtures("base", "hello"))]
async fn test_dedicated_worker_preprocessor_bun(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let test = || async {
let db = &db;
let job = RunJob::from(JobPayload::ScriptHash {
hash: ScriptHash(123414),
path: "f/system/hello_preprocessor_dedicated_bun".to_string(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: true,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
debouncing_settings:
windmill_common::runnable_settings::DebouncingSettings::default(),
})
.arg("foo", json!("hello"))
.arg("bar", json!("world"))
.run_until_complete_with(db, false, port, |id| async move {
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(false));
})
.await;
let args = job.args.as_ref().unwrap();
assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed")));
assert_eq!(args.get("bar"), Some(&json!("world_preprocessed")));
assert_eq!(
job.json_result().unwrap(),
json!("Hello hello_preprocessed world_preprocessed")
);
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(true));
};
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
Ok(())
}
#[sqlx::test(fixtures("base", "hello"))]
async fn test_dedicated_worker_preprocessor_python(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let test = || async {
let db = &db;
let job = RunJob::from(JobPayload::ScriptHash {
hash: ScriptHash(123415),
path: "f/system/hello_preprocessor_dedicated_python".to_string(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Python3,
priority: None,
apply_preprocessor: true,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
debouncing_settings:
windmill_common::runnable_settings::DebouncingSettings::default(),
})
.arg("foo", json!("hello"))
.arg("bar", json!("world"))
.run_until_complete_with(db, false, port, |id| async move {
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(false));
})
.await;
let args = job.args.as_ref().unwrap();
assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed")));
assert_eq!(args.get("bar"), Some(&json!("world_preprocessed")));
assert_eq!(
job.json_result().unwrap(),
json!("Hello hello_preprocessed world_preprocessed")
);
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(true));
};
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
Ok(())
}
#[sqlx::test(fixtures("base", "hello"))]
async fn test_dedicated_worker_preprocessor_deno(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let test = || async {
let db = &db;
let job = RunJob::from(JobPayload::ScriptHash {
hash: ScriptHash(123416),
path: "f/system/hello_preprocessor_dedicated_deno".to_string(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
priority: None,
apply_preprocessor: true,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
debouncing_settings:
windmill_common::runnable_settings::DebouncingSettings::default(),
})
.arg("foo", json!("hello"))
.arg("bar", json!("world"))
.run_until_complete_with(db, false, port, |id| async move {
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(false));
})
.await;
let args = job.args.as_ref().unwrap();
assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed")));
assert_eq!(args.get("bar"), Some(&json!("world_preprocessed")));
assert_eq!(
job.json_result().unwrap(),
json!("Hello hello_preprocessed world_preprocessed")
);
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(true));
};
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
Ok(())
}
#[sqlx::test(fixtures("base", "hello"))]
async fn test_bunnative_preprocessor(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let test = || async {
let db = &db;
let job = RunJob::from(JobPayload::ScriptHash {
hash: ScriptHash(123417),
path: "f/system/hello_preprocessor_bunnative".to_string(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Bunnative,
priority: None,
apply_preprocessor: true,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
debouncing_settings:
windmill_common::runnable_settings::DebouncingSettings::default(),
})
.arg("foo", json!("hello"))
.arg("bar", json!("world"))
.run_until_complete_with(db, false, port, |id| async move {
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(false));
})
.await;
let args = job.args.as_ref().unwrap();
assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed")));
assert_eq!(args.get("bar"), Some(&json!("world_preprocessed")));
assert_eq!(
job.json_result().unwrap(),
json!("Hello hello_preprocessed world_preprocessed")
);
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(true));
};
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
Ok(())
}
#[sqlx::test(fixtures("base", "hello"))]
async fn test_dedicated_worker_preprocessor_bunnative(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let test = || async {
let db = &db;
let job = RunJob::from(JobPayload::ScriptHash {
hash: ScriptHash(123418),
path: "f/system/hello_preprocessor_dedicated_bunnative".to_string(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Bunnative,
priority: None,
apply_preprocessor: true,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
debouncing_settings:
windmill_common::runnable_settings::DebouncingSettings::default(),
})
.arg("foo", json!("hello"))
.arg("bar", json!("world"))
.run_until_complete_with(db, false, port, |id| async move {
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(false));
})
.await;
let args = job.args.as_ref().unwrap();
assert_eq!(args.get("foo"), Some(&json!("hello_preprocessed")));
assert_eq!(args.get("bar"), Some(&json!("world_preprocessed")));
assert_eq!(
job.json_result().unwrap(),
json!("Hello hello_preprocessed world_preprocessed")
);
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", job.id)
.fetch_one(db)
.await
.unwrap();
assert_eq!(job.preprocessed, Some(true));
};
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
Ok(())
}
}

View File

@@ -80,13 +80,8 @@ mod prewarmed_isolate_tests {
let mut results = Vec::new();
for job_args in &jobs {
let mut isolate = PrewarmedIsolate::spawn(
"".to_string(),
js.clone(),
ann.clone(),
arg_names.clone(),
None,
);
let mut isolate =
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
isolate.wait_ready().await.expect("isolate failed to warm");
let args = serde_json::to_string(job_args).unwrap();
@@ -185,13 +180,8 @@ export function main(n: number): number {
let ann = default_annotation();
// Pre-warm first isolate
let mut warm = PrewarmedIsolate::spawn(
"".to_string(),
js.clone(),
ann.clone(),
arg_names.clone(),
None,
);
let mut warm =
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
warm.wait_ready()
.await
.expect("first isolate failed to warm");
@@ -203,13 +193,8 @@ export function main(n: number): number {
let executing = warm.start_execution(args);
// Pipeline: start pre-warming next isolate while current one runs
warm = PrewarmedIsolate::spawn(
"".to_string(),
js.clone(),
ann.clone(),
arg_names.clone(),
None,
);
warm =
PrewarmedIsolate::spawn("".to_string(), js.clone(), ann.clone(), arg_names.clone());
let prewarmed_result = executing.wait().await.expect("isolate execution failed");
match prewarmed_result.result {

View File

@@ -241,6 +241,7 @@ fn spawn_workers(
rx,
tx2,
&base_internal_url,
None,
)
.await;
};

View File

@@ -3548,170 +3548,3 @@ async fn test_flow_substep_tag_availability_check(db: Pool<Postgres>) -> anyhow:
Ok(())
}
#[cfg(all(feature = "quickjs", feature = "python"))]
#[sqlx::test(fixtures("base"))]
async fn test_stop_after_all_iters_if_bad_expr_parallel_branchall(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let port = 123;
let flow: FlowValue = serde_json::from_value(serde_json::json!({
"modules": [
{
"id": "a",
"value": {
"branches": [
{"modules": [{
"id": "b",
"value": {
"input_transforms": { "n": { "type": "javascript", "expr": "flow_input.n" } },
"type": "rawscript",
"language": "python3",
"content": "def main(n): return n",
},
}]}
],
"type": "branchall",
"parallel": true,
},
"stop_after_all_iters_if": {
"expr": "invalid!!!syntax",
"skip_if_stopped": false,
},
},
],
}))
.unwrap();
let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
let cjob = RunJob::from(job)
.arg("n", json!(42))
.run_until_complete(&db, false, port)
.await;
assert!(
!cjob.success,
"flow should fail when stop_after_all_iters_if has bad expression"
);
let result = cjob.json_result().unwrap();
let error_msg = result["error"]["message"].as_str().unwrap_or("");
assert!(
error_msg.contains("stop_after_all_iters_if"),
"error should mention stop_after_all_iters_if, got: {error_msg}"
);
Ok(())
}
#[cfg(all(feature = "quickjs", feature = "python"))]
#[sqlx::test(fixtures("base"))]
async fn test_stop_after_all_iters_if_bad_expr_parallel_forloop(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let port = 123;
let flow: FlowValue = serde_json::from_value(serde_json::json!({
"modules": [
{
"id": "a",
"value": {
"type": "forloopflow",
"iterator": { "type": "javascript", "expr": "result.items" },
"skip_failures": false,
"parallel": true,
"modules": [{
"value": {
"input_transforms": {
"n": { "type": "javascript", "expr": "flow_input.iter.value" },
},
"type": "rawscript",
"language": "python3",
"content": "def main(n): return n",
},
}],
},
"stop_after_all_iters_if": {
"expr": "invalid!!!syntax",
"skip_if_stopped": false,
},
},
],
}))
.unwrap();
let job = JobPayload::RawFlow { value: flow, path: None, restarted_from: None };
let cjob = RunJob::from(job)
.arg("items", json!([1, 2, 3]))
.run_until_complete(&db, false, port)
.await;
assert!(
!cjob.success,
"flow should fail when stop_after_all_iters_if has bad expression"
);
let result = cjob.json_result().unwrap();
let error_msg = result["error"]["message"].as_str().unwrap_or("");
assert!(
error_msg.contains("stop_after_all_iters_if"),
"error should mention stop_after_all_iters_if, got: {error_msg}"
);
Ok(())
}
#[cfg(all(feature = "quickjs", feature = "python"))]
#[sqlx::test(fixtures("base"))]
async fn test_results_length_in_input_transform(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// Step a returns a list, step b accesses results.a.length via input transform.
// This tests that the handle_full_regex fast path falls through to QuickJS
// when the SQL JSON path operator can't resolve JS properties like .length.
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "a",
"value": {
"type": "rawscript",
"language": "python3",
"content": "def main(): return [10, 20, 30]",
},
},
{
"id": "b",
"value": {
"input_transforms": {
"v": { "type": "javascript", "expr": "results.a.length" },
},
"type": "rawscript",
"language": "python3",
"content": "def main(v): return v",
},
},
],
}))
.unwrap();
let result =
RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
.run_until_complete(&db, false, port)
.await
.json_result()
.unwrap();
assert_eq!(
result,
json!(3),
"results.a.length should resolve to 3, not null"
);
Ok(())
}

View File

@@ -19,7 +19,10 @@ use windmill_common::DB;
use axum::Router;
#[cfg(not(feature = "private"))]
pub fn global_service(_job_completed_tx: windmill_worker::JobCompletedSender) -> Router {
pub fn global_service(
_job_completed_tx: windmill_worker::JobCompletedSender,
_batch_buffer: Option<()>,
) -> Router {
Router::new()
}
@@ -31,6 +34,7 @@ pub fn workspaced_service(
Router,
Vec<tokio::task::JoinHandle<()>>,
Option<windmill_worker::JobCompletedSender>,
Option<()>,
) {
use windmill_common::worker::Connection;
use windmill_worker::JobCompletedSender;
@@ -40,7 +44,7 @@ pub fn workspaced_service(
let router = Router::new();
(router, vec![], Some(job_completed_tx))
(router, vec![], Some(job_completed_tx), None)
}
#[cfg(not(feature = "private"))]

View File

@@ -1,452 +0,0 @@
/*!
* Integration tests for workspace dependencies git sync.
*
* These tests verify that creating, archiving, and deleting workspace dependencies
* triggers deployment callback jobs with the correct arguments for git sync.
*
* Run with enterprise features:
* ```bash
* cargo test --test workspace_dependencies_git_sync --features enterprise,private
* ```
*/
use serde_json::json;
use sqlx::{Pool, Postgres};
use std::time::Duration;
use windmill_test_utils::*;
/// Row shape for querying deployment callback jobs from v2_job_queue
#[derive(Debug)]
#[allow(dead_code)]
struct DeploymentCallbackJob {
id: uuid::Uuid,
runnable_path: Option<String>,
args: Option<serde_json::Value>,
kind: String,
}
/// Poll for deployment callback jobs in the queue for a given script path
async fn get_deployment_callback_jobs(
db: &Pool<Postgres>,
script_path: &str,
timeout: Duration,
) -> anyhow::Result<Vec<DeploymentCallbackJob>> {
let deadline = tokio::time::Instant::now() + timeout;
loop {
let rows = sqlx::query_as!(
DeploymentCallbackJob,
r#"
SELECT j.id, j.runnable_path, j.args, j.kind::text AS "kind!"
FROM v2_job j
JOIN v2_job_queue q ON j.id = q.id
WHERE j.runnable_path = $1
AND j.kind = 'deploymentcallback'
ORDER BY j.created_at DESC
"#,
script_path,
)
.fetch_all(db)
.await?;
if !rows.is_empty() {
return Ok(rows);
}
if tokio::time::Instant::now() >= deadline {
// Return empty if timeout - caller will handle assertion
return Ok(vec![]);
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
/// Configure git sync for the test workspace with workspace dependencies enabled
async fn setup_git_sync_config(db: &Pool<Postgres>, sync_script_path: &str) -> anyhow::Result<()> {
let git_sync_config = json!({
"include_type": ["workspacedependencies"],
"include_path": ["**"],
"repositories": [{
"script_path": sync_script_path,
"git_repo_resource_path": "$res:u/test-user/test_git_repo",
"use_individual_branch": false,
"group_by_folder": false
}]
});
sqlx::query!(
"UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
git_sync_config,
"test-workspace"
)
.execute(db)
.await?;
Ok(())
}
/// Create a git repository resource for testing
async fn create_git_repo_resource(db: &Pool<Postgres>) -> anyhow::Result<()> {
sqlx::query(
r#"
INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by)
VALUES ('test-workspace', 'u/test-user/test_git_repo', $1::jsonb, 'git_repository', '{}'::jsonb, 'test-user')
ON CONFLICT (workspace_id, path) DO NOTHING
"#,
)
.bind(json!({
"url": "https://github.com/test/test.git",
"branch": "main",
"token": "test-token"
}))
.execute(db)
.await?;
Ok(())
}
/// Create a dummy sync script for testing (with version >= 28103 for debouncing support)
async fn create_sync_script(db: &Pool<Postgres>, path: &str) -> anyhow::Result<i64> {
let hash: i64 = rand::random::<i64>().unsigned_abs() as i64;
sqlx::query(
r#"
INSERT INTO script (workspace_id, hash, path, summary, description, content,
created_by, language, kind, lock)
VALUES ('test-workspace', $1, $2, 'sync script', '',
'export function main(items: any[]) { return { synced: items.length }; }',
'test-user', 'bun', 'script', '')
"#,
)
.bind(hash)
.bind(path)
.execute(db)
.await?;
Ok(hash)
}
/// Create a folder for the versioned script path
async fn create_folder(db: &Pool<Postgres>, name: &str) -> anyhow::Result<()> {
sqlx::query(
r#"
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by)
VALUES ('test-workspace', $1, $1, ARRAY['u/test-user'], '{}'::jsonb, 'test-user')
ON CONFLICT (workspace_id, name) DO NOTHING
"#,
)
.bind(name)
.execute(db)
.await?;
Ok(())
}
// ============================================================================
// Tests
// ============================================================================
/// Test that creating a workspace dependency triggers a git sync deployment callback
/// with the correct path_type and path arguments.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_create_workspace_dependencies_triggers_git_sync(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
// Setup: Create folder, git repo resource, sync script, and configure git sync
create_folder(&db, "28103").await?;
create_git_repo_resource(&db).await?;
let sync_script_path = "f/28103/test_sync_script";
create_sync_script(&db, sync_script_path).await?;
setup_git_sync_config(&db, sync_script_path).await?;
// Start API server
let (client, _port, _server) = init_client(db.clone()).await;
// Create workspace dependency via API
let response = client
.client()
.post(format!(
"{}/w/test-workspace/workspace_dependencies/create",
client.baseurl()
))
.json(&json!({
"workspace_id": "test-workspace",
"language": "python3",
"name": "test-deps",
"content": "requests==2.28.0\nnumpy==1.24.0"
}))
.send()
.await?;
assert!(
response.status().is_success(),
"Failed to create workspace dependency: {:?}",
response.text().await
);
// Wait for deployment callback job to be created
tokio::time::sleep(Duration::from_millis(500)).await;
// Query for deployment callback jobs
let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?;
assert!(
!jobs.is_empty(),
"Expected at least one deployment callback job to be created"
);
// Verify the job arguments
let job = &jobs[0];
let args = job.args.as_ref().expect("Job should have args");
// Check that path_type is "workspace_dependencies" (or check items array)
// The exact structure depends on whether debouncing is enabled
if let Some(items) = args.get("items") {
// Debounced format: items is an array
let items_arr = items.as_array().expect("items should be an array");
assert!(!items_arr.is_empty(), "items array should not be empty");
let item = &items_arr[0];
assert_eq!(
item.get("path_type").and_then(|v| v.as_str()),
Some("workspace_dependencies"),
"path_type should be 'workspace_dependencies'"
);
// Path should be "dependencies/test-deps.requirements.in" or similar
let path = item.get("path").and_then(|v| v.as_str()).unwrap_or("");
assert!(
path.contains("dependencies") || path.contains("requirements"),
"path should contain dependencies or requirements: got {}",
path
);
} else if let Some(path_type) = args.get("path_type") {
// Non-debounced format: path_type is a direct field
assert_eq!(
path_type.as_str(),
Some("workspace_dependencies"),
"path_type should be 'workspace_dependencies'"
);
} else {
panic!(
"Job args should contain either 'items' array or 'path_type' field: {:?}",
args
);
}
Ok(())
}
/// Test that archiving a workspace dependency triggers a git sync deployment callback
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_archive_workspace_dependencies_triggers_git_sync(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
// Setup
create_folder(&db, "28103").await?;
create_git_repo_resource(&db).await?;
let sync_script_path = "f/28103/test_sync_script_archive";
create_sync_script(&db, sync_script_path).await?;
setup_git_sync_config(&db, sync_script_path).await?;
let (client, _port, _server) = init_client(db.clone()).await;
// First create a workspace dependency
let create_response = client
.client()
.post(format!(
"{}/w/test-workspace/workspace_dependencies/create",
client.baseurl()
))
.json(&json!({
"workspace_id": "test-workspace",
"language": "python3",
"name": "archive-test-deps",
"content": "flask==2.0.0"
}))
.send()
.await?;
assert!(create_response.status().is_success());
// Wait a bit for the create job to be processed
tokio::time::sleep(Duration::from_millis(300)).await;
// Now archive it
let archive_response = client
.client()
.post(format!(
"{}/w/test-workspace/workspace_dependencies/archive/python3?name=archive-test-deps",
client.baseurl()
))
.send()
.await?;
assert!(
archive_response.status().is_success(),
"Failed to archive workspace dependency: {:?}",
archive_response.text().await
);
// Wait for deployment callback job
tokio::time::sleep(Duration::from_millis(500)).await;
// Verify at least one deployment callback job exists
// (create test already validates create triggers git sync; this test validates archive does too)
let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?;
assert!(
!jobs.is_empty(),
"Expected at least one deployment callback job after archive"
);
Ok(())
}
/// Test that workspace dependencies are NOT synced when workspacedependencies type is excluded
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_workspace_dependencies_respects_include_type_filter(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
// Setup with git sync that EXCLUDES workspacedependencies
create_folder(&db, "28103").await?;
create_git_repo_resource(&db).await?;
let sync_script_path = "f/28103/test_sync_script_filter";
create_sync_script(&db, sync_script_path).await?;
// Configure git sync to only include scripts (not workspace dependencies)
let git_sync_config = json!({
"include_type": ["script"], // Note: workspacedependencies is NOT included
"include_path": ["**"],
"repositories": [{
"script_path": sync_script_path,
"git_repo_resource_path": "$res:u/test-user/test_git_repo",
"use_individual_branch": false,
"group_by_folder": false
}]
});
sqlx::query!(
"UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
git_sync_config,
"test-workspace"
)
.execute(&db)
.await?;
let (client, _port, _server) = init_client(db.clone()).await;
// Create workspace dependency
let response = client
.client()
.post(format!(
"{}/w/test-workspace/workspace_dependencies/create",
client.baseurl()
))
.json(&json!({
"workspace_id": "test-workspace",
"language": "python3",
"name": "filtered-deps",
"content": "django==4.0.0"
}))
.send()
.await?;
assert!(response.status().is_success());
// Wait a bit
tokio::time::sleep(Duration::from_secs(1)).await;
// Should NOT have any deployment callback jobs because workspacedependencies is filtered out
let jobs =
get_deployment_callback_jobs(&db, sync_script_path, Duration::from_millis(500)).await?;
assert!(
jobs.is_empty(),
"Expected NO deployment callback jobs when workspacedependencies is not in include_type, got {}",
jobs.len()
);
Ok(())
}
/// Test that the commit message is correctly generated for workspace dependencies
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_workspace_dependencies_commit_message(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
// Setup
create_folder(&db, "28103").await?;
create_git_repo_resource(&db).await?;
let sync_script_path = "f/28103/test_sync_script_msg";
create_sync_script(&db, sync_script_path).await?;
setup_git_sync_config(&db, sync_script_path).await?;
let (client, _port, _server) = init_client(db.clone()).await;
// Create workspace dependency
let response = client
.client()
.post(format!(
"{}/w/test-workspace/workspace_dependencies/create",
client.baseurl()
))
.json(&json!({
"workspace_id": "test-workspace",
"language": "bun",
"name": null, // unnamed/default dependency
"content": "lodash: ^4.17.21"
}))
.send()
.await?;
assert!(response.status().is_success());
tokio::time::sleep(Duration::from_millis(500)).await;
let jobs = get_deployment_callback_jobs(&db, sync_script_path, Duration::from_secs(5)).await?;
assert!(!jobs.is_empty());
let job = &jobs[0];
let args = job.args.as_ref().expect("Job should have args");
// Check commit message format
if let Some(items) = args.get("items") {
let items_arr = items.as_array().expect("items should be an array");
if !items_arr.is_empty() {
let commit_msg = items_arr[0]
.get("commit_msg")
.and_then(|v| v.as_str())
.unwrap_or("");
assert!(
commit_msg.contains("[WM]"),
"Commit message should contain '[WM]' prefix: {}",
commit_msg
);
assert!(
commit_msg.to_lowercase().contains("workspace")
|| commit_msg.to_lowercase().contains("dependency")
|| commit_msg.to_lowercase().contains("deployed"),
"Commit message should mention workspace dependency or deployed: {}",
commit_msg
);
}
} else if let Some(commit_msg) = args.get("commit_msg").and_then(|v| v.as_str()) {
assert!(
commit_msg.contains("[WM]"),
"Commit message should contain '[WM]' prefix: {}",
commit_msg
);
}
Ok(())
}

View File

@@ -79,7 +79,7 @@ async fn get_job_metrics(
>,
) -> error::JsonResult<JobStatsResponse> {
let records = sqlx::query_as::<_, JobStatsRecord>(
"SELECT workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float, timeseries_start, offsets_cs FROM job_stats WHERE workspace_id = $1 AND job_id = $2",
"SELECT * FROM job_stats where workspace_id = $1 and job_id = $2",
)
.bind(w_id)
.bind(job_id)
@@ -91,7 +91,7 @@ async fn get_job_metrics(
let mut timeseries_metrics: Vec<TimeseriesMetric> = vec![];
for record in records {
let metric_id = record.metric_id.clone();
let metric_id = record.metric_id;
match record.metric_kind {
MetricKind::ScalarInt => {
let value = record.scalar_int.unwrap_or_default() as f64;
@@ -102,43 +102,47 @@ async fn get_job_metrics(
scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value });
}
MetricKind::TimeseriesInt => {
let timestamps = resolve_timestamps(&record);
let timeseries_int = record.timeseries_int.unwrap_or_default();
if timestamps.len() != timeseries_int.len() {
tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_int.len());
if record.timestamps.clone().unwrap_or_default().len()
!= record.timeseries_int.clone().unwrap_or_default().len()
{
tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int)
}
let (timestamps, timeseries_int) = timeseries_sample(
from_timestamp,
to_timestamp,
timeseries_max_datapoints,
timestamps,
timeseries_int,
record.timestamps.unwrap_or_default(),
record.timeseries_int.unwrap_or_default(),
);
let values: Vec<DataPoint> = timestamps
.iter()
.zip(timeseries_int.iter())
.map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 })
.collect();
let mut values: Vec<DataPoint> = vec![];
for (idx, value) in timeseries_int.iter().enumerate() {
values.push(DataPoint {
timestamp: timestamps[idx],
value: value.to_owned() as f64,
});
}
timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
}
MetricKind::TimeseriesFloat => {
let timestamps = resolve_timestamps(&record);
let timeseries_float = record.timeseries_float.unwrap_or_default();
if timestamps.len() != timeseries_float.len() {
tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_float.len());
if record.timestamps.clone().unwrap_or_default().len()
!= record.timeseries_int.clone().unwrap_or_default().len()
{
tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float)
}
let (timestamps, timeseries_float) = timeseries_sample(
from_timestamp,
to_timestamp,
timeseries_max_datapoints,
timestamps,
timeseries_float,
record.timestamps.unwrap_or_default(),
record.timeseries_float.unwrap_or_default(),
);
let values: Vec<DataPoint> = timestamps
.iter()
.zip(timeseries_float.iter())
.map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 })
.collect();
let mut values: Vec<DataPoint> = vec![];
for (idx, value) in timeseries_float.iter().enumerate() {
values.push(DataPoint {
timestamp: timestamps[idx],
value: value.to_owned() as f64,
});
}
timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values });
}
};
@@ -148,21 +152,6 @@ async fn get_job_metrics(
let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics };
Ok(Json(response))
}
/// Reconstruct full timestamps from `timeseries_start` + `offsets_cs` if available,
/// otherwise fall back to legacy `timestamps` column.
fn resolve_timestamps(record: &JobStatsRecord) -> Vec<chrono::DateTime<chrono::Utc>> {
if let (Some(start), Some(offsets)) = (record.timeseries_start, &record.offsets_cs) {
if !offsets.is_empty() {
return offsets
.iter()
.map(|&cs| start + chrono::Duration::milliseconds(cs as i64 * 10))
.collect();
}
}
// Legacy fallback: use the full timestamps column
record.timestamps.clone().unwrap_or_default()
}
#[derive(Deserialize)]
struct JobProgressSetRequest {
percent: i32,

View File

@@ -316,11 +316,7 @@ pub async fn set_global_setting_internal(
)
.execute(db)
.await?;
tracing::info!(
"Set global setting {} to {}",
key,
instance_config::format_setting_value(&key, &v)
);
tracing::info!("Set global setting {} to {}", key, v);
}
};
@@ -801,7 +797,11 @@ async fn list_custom_instance_pg_databases(
return Ok(Json(result));
}
pub async fn refresh_custom_instance_user_pwd_inner(db: &DB) -> Result<()> {
async fn refresh_custom_instance_user_pwd(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<()> {
require_super_admin(&db, &authed.email).await?;
// 20251208123907_safety_custom_instance_db_user_pwd.up
let query = r#"
DO $$
@@ -809,7 +809,7 @@ pub async fn refresh_custom_instance_user_pwd_inner(db: &DB) -> Result<()> {
pwd text;
BEGIN
SELECT gen_random_uuid()::text INTO pwd;
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN
EXECUTE format('ALTER USER custom_instance_user WITH PASSWORD %L', pwd);
RAISE NOTICE 'Updated password for existing user custom_instance_user';
@@ -834,16 +834,7 @@ pub async fn refresh_custom_instance_user_pwd_inner(db: &DB) -> Result<()> {
END
$$;
"#;
sqlx::query(query).execute(db).await?;
Ok(())
}
async fn refresh_custom_instance_user_pwd(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<()> {
require_super_admin(&db, &authed.email).await?;
refresh_custom_instance_user_pwd_inner(&db).await?;
sqlx::query(query).execute(&db).await?;
Ok(Json(()))
}

View File

@@ -393,7 +393,7 @@ async fn list_users_as_super_admin(
let rows = if active_only.is_some_and(|x| x) {
sqlx::query_as!(
GlobalUserInfo,
"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
"WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)
SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user
FROM password

View File

@@ -10,7 +10,7 @@ path = "src/lib.rs"
[features]
default = []
private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"]
private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"]
stripe = []
run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"]
@@ -174,7 +174,6 @@ aws-sdk-bedrock = { workspace = true, optional = true }
aws-sdk-bedrockruntime = { workspace = true, optional = true }
aws-smithy-types = { workspace = true, optional = true }
async-trait.workspace = true
eventsource-stream.workspace = true
windmill-jseval.workspace = true
tar.workspace = true
flate2.workspace = true

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.654.0
version: 1.651.1
title: Windmill API
contact:
@@ -12004,19 +12004,6 @@ paths:
schema:
type: string
/w/{workspace}/kafka_triggers/reset_offsets/{path}:
post:
summary: reset kafka trigger offsets to earliest
operationId: resetKafkaOffsets
tags:
- kafka_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: kafka trigger offsets reset successfully
/w/{workspace}/nats_triggers/create:
post:
summary: create nats trigger
@@ -16889,9 +16876,6 @@ paths:
lost_lock_ownership:
description: Is the current indexer service being replaced
type: boolean
max_index_time_window_secs:
description: Maximum time window in seconds for indexing
type: number
/srch/index/search/service_logs:
get:
@@ -18472,7 +18456,6 @@ components:
- trigger
- settings
- key
- workspacedependencies
AIProviderModel:
type: object
@@ -18820,6 +18803,7 @@ components:
required:
- path
- summary
- description
- content
- language
@@ -22074,7 +22058,7 @@ components:
description: Path to the Kafka resource containing connection configuration
group_id:
type: string
description: Kafka consumer group ID for this trigger
description: Kafka consumer group ID for this trigger
topics:
type: array
items:
@@ -22091,13 +22075,6 @@ components:
required:
- key
- value
auto_offset_reset:
type: string
enum:
- latest
- earliest
default: latest
description: "Initial offset behavior when consumer group has no committed offset. 'latest' starts from new messages only, 'earliest' starts from the beginning."
server_id:
type: string
description: ID of the server currently handling this trigger (internal)
@@ -22158,13 +22135,6 @@ components:
required:
- key
- value
auto_offset_reset:
type: string
enum:
- latest
- earliest
default: latest
description: "Initial offset behavior when consumer group has no committed offset."
mode:
$ref: "#/components/schemas/TriggerMode"
error_handler_path:
@@ -22217,13 +22187,6 @@ components:
required:
- key
- value
auto_offset_reset:
type: string
enum:
- latest
- earliest
default: latest
description: "Initial offset behavior when consumer group has no committed offset."
path:
type: string
description: The unique path identifier for this trigger

View File

@@ -17,7 +17,7 @@ use std::collections::HashMap;
use std::time::Duration;
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::ai_providers::{
empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel,
empty_string_as_none, AIProvider, ProviderConfig, ProviderModel,
};
use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::utils::configure_client;
@@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours
const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour
const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10;
const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90;
pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15;
const KEEPALIVE_INTERVAL_SECS: u64 = 15;
lazy_static::lazy_static! {
/// AI request timeout in seconds.
@@ -87,7 +87,7 @@ lazy_static::lazy_static! {
}
};
pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
.timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS))
.pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST)
.pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS)))
@@ -135,6 +135,15 @@ struct AIOAuthResource {
user: Option<String>,
}
/// Platform for Anthropic API
#[derive(Deserialize, Debug, Clone, Default, PartialEq)]
#[serde(rename_all = "snake_case")]
enum AnthropicPlatform {
#[default]
Standard,
GoogleVertexAi,
}
#[derive(Deserialize, Debug)]
struct AIStandardResource {
#[serde(alias = "baseUrl", default, deserialize_with = "empty_string_as_none")]
@@ -163,9 +172,9 @@ struct AIStandardResource {
deserialize_with = "empty_string_as_none"
)]
aws_session_token: Option<String>,
/// Platform (standard or google_vertex_ai)
/// Platform for Anthropic API (standard or google_vertex_ai)
#[serde(default)]
platform: AIPlatform,
platform: AnthropicPlatform,
/// Enable 1M context window for Anthropic
#[serde(alias = "enable_1M_context", default)]
enable_1m_context: bool,
@@ -198,7 +207,7 @@ struct AIRequestConfig {
pub aws_secret_access_key: Option<String>,
#[allow(dead_code)]
pub aws_session_token: Option<String>,
pub platform: AIPlatform,
pub platform: AnthropicPlatform,
pub enable_1m_context: bool,
}
@@ -292,7 +301,7 @@ impl AIRequestConfig {
None,
None,
None,
AIPlatform::Standard,
AnthropicPlatform::Standard,
false,
)
}
@@ -365,11 +374,16 @@ impl AIRequestConfig {
let is_azure = provider.is_azure_openai(base_url);
let is_anthropic = matches!(provider, AIProvider::Anthropic);
let is_anthropic_vertex =
is_anthropic && self.platform == AIPlatform::GoogleVertexAi;
is_anthropic && self.platform == AnthropicPlatform::GoogleVertexAi;
let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some();
let is_google_ai = matches!(provider, AIProvider::GoogleAI);
let base_url = base_url.to_string();
// GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent
let base_url = if is_google_ai {
format!("{}/openai", base_url)
} else {
base_url.to_string()
};
let base_url = base_url.as_str();
// Build URL based on provider
@@ -414,11 +428,6 @@ impl AIRequestConfig {
if let Some(api_key) = self.api_key {
if is_azure {
request = request.header("api-key", api_key.clone())
} else if is_google_ai {
// Note: GoogleAI requests are intercepted earlier (see the GoogleAI
// handler block above) and never reach this code path. This branch
// is kept as a safety net for the standard Gemini API auth format.
request = request.header("x-goog-api-key", api_key.clone())
} else {
request = request.header("authorization", format!("Bearer {}", api_key.clone()))
}
@@ -602,7 +611,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool {
.unwrap_or(false)
}
pub(crate) fn inject_keepalives<S>(
fn inject_keepalives<S>(
upstream: S,
interval: Duration,
) -> impl futures::Stream<Item = std::result::Result<Bytes, reqwest::Error>>
@@ -821,39 +830,6 @@ async fn proxy(
ai_path = chat_path;
}
// Handle GoogleAI (Gemini) using the native Gemini API
if matches!(provider, AIProvider::GoogleAI) {
let api_key = request_config.api_key.as_deref().unwrap_or("");
let base_url = request_config.base_url.trim_end_matches('/');
let is_vertex = request_config.platform == AIPlatform::GoogleVertexAi;
let mut tx = db.begin().await?;
audit_log(
&mut *tx,
&authed,
"ai.request",
ActionKind::Execute,
&w_id,
Some(&authed.email),
Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()),
)
.await?;
tx.commit().await?;
return match ai_path.as_str() {
"chat/completions" => {
crate::google::handle_google_ai_chat(&body, api_key, base_url, is_vertex).await
}
"models" => {
crate::google::handle_google_ai_models(api_key, base_url, is_vertex).await
}
_ => Err(Error::BadRequest(format!(
"Unsupported Google AI path: {}",
ai_path
))),
};
}
// Handle Bedrock-specific logic when the feature is enabled
#[cfg(feature = "bedrock")]
{

View File

@@ -284,9 +284,6 @@ pub async fn migrate(
20260207000004,
];
for m in migrator.migrations.iter() {
if m.migration_type.is_down_migration() {
continue;
}
if potentially_stale.contains(&m.version) {
if let Err(err) =
sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2")

View File

@@ -1,354 +0,0 @@
//! Google AI (Gemini API) handler for the AI chat proxy.
//!
//! Handles POST `chat/completions` requests using the native Gemini API,
//! converting from/to OpenAI format so the existing frontend parsers continue to work.
//!
//! Supports both standard Google AI (generativelanguage.googleapis.com) and
//! Google Vertex AI ({region}-aiplatform.googleapis.com) endpoints.
//!
//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`.
//! Shared conversion logic lives in `windmill_common::ai_google`.
use axum::body::Body;
use bytes::Bytes;
use eventsource_stream::Eventsource;
use futures::StreamExt;
use serde::Deserialize;
use serde_json::json;
use windmill_common::{
ai_google::{
gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini,
parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google,
GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool,
},
ai_types::OpenAIMessage,
error::{Error, Result},
};
use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS};
// ============================================================================
// Request type (OpenAI format received from the frontend)
// ============================================================================
#[derive(Deserialize, Debug)]
struct ChatRequest {
model: String,
messages: Vec<OpenAIMessage>,
#[serde(default)]
stream: bool,
#[serde(default)]
temperature: Option<f32>,
#[serde(default)]
max_tokens: Option<u32>,
#[serde(default)]
tools: Option<Vec<ChatRequestTool>>,
}
#[derive(Deserialize, Debug)]
struct ChatRequestTool {
function: ChatRequestToolFunction,
}
#[derive(Deserialize, Debug)]
struct ChatRequestToolFunction {
name: String,
#[serde(default)]
description: Option<String>,
#[serde(default)]
parameters: Option<serde_json::Value>,
}
// ============================================================================
// Helpers for Vertex AI vs standard Google AI URL/auth
// ============================================================================
/// Build the endpoint URL for a model action (streamGenerateContent, generateContent, predict).
///
/// - Standard: `{base_url}/models/{model}:{action}`
/// - Vertex AI: `{base_url}/{model}:{action}` (base_url already contains .../publishers/google/models)
fn build_model_endpoint(base_url: &str, model: &str, action: &str, is_vertex: bool) -> String {
if is_vertex {
format!("{}/{}:{}", base_url, model, action)
} else {
format!("{}/models/{}:{}", base_url, model, action)
}
}
/// Set the appropriate auth header on a request builder.
///
/// - Standard: `x-goog-api-key` header
/// - Vertex AI: `Authorization: Bearer` header
fn set_auth(
request: reqwest::RequestBuilder,
api_key: &str,
is_vertex: bool,
) -> reqwest::RequestBuilder {
if is_vertex {
request.header("Authorization", format!("Bearer {}", api_key))
} else {
request.header("x-goog-api-key", api_key)
}
}
// ============================================================================
// Public handler
// ============================================================================
/// Handle a `chat/completions` POST request using the native Gemini API.
///
/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it
/// to the appropriate Gemini endpoint, and converts the response back to the
/// OpenAI SSE or JSON format that the frontend expects.
pub async fn handle_google_ai_chat(
body: &Bytes,
api_key: &str,
base_url: &str,
is_vertex: bool,
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
let request: ChatRequest = serde_json::from_slice(body)
.map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?;
let (contents, system_instruction) = openai_messages_to_gemini(&request.messages);
let generation_config =
if request.temperature.is_some() || request.max_tokens.is_some() {
Some(GeminiGenerationConfig {
temperature: request.temperature,
max_output_tokens: request.max_tokens,
response_mime_type: None,
response_schema: None,
})
} else {
None
};
let gemini_tools = request.tools.as_ref().map(|tools| {
let declarations: Vec<GeminiFunctionDeclaration> = tools
.iter()
.map(|t| {
let mut params = t.function.parameters.clone().unwrap_or(json!({}));
sanitize_schema_for_google(&mut params);
GeminiFunctionDeclaration {
name: t.function.name.clone(),
description: t.function.description.clone(),
parameters: params,
}
})
.collect();
vec![GeminiTool {
function_declarations: Some(declarations),
google_search: None,
}]
});
let gemini_request = GeminiTextRequest {
contents,
tools: gemini_tools,
tool_config: None,
system_instruction,
generation_config,
};
let request_body = serde_json::to_string(&gemini_request)
.map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?;
let base_url = base_url.trim_end_matches('/');
if request.stream {
handle_streaming(&request.model, request_body, api_key, base_url, is_vertex).await
} else {
handle_non_streaming(&request.model, request_body, api_key, base_url, is_vertex).await
}
}
// ============================================================================
// Streaming path
// ============================================================================
async fn handle_streaming(
model: &str,
request_body: String,
api_key: &str,
base_url: &str,
is_vertex: bool,
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
let endpoint = format!(
"{}?alt=sse",
build_model_endpoint(base_url, model, "streamGenerateContent", is_vertex)
);
let request = HTTP_CLIENT
.post(&endpoint)
.header("content-type", "application/json")
.body(request_body);
let request = set_auth(request, api_key, is_vertex);
let response = request.send().await.map_err(|e| {
Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
})?;
if let Err(e) = response.error_for_status_ref() {
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
let body = response.text().await.unwrap_or_default();
return Err(Error::AIError(format!("{}: {}", status, body)));
}
let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple());
let model_str = model.to_string();
let gemini_sse_stream = response.bytes_stream().eventsource();
let openai_sse_stream = async_stream::stream! {
tokio::pin!(gemini_sse_stream);
let mut tool_call_index: usize = 0;
while let Some(event) = gemini_sse_stream.next().await {
match event {
Ok(event) => match parse_gemini_sse_event(&event.data) {
Ok(Some(parsed)) => {
for chunk in gemini_event_to_openai_sse_chunks(
&parsed, &id, &model_str, &mut tool_call_index,
) {
yield Ok::<Bytes, reqwest::Error>(Bytes::from(chunk));
}
}
Ok(None) => {}
Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e),
},
Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e),
}
}
yield Ok::<Bytes, reqwest::Error>(Bytes::from("data: [DONE]\n\n"));
};
let mut headers = http::HeaderMap::new();
headers.insert("content-type", "text/event-stream".parse().unwrap());
headers.insert("cache-control", "no-cache".parse().unwrap());
headers.insert("connection", "keep-alive".parse().unwrap());
Ok((
http::StatusCode::OK,
headers,
Body::from_stream(inject_keepalives(
Box::pin(openai_sse_stream),
std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS),
)),
))
}
// ============================================================================
// Model listing
// ============================================================================
/// List available Gemini models and convert to OpenAI format.
///
/// - Standard: `GET {base_url}/models` — returns `{ models: [...] }`
/// - Vertex AI: `GET {base_url}` — returns `{ models: [...] }` (base_url already ends with .../models)
pub async fn handle_google_ai_models(
api_key: &str,
base_url: &str,
is_vertex: bool,
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
#[derive(Deserialize)]
struct GeminiModel {
name: String,
#[serde(rename = "displayName", default)]
display_name: String,
}
#[derive(Deserialize)]
struct GeminiModelsResponse {
#[serde(default)]
models: Vec<GeminiModel>,
}
let base_url = base_url.trim_end_matches('/');
let endpoint = if is_vertex {
// Vertex AI: base_url is .../publishers/google/models
base_url.to_string()
} else {
// Standard: append /models
format!("{}/models", base_url)
};
let request = HTTP_CLIENT.get(&endpoint);
let request = set_auth(request, api_key, is_vertex);
let response = request.send().await.map_err(|e| {
Error::internal_err(format!("Failed to fetch Gemini models: {}", e))
})?;
if let Err(e) = response.error_for_status_ref() {
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
let body = response.text().await.unwrap_or_default();
return Err(Error::AIError(format!("{}: {}", status, body)));
}
let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| {
Error::internal_err(format!("Failed to parse Gemini models response: {}", e))
})?;
let data: Vec<serde_json::Value> = gemini_resp
.models
.into_iter()
.map(|m| {
json!({
"id": m.name,
"object": "model",
"display_name": m.display_name,
})
})
.collect();
let body_bytes = serde_json::to_vec(&json!({ "data": data }))
.map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?;
let mut headers = http::HeaderMap::new();
headers.insert("content-type", "application/json".parse().unwrap());
Ok((http::StatusCode::OK, headers, Body::from(body_bytes)))
}
// ============================================================================
// Non-streaming path
// ============================================================================
async fn handle_non_streaming(
model: &str,
request_body: String,
api_key: &str,
base_url: &str,
is_vertex: bool,
) -> Result<(http::StatusCode, http::HeaderMap, Body)> {
let endpoint = build_model_endpoint(base_url, model, "generateContent", is_vertex);
let request = HTTP_CLIENT
.post(&endpoint)
.header("content-type", "application/json")
.body(request_body);
let request = set_auth(request, api_key, is_vertex);
let response = request.send().await.map_err(|e| {
Error::internal_err(format!("Failed to send request to Gemini API: {}", e))
})?;
if let Err(e) = response.error_for_status_ref() {
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
let body = response.text().await.unwrap_or_default();
return Err(Error::AIError(format!("{}: {}", status, body)));
}
let body = response.bytes().await.map_err(|e| {
Error::internal_err(format!("Failed to read Gemini response body: {}", e))
})?;
let parsed = parse_gemini_response(&body)?;
let openai_response = gemini_response_to_openai(&parsed, model);
let body_bytes = serde_json::to_vec(&openai_response)
.map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?;
let mut headers = http::HeaderMap::new();
headers.insert("content-type", "application/json".parse().unwrap());
Ok((http::StatusCode::OK, headers, Body::from(body_bytes)))
}

View File

@@ -2255,13 +2255,12 @@ async fn resume_suspended_job_internal(
let value = value.unwrap_or(serde_json::Value::Null);
verify_suspended_secret(&w_id, &db, job_id, resume_id, &approver, secret).await?;
// Get flow info - works for step-level, flow-level, and WAC approval
let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?;
// Get flow info - works for both step-level (job_id is a step) and flow-level (job_id is the flow)
let (flow_info, is_flow_level) = get_flow_info_for_resume(job_id, &db).await?;
// For step-level resumes, verify user auth and flow status
// For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet
// For WAC approvals, skip flow status checks (there is no flow)
if !is_flow_level && !is_wac {
if !is_flow_level {
let parent_flow = GetQuery::new()
.without_logs()
.without_code()
@@ -2323,16 +2322,6 @@ async fn resume_suspended_job_internal(
)
.execute(&mut *tx)
.await?;
} else if is_wac {
// WAC approval: decrement suspend counter directly on the WAC parent job
if flow_info.suspend > 0 {
sqlx::query!(
"UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1",
flow_info.id,
)
.execute(&mut *tx)
.await?;
}
} else if is_flow_level {
// For flow-level resumes, decrement the suspend counter if the flow is currently suspended
// The approval will be matched when the worker checks for resumes (both step-level and flow-level)
@@ -2490,15 +2479,10 @@ struct FlowInfo {
email: Option<String>,
}
/// Get flow info from either a step job (by looking up its parent), a flow job directly,
/// or a WAC workflow job (self-suspended for approval).
/// Returns (FlowInfo, is_flow_level, is_wac) where:
/// - is_flow_level: job_id was a flow job (pre-approval)
/// - is_wac: job_id is a WAC workflow suspended for approval (target is itself)
async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool, bool)> {
// Single query that determines if job_id is a flow, step, or WAC job,
// and fetches the appropriate suspended job info.
// For WAC jobs (no parent, not a flow), the job itself is the suspended target.
/// Get flow info from either a step job (by looking up its parent) or a flow job directly.
/// Returns (FlowInfo, is_flow_level) where is_flow_level indicates if job_id was a flow job.
async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool)> {
// Single query that determines if job_id is a flow or step, and fetches the appropriate flow info
let result = sqlx::query!(
r#"
WITH job_info AS (
@@ -2512,15 +2496,14 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI
q.suspend AS "suspend!",
j.runnable_path AS script_path,
j.permissioned_as_email AS email,
(ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!",
(ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS "is_wac!"
(ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!"
FROM job_info ji
JOIN v2_job_queue q ON q.id = CASE
WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id
ELSE COALESCE(ji.parent_job, ji.id)
ELSE ji.parent_job
END
JOIN v2_job j ON j.id = q.id
LEFT JOIN v2_job_status s ON s.id = q.id
JOIN v2_job_status s ON s.id = q.id
FOR UPDATE OF q
"#,
job_id,
@@ -2537,7 +2520,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI
email: Some(result.email),
};
Ok((flow_info, result.is_flow_level, result.is_wac))
Ok((flow_info, result.is_flow_level))
}
async fn get_suspended_flow_info<'c>(
@@ -5433,12 +5416,12 @@ async fn add_batch_jobs(
if dedicated_worker && path.is_some() {
windmill_common::worker::dedicated_worker_tag(&w_id, &path.clone().unwrap())
} else {
format!("{}", language.as_str())
language.as_worker_tag(false).to_string()
}
} else if let Some(tag) = batch_info.tag {
tag
} else {
format!("{}", language.as_str())
language.as_worker_tag(false).to_string()
};
let mut tx = user_db.begin(&authed).await?;

View File

@@ -64,7 +64,6 @@ use crate::scim_oss::has_scim_token;
use windmill_common::error::AppError;
mod ai;
mod google;
mod apps;
pub mod args;
mod audit;
@@ -494,12 +493,16 @@ pub async fn run_server(
};
#[cfg(feature = "agent_worker_server")]
let (agent_workers_router, agent_workers_bg_processor, agent_workers_job_completed_tx) =
if server_mode {
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
} else {
(Router::new(), vec![], None)
};
let (
agent_workers_router,
agent_workers_bg_processor,
agent_workers_job_completed_tx,
batch_buffer,
) = if server_mode {
windmill_api_agent_workers::workspaced_service(db.clone(), _base_internal_url.clone())
} else {
(Router::new(), vec![], None, None)
};
#[cfg(feature = "agent_worker_server")]
let agent_cache = Arc::new(AgentCache::new());
@@ -685,6 +688,7 @@ pub async fn run_server(
{
windmill_api_agent_workers::global_service(
agent_workers_job_completed_tx,
batch_buffer.clone(),
)
.layer(Extension(agent_cache.clone()))
} else {

View File

@@ -16,7 +16,6 @@ use windmill_common::{
use windmill_dep_map::workspace_dependencies::{
trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use crate::db::ApiAuthed;
@@ -38,36 +37,21 @@ async fn create(
) -> error::Result<(StatusCode, String)> {
tracing::info!(workspace_id = %nwd.workspace_id, name = ?nwd.name, language = ?nwd.language, "create workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let dep_path = WorkspaceDependencies::to_path(&nwd.name, nwd.language)?;
let w_id = nwd.workspace_id.clone();
let email = authed.email.clone();
let username = authed.username.clone();
let id = nwd
.create(
(
authed.email,
username_to_permissioned_as(&authed.username),
authed.username,
),
db.clone(),
)
.await?;
handle_deployment_metadata(
&email,
&username,
&db,
&w_id,
DeployedObject::WorkspaceDependencies { path: dep_path },
None,
true,
None,
)
.await?;
Ok((StatusCode::CREATED, format!("{}", id)))
Ok((
StatusCode::CREATED,
format!(
"{}",
nwd.create(
(
authed.email,
username_to_permissioned_as(&authed.username),
authed.username,
),
db
)
.await?
),
))
}
#[axum::debug_handler]
@@ -108,21 +92,8 @@ async fn archive(
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "archive workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let db = &db;
let dep_path = WorkspaceDependencies::to_path(&params.name, language)?;
WorkspaceDependencies::archive(params.name.clone(), language, &w_id, db).await?;
handle_deployment_metadata(
&authed.email,
&authed.username,
db,
&w_id,
DeployedObject::WorkspaceDependencies { path: dep_path.clone() },
None,
true,
None,
)
.await?;
trigger_dependents_to_recompute_dependencies_in_the_background(
params.name.is_none(),
w_id,
@@ -132,7 +103,7 @@ async fn archive(
username_to_permissioned_as(&authed.username),
authed.username,
),
dep_path,
WorkspaceDependencies::to_path(&params.name, language)?,
db.clone(),
)
.await;
@@ -150,21 +121,8 @@ async fn delete(
tracing::info!(workspace_id = %w_id, language = ?language, name = ?params.name, "delete workspace dependencies");
require_admin(authed.is_admin, &authed.username)?;
let db = &db;
let dep_path = WorkspaceDependencies::to_path(&params.name, language)?;
WorkspaceDependencies::delete(params.name.clone(), language, &w_id, db).await?;
handle_deployment_metadata(
&authed.email,
&authed.username,
db,
&w_id,
DeployedObject::WorkspaceDependencies { path: dep_path.clone() },
None,
true,
None,
)
.await?;
trigger_dependents_to_recompute_dependencies_in_the_background(
params.name.is_none(),
w_id,
@@ -174,7 +132,7 @@ async fn delete(
username_to_permissioned_as(&authed.username),
authed.username,
),
dep_path,
WorkspaceDependencies::to_path(&params.name, language)?,
db.clone(),
)
.await;

View File

@@ -282,12 +282,6 @@ struct SimplifiedSettings {
color: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
operator_settings: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
slack_team_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
slack_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
slack_command_script: Option<String>,
}
// V1 format: Legacy flat format for backward compatibility (matches main branch exactly)
@@ -322,12 +316,6 @@ struct SimplifiedSettingsLegacy {
color: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
operator_settings: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
slack_team_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
slack_name: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
slack_command_script: Option<String>,
}
// Internal struct for querying database
@@ -347,9 +335,6 @@ struct SettingsRow {
mute_critical_alerts: Option<bool>,
color: Option<String>,
operator_settings: Option<serde_json::Value>,
slack_team_id: Option<String>,
slack_name: Option<String>,
slack_command_script: Option<String>,
}
pub(crate) async fn tarball_workspace(
@@ -954,10 +939,7 @@ pub(crate) async fn tarball_workspace(
workspace.name as name,
mute_critical_alerts,
color,
operator_settings,
slack_team_id,
slack_name,
slack_command_script
operator_settings
FROM workspace_settings
LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id
WHERE workspace_id = $1"#,
@@ -983,9 +965,6 @@ pub(crate) async fn tarball_workspace(
mute_critical_alerts: row.mute_critical_alerts,
color: row.color.clone(),
operator_settings: row.operator_settings.clone(),
slack_team_id: row.slack_team_id.clone(),
slack_name: row.slack_name.clone(),
slack_command_script: row.slack_command_script.clone(),
};
serde_json::to_value(settings)
.map(|v| serde_json::to_string_pretty(&v).ok())
@@ -1045,9 +1024,6 @@ pub(crate) async fn tarball_workspace(
mute_critical_alerts: row.mute_critical_alerts,
color: row.color,
operator_settings: row.operator_settings,
slack_team_id: row.slack_team_id,
slack_name: row.slack_name,
slack_command_script: row.slack_command_script,
};
serde_json::to_value(settings)
.map(|v| serde_json::to_string_pretty(&v).ok())

View File

@@ -1,726 +0,0 @@
//! Shared Google AI (Gemini API) types and conversion utilities.
//!
//! This module provides:
//! - Gemini request/response types
//! - OpenAI → Gemini message conversion
//! - Gemini SSE event parsing
//!
//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent).
use serde::{Deserialize, Serialize};
use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation};
use crate::error::Error;
// ============================================================================
// Request / Content Types
// ============================================================================
/// Inline data for binary content (images).
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct GeminiInlineData {
#[serde(rename = "mimeType")]
pub mime_type: String,
pub data: String,
}
/// A part of content — text, inline data, function call, or function response.
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(untagged)]
pub enum GeminiPart {
Text {
text: String,
},
InlineData {
#[serde(rename = "inlineData")]
inline_data: GeminiInlineData,
},
FunctionCall {
#[serde(rename = "functionCall")]
function_call: GeminiFunctionCall,
/// Thought signature for Gemini 3+ models — required when replaying function calls.
#[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")]
thought_signature: Option<String>,
},
FunctionResponse {
#[serde(rename = "functionResponse")]
function_response: GeminiFunctionResponse,
},
}
/// A function call from the model.
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct GeminiFunctionCall {
pub name: String,
pub args: serde_json::Value,
}
/// A function response sent back to the model.
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct GeminiFunctionResponse {
pub name: String,
pub response: serde_json::Value,
}
/// Content message with an optional role and a list of parts.
#[derive(Serialize, Clone, Debug)]
pub struct GeminiContentMessage {
#[serde(skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
pub parts: Vec<GeminiPart>,
}
/// Main request body for `generateContent` / `streamGenerateContent`.
#[derive(Serialize)]
pub struct GeminiTextRequest {
pub contents: Vec<GeminiContentMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<GeminiTool>>,
#[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")]
pub tool_config: Option<GeminiToolConfig>,
#[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")]
pub system_instruction: Option<GeminiContentMessage>,
#[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")]
pub generation_config: Option<GeminiGenerationConfig>,
}
/// Tool definition — function declarations and/or Google Search grounding.
#[derive(Serialize)]
pub struct GeminiTool {
#[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")]
pub function_declarations: Option<Vec<GeminiFunctionDeclaration>>,
#[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")]
pub google_search: Option<serde_json::Value>,
}
/// A single function declaration.
///
/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema.
#[derive(Serialize)]
pub struct GeminiFunctionDeclaration {
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
pub parameters: serde_json::Value,
}
/// Tool configuration controlling when and how functions are called.
#[derive(Serialize)]
pub struct GeminiToolConfig {
#[serde(rename = "functionCallingConfig")]
pub function_calling_config: GeminiFunctionCallingConfig,
}
/// Function calling mode and optional allow-list.
#[derive(Serialize)]
pub struct GeminiFunctionCallingConfig {
pub mode: String,
#[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")]
pub allowed_function_names: Option<Vec<String>>,
}
/// Generation parameters (temperature, token limits, structured output).
#[derive(Serialize)]
pub struct GeminiGenerationConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f32>,
#[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")]
pub max_output_tokens: Option<u32>,
#[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")]
pub response_mime_type: Option<String>,
#[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")]
pub response_schema: Option<serde_json::Value>,
}
// ============================================================================
// Image Generation Types
// ============================================================================
/// Request body for Imagen / Gemini image generation.
#[derive(Serialize)]
pub struct GeminiImageRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub contents: Option<Vec<GeminiImageContent>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub instances: Option<Vec<GeminiPredictContent>>,
}
/// Content wrapper used in `generateContent` image requests.
#[derive(Serialize)]
pub struct GeminiImageContent {
pub parts: Vec<GeminiPart>,
}
/// Prompt wrapper for Imagen `predict` endpoint.
#[derive(Serialize)]
pub struct GeminiPredictContent {
pub prompt: String,
}
/// Top-level response from Gemini/Imagen image generation.
#[derive(Deserialize)]
pub struct GeminiImageResponse {
pub candidates: Option<Vec<GeminiImageCandidate>>,
pub predictions: Option<Vec<GeminiPredictCandidate>>,
}
#[derive(Deserialize)]
pub struct GeminiImageCandidate {
pub content: GeminiImageCandidateContent,
}
#[derive(Deserialize)]
pub struct GeminiImageCandidateContent {
pub parts: Vec<GeminiImageCandidatePart>,
}
#[derive(Deserialize)]
pub struct GeminiImageCandidatePart {
#[serde(rename = "inlineData")]
pub inline_data: Option<GeminiInlineData>,
}
#[derive(Deserialize)]
pub struct GeminiPredictCandidate {
#[serde(rename = "bytesBase64Encoded")]
pub bytes_base64_encoded: String,
}
// ============================================================================
// SSE Response Types
// ============================================================================
/// One part inside a streaming candidate — text, function call, or thought signature.
#[derive(Deserialize, Debug)]
pub struct GeminiSSEPart {
#[serde(default)]
pub text: Option<String>,
#[serde(rename = "functionCall")]
pub function_call: Option<GeminiSSEFunctionCall>,
/// Thought signature for Gemini 3+ models.
#[serde(rename = "thoughtSignature")]
pub thought_signature: Option<String>,
}
/// Function call contained in a streaming part.
#[derive(Deserialize, Debug)]
pub struct GeminiSSEFunctionCall {
pub name: String,
pub args: serde_json::Value,
}
/// Content block inside a streaming candidate.
#[derive(Deserialize, Debug)]
pub struct GeminiSSEContent {
pub parts: Option<Vec<GeminiSSEPart>>,
}
/// Web source from a Gemini grounding chunk.
#[derive(Deserialize, Debug)]
pub struct GeminiGroundingChunkWeb {
pub uri: String,
#[serde(default)]
pub title: Option<String>,
}
/// One grounding chunk (search result) from Gemini web search.
#[derive(Deserialize, Debug)]
pub struct GeminiGroundingChunk {
pub web: Option<GeminiGroundingChunkWeb>,
}
/// Grounding metadata attached to a streaming candidate.
#[derive(Deserialize, Debug)]
pub struct GeminiGroundingMetadata {
#[serde(rename = "groundingChunks", default)]
pub grounding_chunks: Vec<GeminiGroundingChunk>,
#[serde(rename = "webSearchQueries", default)]
pub web_search_queries: Vec<String>,
}
/// One candidate inside a streaming Gemini response.
#[derive(Deserialize, Debug)]
pub struct GeminiSSECandidate {
pub content: Option<GeminiSSEContent>,
#[serde(rename = "finishReason")]
pub finish_reason: Option<String>,
#[serde(rename = "groundingMetadata")]
pub grounding_metadata: Option<GeminiGroundingMetadata>,
}
/// Token usage from the `usageMetadata` field of a Gemini SSE event.
#[derive(Deserialize, Debug, Clone)]
pub struct GeminiUsageMetadata {
#[serde(rename = "promptTokenCount", default)]
pub prompt_token_count: Option<i32>,
#[serde(rename = "candidatesTokenCount", default)]
pub candidates_token_count: Option<i32>,
#[serde(rename = "totalTokenCount", default)]
pub total_token_count: Option<i32>,
}
/// Top-level structure of one Gemini SSE event.
#[derive(Deserialize, Debug)]
pub struct GeminiSSEEvent {
pub candidates: Option<Vec<GeminiSSECandidate>>,
#[serde(rename = "usageMetadata")]
pub usage_metadata: Option<GeminiUsageMetadata>,
}
// ============================================================================
// Parsed Event Result
// ============================================================================
/// A single function call extracted from a Gemini SSE event.
#[derive(Debug)]
pub struct GeminiToolCallEvent {
pub name: String,
pub args: serde_json::Value,
pub thought_signature: Option<String>,
}
impl GeminiToolCallEvent {
/// Convert the thought signature (if present) into an [`ExtraContent`].
pub fn to_extra_content(&self) -> Option<ExtraContent> {
self.thought_signature.as_ref().map(|sig| ExtraContent {
google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }),
})
}
}
/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body).
#[derive(Debug, Default)]
pub struct GeminiParsedEvent {
pub text: Option<String>,
pub tool_calls: Vec<GeminiToolCallEvent>,
pub annotations: Vec<UrlCitation>,
pub used_websearch: bool,
pub usage: Option<GeminiUsageMetadata>,
pub finish_reason: Option<String>,
}
// ============================================================================
// Helper Functions
// ============================================================================
/// Parse a data URL into `(mime_type, base64_data)`.
///
/// Expected format: `data:<mime_type>;base64,<data>`.
pub fn parse_data_url(url: &str) -> Option<(String, String)> {
let rest = url.strip_prefix("data:")?;
let (header, data) = rest.split_once(',')?;
let media_type = header.strip_suffix(";base64")?;
Some((media_type.to_string(), data.to_string()))
}
/// Find the function name associated with a `tool_call_id` by scanning prior messages.
pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String {
messages
.iter()
.filter_map(|msg| msg.tool_calls.as_ref())
.flatten()
.find(|tc| tc.id == tool_call_id)
.map(|tc| tc.function.name.clone())
.unwrap_or_else(|| "unknown_function".to_string())
}
/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s.
///
/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here;
/// the worker handles them by downloading and injecting inline data beforehand.
pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec<GeminiPart> {
match content {
OpenAIContent::Text(text) if !text.is_empty() => {
vec![GeminiPart::Text { text: text.clone() }]
}
OpenAIContent::Text(_) => vec![],
OpenAIContent::Parts(parts) => parts
.iter()
.filter_map(|part| match part {
ContentPart::Text { text } if !text.is_empty() => {
Some(GeminiPart::Text { text: text.clone() })
}
ContentPart::ImageUrl { image_url } => {
parse_data_url(&image_url.url).map(|(mime_type, data)| {
GeminiPart::InlineData {
inline_data: GeminiInlineData { mime_type, data },
}
})
}
// S3Objects are handled by the worker
_ => None,
})
.collect(),
}
}
/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`.
///
/// Returns `(contents, system_instruction)`.
///
/// `S3Object` images in content parts are skipped (the worker pre-converts them).
/// Tool call history is preserved correctly for multi-turn agent conversations.
pub fn openai_messages_to_gemini(
messages: &[OpenAIMessage],
) -> (Vec<GeminiContentMessage>, Option<GeminiContentMessage>) {
let mut contents: Vec<GeminiContentMessage> = Vec::new();
let mut system_instruction: Option<GeminiContentMessage> = None;
for msg in messages {
match msg.role.as_str() {
"system" => {
if let Some(content) = &msg.content {
let parts = convert_content_to_gemini_parts(content);
if !parts.is_empty() {
system_instruction =
Some(GeminiContentMessage { role: None, parts });
}
}
}
"tool" => {
if let (Some(tool_call_id), Some(content)) =
(&msg.tool_call_id, &msg.content)
{
let func_name = find_gemini_function_name(messages, tool_call_id);
let response_text = match content {
OpenAIContent::Text(text) => text.clone(),
OpenAIContent::Parts(parts) => parts
.iter()
.filter_map(|p| {
if let ContentPart::Text { text } = p {
Some(text.as_str())
} else {
None
}
})
.collect::<Vec<_>>()
.join(" "),
};
contents.push(GeminiContentMessage {
role: Some("user".to_string()),
parts: vec![GeminiPart::FunctionResponse {
function_response: GeminiFunctionResponse {
name: func_name,
response: serde_json::json!({ "result": response_text }),
},
}],
});
}
}
role => {
let gemini_role = if role == "assistant" { "model" } else { "user" };
let mut parts: Vec<GeminiPart> = Vec::new();
if let Some(content) = &msg.content {
parts.extend(convert_content_to_gemini_parts(content));
}
if let Some(tool_calls) = &msg.tool_calls {
for tc in tool_calls {
let args: serde_json::Value =
serde_json::from_str(&tc.function.arguments).unwrap_or_default();
let thought_signature = tc
.extra_content
.as_ref()
.and_then(|ec| ec.google.as_ref())
.and_then(|g| g.thought_signature.clone());
parts.push(GeminiPart::FunctionCall {
function_call: GeminiFunctionCall {
name: tc.function.name.clone(),
args,
},
thought_signature,
});
}
}
if !parts.is_empty() {
contents.push(GeminiContentMessage {
role: Some(gemini_role.to_string()),
parts,
});
}
}
}
}
(contents, system_instruction)
}
/// Convert OpenAI tool definitions to Gemini format.
///
/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google)
/// JSON schema values, one per entry in `tools` in the same order.
pub fn openai_tools_to_gemini(
tools: &[ToolDef],
tool_params: &[serde_json::Value],
has_websearch: bool,
) -> Option<Vec<GeminiTool>> {
let mut gemini_tools: Vec<GeminiTool> = Vec::new();
let declarations: Vec<GeminiFunctionDeclaration> = tools
.iter()
.zip(tool_params.iter())
.map(|(t, params)| GeminiFunctionDeclaration {
name: t.function.name.clone(),
description: t.function.description.clone(),
parameters: params.clone(),
})
.collect();
if !declarations.is_empty() {
gemini_tools.push(GeminiTool {
function_declarations: Some(declarations),
google_search: None,
});
}
if has_websearch {
gemini_tools.push(GeminiTool {
function_declarations: None,
google_search: Some(serde_json::json!({})),
});
}
if gemini_tools.is_empty() {
None
} else {
Some(gemini_tools)
}
}
/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`].
///
/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`).
/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating.
pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, Error> {
if data.is_empty() || data == "[DONE]" {
return Ok(None);
}
let event: GeminiSSEEvent = match serde_json::from_str(data) {
Ok(e) => e,
Err(e) => {
tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e);
return Ok(None);
}
};
let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
let Some(candidates) = event.candidates else {
return Ok(Some(parsed));
};
extract_candidates_into(&candidates, &mut parsed);
Ok(Some(parsed))
}
/// Parse a non-streaming Gemini `generateContent` response body.
pub fn parse_gemini_response(data: &[u8]) -> Result<GeminiParsedEvent, Error> {
let event: GeminiSSEEvent = serde_json::from_slice(data)
.map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?;
let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
if let Some(candidates) = event.candidates {
extract_candidates_into(&candidates, &mut parsed);
}
Ok(parsed)
}
// ============================================================================
// Gemini → OpenAI Format Conversion
// ============================================================================
/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON.
pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value {
let content = parsed.text.as_deref().unwrap_or_default();
let tool_calls: Vec<serde_json::Value> = parsed
.tool_calls
.iter()
.enumerate()
.map(|(i, tc)| {
serde_json::json!({
"index": i,
"id": format!("call_{}", uuid::Uuid::new_v4().simple()),
"type": "function",
"function": {
"name": tc.name,
"arguments": serde_json::to_string(&tc.args).unwrap_or_default()
}
})
})
.collect();
let finish_reason = parsed
.finish_reason
.as_deref()
.map(|r| r.to_lowercase())
.unwrap_or_else(|| "stop".to_string());
let usage = parsed.usage.as_ref().map(|u| {
serde_json::json!({
"prompt_tokens": u.prompt_token_count.unwrap_or(0),
"completion_tokens": u.candidates_token_count.unwrap_or(0),
"total_tokens": u.total_token_count.unwrap_or(0),
})
});
let mut message = serde_json::json!({
"role": "assistant",
"content": content,
});
if !tool_calls.is_empty() {
message["tool_calls"] = serde_json::json!(tool_calls);
}
serde_json::json!({
"id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()),
"object": "chat.completion",
"model": model,
"choices": [{
"index": 0,
"message": message,
"finish_reason": finish_reason,
}],
"usage": usage,
})
}
/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines.
///
/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream.
/// `tool_call_index` is mutated to track the running index across multiple SSE events.
pub fn gemini_event_to_openai_sse_chunks(
parsed: &GeminiParsedEvent,
id: &str,
model: &str,
tool_call_index: &mut usize,
) -> Vec<String> {
let mut chunks = Vec::new();
if let Some(text) = &parsed.text {
let chunk = serde_json::json!({
"id": id,
"object": "chat.completion.chunk",
"model": model,
"choices": [{
"index": 0,
"delta": { "content": text },
"finish_reason": null,
}]
});
chunks.push(format!("data: {}\n\n", chunk));
}
for tc in &parsed.tool_calls {
let args_str = serde_json::to_string(&tc.args).unwrap_or_default();
let call_id = format!("call_{}", uuid::Uuid::new_v4().simple());
let chunk = serde_json::json!({
"id": id,
"object": "chat.completion.chunk",
"model": model,
"choices": [{
"index": 0,
"delta": {
"tool_calls": [{
"index": *tool_call_index,
"id": call_id,
"type": "function",
"function": {
"name": tc.name,
"arguments": args_str,
}
}]
},
"finish_reason": null,
}]
});
chunks.push(format!("data: {}\n\n", chunk));
*tool_call_index += 1;
}
chunks
}
/// Recursively remove JSON Schema fields unsupported by the Gemini API.
pub fn sanitize_schema_for_google(value: &mut serde_json::Value) {
const UNSUPPORTED: &[&str] = &[
"additionalProperties",
"strict",
"$schema",
"default",
"exclusiveMinimum",
"exclusiveMaximum",
"const",
"multipleOf",
];
if let Some(obj) = value.as_object_mut() {
for field in UNSUPPORTED {
obj.remove(*field);
}
for v in obj.values_mut() {
sanitize_schema_for_google(v);
}
} else if let Some(arr) = value.as_array_mut() {
for v in arr.iter_mut() {
sanitize_schema_for_google(v);
}
}
}
// ============================================================================
// Internal Helpers
// ============================================================================
fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) {
for candidate in candidates {
if let Some(content) = &candidate.content {
if let Some(parts) = &content.parts {
for part in parts {
if let Some(text) = &part.text {
if !text.is_empty() {
match parsed.text.as_mut() {
Some(existing) => existing.push_str(text),
None => parsed.text = Some(text.clone()),
}
}
}
if let Some(function_call) = &part.function_call {
parsed.tool_calls.push(GeminiToolCallEvent {
name: function_call.name.clone(),
args: function_call.args.clone(),
thought_signature: part.thought_signature.clone(),
});
}
}
}
}
if candidate.finish_reason.is_some() {
parsed.finish_reason = candidate.finish_reason.clone();
}
if let Some(grounding) = &candidate.grounding_metadata {
if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() {
parsed.used_websearch = true;
}
for chunk in &grounding.grounding_chunks {
if let Some(web) = &chunk.web {
parsed.annotations.push(UrlCitation {
start_index: 0,
end_index: 0,
url: web.uri.clone(),
title: web.title.clone(),
});
}
}
}
}
}

View File

@@ -29,15 +29,6 @@ pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/
/// (e.g., AWS_REGION or AWS_DEFAULT_REGION env vars, or ~/.aws/config)
pub const USE_ENV_REGION: &str = "";
/// Platform variant for providers that support Google Vertex AI (Anthropic, GoogleAI).
#[derive(Deserialize, Debug, Clone, Default, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum AIPlatform {
#[default]
Standard,
GoogleVertexAi,
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)]
#[serde(rename_all = "lowercase")]
pub enum AIProvider {

View File

@@ -339,7 +339,7 @@ async fn fetch_authed_from_permissioned_as_inner(
if let Some(r) = r {
(r.is_admin, r.operator)
} else {
return Err(Error::NotFound(format!(
return Err(Error::internal_err(format!(
"user {name} not found in workspace {w_id}"
)));
}

View File

@@ -78,8 +78,6 @@ pub enum Error {
AIError(String),
#[error("{0}")]
AlreadyCompleted(String),
#[error("WAC job suspended: {0}")]
WacSuspended(String),
#[error("Find python error: {0}")]
FindPythonError(String),
#[error("Problem with arguments: {0}")]
@@ -110,7 +108,6 @@ impl Error {
Self::JsonErr(_) => "JsonErr",
Self::AIError(_) => "AIError",
Self::AlreadyCompleted(_) => "AlreadyCompleted",
Self::WacSuspended(_) => "WacSuspended",
Self::FindPythonError(_) => "FindPythonError",
Self::ArgumentErr(_) => "ArgumentErr",
Self::Generic(_, _) => "Generic",

View File

@@ -4,7 +4,6 @@ pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces";
pub const BASE_URL_SETTING: &str = "base_url";
pub const OAUTH_SETTING: &str = "oauths";
pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
pub const AUDIT_LOG_RETENTION_DAYS_SETTING: &str = "audit_log_retention_days";
pub const MONITOR_LOGS_ON_OBJECT_STORE_SETTING: &str = "monitor_logs_on_s3";
pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout";
pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";

View File

@@ -13,7 +13,6 @@ pub struct TantivyIndexerSettings {
pub refresh_index_period: u64,
pub refresh_log_index_period: u64,
pub max_indexed_job_log_size: usize,
pub max_index_time_window_secs: i64,
pub should_clear_job_index: bool,
pub should_clear_log_index: bool,
}
@@ -27,7 +26,6 @@ impl Default for TantivyIndexerSettings {
refresh_index_period: 300,
refresh_log_index_period: 300,
max_indexed_job_log_size: 1_000_000,
max_index_time_window_secs: 60 * 60 * 24 * 7, // 7 days
should_clear_job_index: false,
should_clear_log_index: false,
}
@@ -41,7 +39,6 @@ pub struct TantivyIndexerSettingsOpt {
pub refresh_index_period: Option<u64>,
pub refresh_log_index_period: Option<u64>,
pub max_indexed_job_log_size: Option<usize>,
pub max_index_time_window_secs: Option<i64>,
pub should_clear_job_index: Option<bool>,
pub should_clear_log_index: Option<bool>,
}
@@ -61,7 +58,6 @@ pub async fn load_indexer_config(db: &DB) -> error::Result<TantivyIndexerSetting
refresh_index_period,
refresh_log_index_period,
max_indexed_job_log_size,
max_index_time_window_secs,
writer_memory_budget,
should_clear_job_index,
should_clear_log_index,
@@ -82,9 +78,6 @@ pub async fn load_indexer_config(db: &DB) -> error::Result<TantivyIndexerSetting
max_indexed_job_log_size: config
.max_indexed_job_log_size
.unwrap_or(max_indexed_job_log_size),
max_index_time_window_secs: config
.max_index_time_window_secs
.unwrap_or(max_index_time_window_secs),
should_clear_job_index: config
.should_clear_job_index
.unwrap_or(should_clear_job_index),
@@ -130,9 +123,6 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings {
if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__KB") {
settings.max_indexed_job_log_size = (b * BYTES_PER_KB) as usize;
}
if let Some(b) = get_env_var("TANTIVY_MAX_INDEX_TIME_WINDOW__S") {
settings.max_index_time_window_secs = b as i64;
}
settings
}

View File

@@ -44,7 +44,7 @@ pub struct EnvRefWrapper {
///
/// `Literal` serializes back to a plain JSON string, preserving backwards
/// compatibility with existing consumers.
#[derive(Deserialize, Serialize, Clone)]
#[derive(Deserialize, Serialize, Clone, Debug)]
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum StringOrSecretRef {
@@ -53,16 +53,6 @@ pub enum StringOrSecretRef {
EnvRef(EnvRefWrapper),
}
impl fmt::Debug for StringOrSecretRef {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Literal(_) => f.write_str("Literal(****)"),
Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(),
Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(),
}
}
}
impl StringOrSecretRef {
/// Returns the literal string value, or `None` if this is an unresolved ref.
pub fn as_literal(&self) -> Option<&str> {
@@ -265,25 +255,25 @@ pub struct GlobalSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub instance_python_version: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pip_index_url: Option<StringOrSecretRef>,
pub pip_index_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub pip_extra_index_url: Option<StringOrSecretRef>,
pub pip_extra_index_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub npm_config_registry: Option<StringOrSecretRef>,
pub npm_config_registry: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bunfig_install_scopes: Option<StringOrSecretRef>,
pub bunfig_install_scopes: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub npmrc: Option<StringOrSecretRef>,
pub npmrc: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub nuget_config: Option<StringOrSecretRef>,
pub nuget_config: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub maven_repos: Option<StringOrSecretRef>,
pub maven_repos: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ruby_repos: Option<StringOrSecretRef>,
pub ruby_repos: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub powershell_repo_url: Option<StringOrSecretRef>,
pub powershell_repo_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub powershell_repo_pat: Option<StringOrSecretRef>,
pub powershell_repo_pat: Option<String>,
// Array settings
#[serde(skip_serializing_if = "Option::is_none")]
@@ -934,7 +924,7 @@ fn redact_string(s: &str) -> String {
}
}
pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String {
fn format_setting_value(key: &str, value: &serde_json::Value) -> String {
if SENSITIVE_SETTINGS.contains(&key) {
return match value {
serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)),
@@ -2219,25 +2209,6 @@ mod tests {
assert_eq!(v, *"world");
}
#[test]
fn string_or_secret_ref_debug_masks_literal() {
let v = StringOrSecretRef::Literal("super-secret-value".to_string());
let debug = format!("{v:?}");
assert_eq!(debug, "Literal(****)");
assert!(!debug.contains("super-secret-value"));
}
#[test]
fn format_setting_value_redacts_oauth_secrets() {
let val = serde_json::json!({
"google": {"id": "client-id", "secret": "my-super-secret-12345"}
});
let formatted = format_setting_value("oauths", &val);
assert!(!formatted.contains("my-super-secret-12345"));
assert!(formatted.contains("client-id"));
assert!(formatted.contains("****"));
}
#[test]
#[should_panic(expected = "literal_value() called on unresolved secret ref")]
fn string_or_secret_ref_literal_value_panics_on_ref() {

View File

@@ -15,11 +15,9 @@ pub struct JobStatsRecord {
pub timestamps: Option<Vec<chrono::DateTime<chrono::Utc>>>,
pub timeseries_int: Option<Vec<i32>>,
pub timeseries_float: Option<Vec<f32>>,
pub timeseries_start: Option<chrono::DateTime<chrono::Utc>>,
pub offsets_cs: Option<Vec<i32>>,
}
#[derive(sqlx::Type, Debug, Clone, PartialEq, Deserialize, Serialize)]
#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)]
#[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")]
pub enum MetricKind {
ScalarInt,
@@ -54,21 +52,29 @@ pub async fn register_metric_for_job(
return Ok(metric_id);
}
let is_timeseries = matches!(
metric_kind,
MetricKind::TimeseriesInt | MetricKind::TimeseriesFloat
);
let (scalar_int, scalar_float, timeseries_int, timeseries_float) = match metric_kind {
let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind
{
MetricKind::ScalarInt | MetricKind::ScalarFloat => {
(None as Option<i32>, None as Option<f32>, None, None)
(None as Option<i32>, None as Option<f32>, None, None, None)
}
MetricKind::TimeseriesInt => (None, None, Some(&[] as &[i32]), None),
MetricKind::TimeseriesFloat => (None, None, None, Some(&[] as &[f32])),
MetricKind::TimeseriesInt => (
None,
None,
Some(&[] as &[chrono::DateTime<chrono::Utc>]),
Some(&[] as &[i32]),
None,
),
MetricKind::TimeseriesFloat => (
None,
None,
Some(&[] as &[chrono::DateTime<chrono::Utc>]),
None,
Some(&[] as &[f32]),
),
};
sqlx::query(
"INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timeseries_int, timeseries_float, timeseries_start, offsets_cs) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CASE WHEN $10 THEN now() ELSE NULL END, CASE WHEN $10 THEN ARRAY[]::int[] ELSE NULL END)",
"INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)",
)
.bind(workspace_id)
.bind(job_id)
@@ -77,9 +83,9 @@ pub async fn register_metric_for_job(
.bind(metric_kind)
.bind(scalar_int)
.bind(scalar_float)
.bind(timestamps)
.bind(timeseries_int)
.bind(timeseries_float)
.bind(is_timeseries)
.execute(db)
.warn_after_seconds(1)
.await?;
@@ -111,30 +117,6 @@ pub async fn record_metric(
}
let metric_kind = metric_kind_opt.unwrap();
record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await
}
/// Record a timeseries metric value without the extra SELECT to look up metric_kind.
/// Use this when the caller already knows the metric kind (e.g. the worker that registered it).
pub async fn record_timeseries_value(
db: &DB,
workspace_id: String,
job_id: Uuid,
metric_id: String,
value: MetricNumericValue,
metric_kind: MetricKind,
) -> error::Result<()> {
record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await
}
async fn record_metric_impl(
db: &DB,
workspace_id: String,
job_id: Uuid,
metric_id: String,
value: MetricNumericValue,
metric_kind: MetricKind,
) -> error::Result<()> {
let (value_int, value_float) = match value {
MetricNumericValue::Integer(val) => {
if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt {
@@ -178,7 +160,7 @@ async fn record_metric_impl(
}
MetricKind::TimeseriesInt => {
sqlx::query!(
"UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
&workspace_id,
&job_id,
&metric_id,
@@ -187,7 +169,7 @@ async fn record_metric_impl(
}
MetricKind::TimeseriesFloat => {
sqlx::query!(
"UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
"UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3",
&workspace_id,
&job_id,
&metric_id,

View File

@@ -29,7 +29,6 @@ use sqlx::{Acquire, Postgres};
pub mod agent_workers;
#[cfg(feature = "bedrock")]
pub mod ai_bedrock;
pub mod ai_google;
pub mod ai_providers;
pub mod ai_types;
pub mod apps;
@@ -205,7 +204,6 @@ lazy_static::lazy_static! {
pub static ref CRITICAL_ALERTS_ON_DB_OVERSIZE: Arc<RwLock<Option<f32>>> = Arc::new(RwLock::new(None));
pub static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
pub static ref AUDIT_LOG_RETENTION_DAYS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
pub static ref MONITOR_LOGS_ON_OBJECT_STORE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));

View File

@@ -288,6 +288,10 @@ pub fn is_native_mode_from_env() -> bool {
/// Use this for hot-path checks (e.g. per-job dispatch) to avoid read-locking WORKER_CONFIG.
pub static NATIVE_MODE_RESOLVED: AtomicBool = AtomicBool::new(false);
/// Whether this worker uses HTTP batch pull (set at startup in main.rs).
/// Reported in worker_ping so the server knows which native workers to batch-pull for.
pub static USES_BATCH_HTTP_PULL: AtomicBool = AtomicBool::new(false);
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
#[derive(Clone)]
pub struct HttpClient {
@@ -516,6 +520,62 @@ pub fn make_pull_query(tags: &[String]) -> String {
query
}
pub fn make_batch_pull_query(tags: &[String], limit: u32) -> String {
format_batch_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
ORDER BY priority DESC NULLS LAST, scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT {limit}",
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
fn format_batch_pull_query(peek: String) -> String {
// Optimizations vs single-row format_pull_query:
// 1. ANY(ARRAY(SELECT ...)) instead of IN (SELECT ...) — forces PG to materialize IDs
// into an array, enabling Bitmap Index Scan instead of Hash Semi Join / Nested Loop
// 2. r CTE chains off q (not peek) — only updates runtime for actually-locked rows,
// avoids re-scanning peek
// 3. No separate j CTE — join v2_job directly in final SELECT off q's IDs
format!(
"WITH peek AS (
{}
), q AS NOT MATERIALIZED (
UPDATE v2_job_queue SET
running = true,
started_at = coalesce(started_at, now()),
suspend_until = null,
worker = $1
WHERE id = ANY(ARRAY(SELECT id FROM peek))
RETURNING
id, started_at, scheduled_for,
canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle
), r AS NOT MATERIALIZED (
UPDATE v2_job_runtime SET
ping = now()
WHERE id = ANY(ARRAY(SELECT id FROM q))
) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for,
j.runnable_id, j.runnable_path, j.args, q.canceled_by,
q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as,
f.flow_status, j.script_lang,
j.same_worker, j.pre_run_error, j.visible_to_owner,
j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job,
j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow,
j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path,
COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
FROM q
JOIN v2_job j ON q.id = j.id
LEFT JOIN v2_job_status f ON f.id = q.id
LEFT JOIN job_perms p ON p.job_id = q.id
LEFT JOIN v2_job pj ON j.parent_job = pj.id
",
peek
)
}
pub async fn store_pull_query(wc: &WorkerConfig) {
let mut queries = vec![];
for tags in wc.priority_tags_sorted.iter() {
@@ -1194,6 +1254,8 @@ pub struct Ping {
pub occupancy_rate_30m: Option<f32>,
pub job_isolation: Option<String>,
pub native_mode: Option<bool>,
#[serde(default)]
pub uses_batch_http_pull: Option<bool>,
pub ping_type: PingType,
}
pub async fn update_ping_http(
@@ -1218,6 +1280,7 @@ pub async fn update_ping_http(
insert_ping.occupancy_rate_5m,
insert_ping.occupancy_rate_30m,
insert_ping.native_mode.unwrap_or(false),
insert_ping.uses_batch_http_pull.unwrap_or(false),
db,
)
.await?
@@ -1245,6 +1308,7 @@ pub async fn update_ping_http(
insert_ping.memory,
insert_ping.job_isolation,
insert_ping.native_mode.unwrap_or(false),
insert_ping.uses_batch_http_pull.unwrap_or(false),
db,
)
.await?;
@@ -1377,11 +1441,12 @@ pub async fn insert_ping_query(
memory: Option<i64>,
job_isolation: Option<String>,
native_mode: bool,
uses_batch_http_pull: bool,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode",
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, uses_batch_http_pull) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) ON CONFLICT (worker)
DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_workers = EXCLUDED.dedicated_workers, native_mode = EXCLUDED.native_mode, uses_batch_http_pull = EXCLUDED.uses_batch_http_pull",
worker_instance,
worker_name,
ip,
@@ -1394,6 +1459,7 @@ pub async fn insert_ping_query(
memory,
job_isolation.as_deref(),
native_mode,
uses_batch_http_pull,
)
.execute(db)
.await?;
@@ -1485,12 +1551,13 @@ pub async fn update_worker_ping_main_loop_query(
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
native_mode: bool,
uses_batch_http_pull: bool,
db: &DB,
) -> anyhow::Result<()> {
timeout(Duration::from_secs(10), sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12 WHERE worker = $6",
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, uses_batch_http_pull = $13 WHERE worker = $6",
jobs_executed,
tags,
occupancy_rate,
@@ -1503,6 +1570,7 @@ pub async fn update_worker_ping_main_loop_query(
occupancy_rate_5m,
occupancy_rate_30m,
native_mode,
uses_batch_http_pull,
)
.execute(db))
.await??;

View File

@@ -146,7 +146,6 @@ pub enum ObjectType {
Trigger,
Settings,
Key,
WorkspaceDependencies,
}
#[derive(Serialize, Deserialize, Debug)]

View File

@@ -12,10 +12,6 @@ use windmill_common::{scripts::ScriptHash, DB};
pub mod git_sync_ee;
pub mod git_sync_oss;
#[cfg(feature = "private")]
pub use git_sync_ee::{handle_deployment_metadata, handle_fork_branch_creation};
#[cfg(not(feature = "private"))]
pub use git_sync_oss::{handle_deployment_metadata, handle_fork_branch_creation};
#[derive(Clone, Debug)]
@@ -42,7 +38,6 @@ pub enum DeployedObject {
EmailTrigger { path: String, parent_path: Option<String> },
Settings { setting_type: String },
Key { key_type: String },
WorkspaceDependencies { path: String },
}
impl DeployedObject {
@@ -70,7 +65,6 @@ impl DeployedObject {
DeployedObject::EmailTrigger { path, .. } => path.to_owned(),
DeployedObject::Settings { .. } => "settings.yaml".to_string(),
DeployedObject::Key { .. } => "encryption_key.yaml".to_string(),
DeployedObject::WorkspaceDependencies { path, .. } => path.to_owned(),
}
}
@@ -80,8 +74,7 @@ impl DeployedObject {
| Self::Group { .. }
| Self::ResourceType { .. }
| Self::Settings { .. }
| Self::Key { .. }
| Self::WorkspaceDependencies { .. } => true,
| Self::Key { .. } => true,
_ => false,
}
}
@@ -110,7 +103,6 @@ impl DeployedObject {
DeployedObject::EmailTrigger { parent_path, .. } => parent_path.to_owned(),
DeployedObject::Settings { .. } => None,
DeployedObject::Key { .. } => None,
DeployedObject::WorkspaceDependencies { .. } => None,
}
}
@@ -138,244 +130,6 @@ impl DeployedObject {
DeployedObject::EmailTrigger { .. } => "email_trigger",
DeployedObject::Settings { .. } => "settings",
DeployedObject::Key { .. } => "key",
DeployedObject::WorkspaceDependencies { .. } => "workspace_dependencies",
}
.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
use windmill_common::scripts::ScriptHash;
// --- DeployedObject::get_path tests ---
#[test]
fn test_get_path_script() {
let obj = DeployedObject::Script {
hash: ScriptHash(123),
path: "f/folder/script".to_string(),
parent_path: None,
};
assert_eq!(obj.get_path(), "f/folder/script");
}
#[test]
fn test_get_path_flow() {
let obj = DeployedObject::Flow {
path: "f/folder/flow".to_string(),
parent_path: Some("f/folder/old_flow".to_string()),
version: 1,
};
assert_eq!(obj.get_path(), "f/folder/flow");
}
#[test]
fn test_get_path_user() {
let obj = DeployedObject::User { email: "user@example.com".to_string() };
assert_eq!(obj.get_path(), "users/user@example.com");
}
#[test]
fn test_get_path_group() {
let obj = DeployedObject::Group { name: "admins".to_string() };
assert_eq!(obj.get_path(), "groups/admins");
}
#[test]
fn test_get_path_settings() {
let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() };
assert_eq!(obj.get_path(), "settings.yaml");
}
#[test]
fn test_get_path_key() {
let obj = DeployedObject::Key { key_type: "encryption".to_string() };
assert_eq!(obj.get_path(), "encryption_key.yaml");
}
#[test]
fn test_get_path_workspace_dependencies() {
let obj = DeployedObject::WorkspaceDependencies {
path: "workspace-dependencies/python".to_string(),
};
assert_eq!(obj.get_path(), "workspace-dependencies/python");
}
// --- DeployedObject::get_ignore_regex_filter tests ---
#[test]
fn test_ignore_regex_filter_user() {
let obj = DeployedObject::User { email: "user@example.com".to_string() };
assert!(obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_group() {
let obj = DeployedObject::Group { name: "admins".to_string() };
assert!(obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_resource_type() {
let obj = DeployedObject::ResourceType { path: "postgresql".to_string() };
assert!(obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_settings() {
let obj = DeployedObject::Settings { setting_type: "error_handler".to_string() };
assert!(obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_key() {
let obj = DeployedObject::Key { key_type: "encryption".to_string() };
assert!(obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_workspace_dependencies() {
let obj = DeployedObject::WorkspaceDependencies {
path: "workspace-dependencies/python".to_string(),
};
assert!(obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_script() {
let obj = DeployedObject::Script {
hash: ScriptHash(123),
path: "f/folder/script".to_string(),
parent_path: None,
};
assert!(!obj.get_ignore_regex_filter());
}
#[test]
fn test_ignore_regex_filter_flow() {
let obj = DeployedObject::Flow {
path: "f/folder/flow".to_string(),
parent_path: None,
version: 1,
};
assert!(!obj.get_ignore_regex_filter());
}
// --- DeployedObject::get_parent_path tests ---
#[test]
fn test_get_parent_path_script_with_parent() {
let obj = DeployedObject::Script {
hash: ScriptHash(123),
path: "f/folder/script".to_string(),
parent_path: Some("f/folder/old_script".to_string()),
};
assert_eq!(obj.get_parent_path(), Some("f/folder/old_script".to_string()));
}
#[test]
fn test_get_parent_path_script_without_parent() {
let obj = DeployedObject::Script {
hash: ScriptHash(123),
path: "f/folder/script".to_string(),
parent_path: None,
};
assert_eq!(obj.get_parent_path(), None);
}
#[test]
fn test_get_parent_path_folder() {
let obj = DeployedObject::Folder { path: "f/folder".to_string() };
assert_eq!(obj.get_parent_path(), None);
}
#[test]
fn test_get_parent_path_workspace_dependencies() {
let obj = DeployedObject::WorkspaceDependencies {
path: "workspace-dependencies/python".to_string(),
};
assert_eq!(obj.get_parent_path(), None);
}
// --- DeployedObject::get_kind tests ---
#[test]
fn test_get_kind_script() {
let obj = DeployedObject::Script {
hash: ScriptHash(123),
path: "test".to_string(),
parent_path: None,
};
assert_eq!(obj.get_kind(), "script");
}
#[test]
fn test_get_kind_flow() {
let obj = DeployedObject::Flow {
path: "test".to_string(),
parent_path: None,
version: 1,
};
assert_eq!(obj.get_kind(), "flow");
}
#[test]
fn test_get_kind_app() {
let obj = DeployedObject::App {
path: "test".to_string(),
version: 1,
parent_path: None,
};
assert_eq!(obj.get_kind(), "app");
}
#[test]
fn test_get_kind_workspace_dependencies() {
let obj = DeployedObject::WorkspaceDependencies {
path: "workspace-dependencies/python".to_string(),
};
assert_eq!(obj.get_kind(), "workspace_dependencies");
}
#[test]
fn test_get_kind_all_triggers() {
assert_eq!(
DeployedObject::HttpTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"http_trigger"
);
assert_eq!(
DeployedObject::WebsocketTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"websocket_trigger"
);
assert_eq!(
DeployedObject::KafkaTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"kafka_trigger"
);
assert_eq!(
DeployedObject::NatsTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"nats_trigger"
);
assert_eq!(
DeployedObject::PostgresTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"postgres_trigger"
);
assert_eq!(
DeployedObject::MqttTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"mqtt_trigger"
);
assert_eq!(
DeployedObject::SqsTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"sqs_trigger"
);
assert_eq!(
DeployedObject::GcpTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"gcp_trigger"
);
assert_eq!(
DeployedObject::EmailTrigger { path: "t".to_string(), parent_path: None }.get_kind(),
"email_trigger"
);
}.to_string()
}
}

View File

@@ -152,21 +152,6 @@ pub fn try_exact_property_access(
None
}
/// JS runtime properties (not methods) that cannot be resolved by PostgreSQL's
/// #> JSON path operator. Function calls like .map(...) already don't match the
/// RE_FULL regex due to parentheses, so only property accesses need listing here.
const JS_ONLY_PROPERTIES: &[&str] = &["length"];
fn ends_with_js_only_property(rest: Option<&str>) -> bool {
match rest {
None => false,
Some(rest) => {
let last_segment = rest.rsplit('.').next().unwrap_or("");
JS_ONLY_PROPERTIES.contains(&last_segment)
}
}
}
pub async fn handle_full_regex(
expr: &str,
authed_client: &AuthedClient,
@@ -177,13 +162,6 @@ pub async fn handle_full_regex(
let obj_key = captures.get(2).unwrap().as_str();
let idx_o = captures.get(3).map(|y| y.as_str());
let rest = captures.get(4).map(|y| y.as_str());
// Skip the SQL fast path when the expression accesses a JS runtime
// property (e.g. .length) that the PostgreSQL #> operator can't resolve.
if ends_with_js_only_property(rest) {
return None;
}
let query = if let Some(idx) = idx_o {
match rest {
Some(rest) => Some(format!("{}{}", idx, rest)),

View File

@@ -94,7 +94,7 @@ pub struct OAuthConfig {
}
/// OAuth client credentials
#[derive(Clone, Serialize, Deserialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct OAuthClient {
#[serde(default = "empty_string")]
pub id: String,
@@ -110,21 +110,6 @@ pub struct OAuthClient {
pub grant_types: Vec<String>,
}
impl std::fmt::Debug for OAuthClient {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OAuthClient")
.field("id", &self.id)
.field("secret", &"***")
.field("display_name", &self.display_name)
.field("allowed_domains", &self.allowed_domains)
.field("connect_config", &self.connect_config)
.field("login_config", &self.login_config)
.field("tenant", &self.tenant)
.field("grant_types", &self.grant_types)
.finish()
}
}
fn empty_string() -> String {
"".to_string()
}
@@ -623,18 +608,7 @@ pub async fn refresh_token<'c>(
.await?;
let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?;
refresh_token_for_account(
tx,
path,
w_id,
id,
db,
account,
oauth_clients,
http_client,
connect_configs_json,
)
.await
refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await
}
/// Refresh an OAuth token given pre-fetched account info (no additional SELECT).

View File

@@ -818,7 +818,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
flow_is_done: bool,
duration: Option<i64>,
from_cache: bool,
) -> Result<(Uuid, i64, Option<serde_json::Value>), Error> {
) -> Result<(Uuid, i64), Error> {
// tracing::error!("Start");
// let start = tokio::time::Instant::now();
@@ -830,7 +830,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
}
let result_columns = result_columns.as_ref();
let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| {
let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| {
commit_completed_job(
db,
completed_job,
@@ -866,7 +866,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout
if let Some(job_id) = opt_uuid {
return Ok((job_id, duration, None));
return Ok((job_id, duration));
}
#[cfg(feature = "cloud")]
@@ -887,7 +887,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// tracing::error!("4 {:?}", start.elapsed());
Ok((completed_job.id, duration, wac_job_ids))
Ok((completed_job.id, duration))
}
async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
@@ -902,7 +902,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
flow_is_done: bool,
duration: Option<i64>,
from_cache: bool,
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool, Option<serde_json::Value>)> {
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool)> {
// let start = std::time::Instant::now();
let mut tx = db.begin().warn_after_seconds(10).await?;
@@ -1003,31 +1003,25 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?;
}
let mut wac_job_ids: Option<serde_json::Value> = None;
if !completed_job.is_flow_step() {
if let Some(parent_job) = completed_job.parent_job {
// Only update WAC parents (v1 or v2). The WHERE condition skips
// non-WAC parents entirely (error handlers, run_script children, etc.).
// Also returns pending_steps.job_ids so WAC v2 child completion
// doesn't need a separate read.
let row = sqlx::query_scalar!(
r#"UPDATE v2_job_status SET
let _ = sqlx::query_scalar!(
"UPDATE v2_job_status SET
workflow_as_code_status = jsonb_set(
jsonb_set(
workflow_as_code_status,
COALESCE(workflow_as_code_status, '{}'::jsonb),
array[$1],
COALESCE(workflow_as_code_status->$1, '{}'::jsonb)
),
array[$1, 'duration_ms'],
to_jsonb($2::bigint)
)
WHERE id = $3 AND workflow_as_code_status IS NOT NULL
RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#,
WHERE id = $3",
&completed_job.id.to_string(),
duration,
parent_job
)
.fetch_optional(&mut *tx)
.execute(&mut *tx)
.warn_after_seconds(10)
.await
.inspect_err(|e| {
@@ -1035,10 +1029,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
"Could not update parent job `duration_ms` in workflow as code status: {}",
e,
)
})
.ok()
.flatten();
wac_job_ids = row.flatten();
});
}
}
// tracing::error!("Added completed job {:#?}", queued_job);
@@ -1259,14 +1250,14 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
completed_job.id
);
// tracing::info!("completed job: {:?}", start.elapsed().as_micros());
Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids))
Ok((None, duration, _skip_downstream_error_handlers))
}
async fn check_result_size<T: ValidableJson>(
db: &Pool<Postgres>,
queued_job: &MiniCompletedJob,
result: Json<&T>,
) -> Option<Result<(Option<Uuid>, i64, bool, Option<serde_json::Value>), Error>> {
) -> Option<Result<(Option<Uuid>, i64, bool), Error>> {
let result_size = result.size() / 1024 / 1024;
if result_size > 2 {
if result_size > *MAX_RESULT_SIZE_MB {
@@ -2951,13 +2942,7 @@ impl PulledJobResult {
.and_then(|x| x.get("triggered_by_relative_import"))
.is_some();
let has_debounce_args = debounce_args_to_accumulate
.as_ref()
.map_or(false, |v| !v.is_empty());
if (is_djob_to_debounce
|| debounce_delay_s.filter(|x| *x > 0).is_some()
|| has_debounce_args)
if (is_djob_to_debounce || debounce_delay_s.filter(|x| *x > 0).is_some())
&& MIN_VERSION_SUPPORTS_DEBOUNCING.met().await
&& !*WMDEBUG_NO_DEBOUNCING
{
@@ -3046,18 +3031,11 @@ impl PulledJobResult {
let new_value = to_raw_value(&accumulated_arg);
let original_value = j
.args
.as_ref()
.and_then(|a| a.get(arg_name_to_accumulate))
.map(|v| v.get().to_string())
.unwrap_or_else(|| "null".to_string());
append_logs(
&j_id,
&j.workspace_id,
format!(
"Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n",
"Substituting `{arg_name_to_accumulate}` with: {}\n\n",
&new_value
),
&(db.into()),
@@ -3068,18 +3046,6 @@ impl PulledJobResult {
.get_or_insert(Json(Default::default()))
.as_mut()
.insert(arg_name_to_accumulate.to_owned(), new_value);
// Persist accumulated args to v2_job so that flow steps
// re-reading from the DB (via get_mini_pulled_job) see them
if let Some(ref args) = j.args {
sqlx::query!(
"UPDATE v2_job SET args = $2 WHERE id = $1",
j_id,
args as &Json<HashMap<String, Box<RawValue>>>,
)
.execute(db)
.await?;
}
}
// Handle dependency job debouncing cleanup when a job is pulled for execution
@@ -3468,6 +3434,38 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
Ok(job_and_suspended)
}
/// Batch-pull up to `limit` jobs in a single query, marking them all as running.
/// The caller controls which tags are queried, so flow/dependency jobs are never
/// pulled (they use distinct tags like "flow" / "dependency").
pub async fn batch_pull(
db: &Pool<Postgres>,
worker_name: &str,
tags: &[String],
limit: u32,
) -> windmill_common::error::Result<Vec<PulledJob>> {
use windmill_common::worker::make_batch_pull_query;
if limit == 0 || tags.is_empty() {
return Ok(vec![]);
}
let query = make_batch_pull_query(tags, limit);
let jobs: Vec<PulledJob> = timeout(
Duration::from_secs(15),
sqlx::query_as::<_, PulledJob>(&query)
.bind(worker_name)
.fetch_all(db),
)
.await
.map_err(|_| {
windmill_common::error::Error::internal_err(
"batch_pull query timed out after 15s".to_string(),
)
})??;
Ok(jobs)
}
pub async fn custom_concurrency_key(
db: &Pool<Postgres>,
job_id: &Uuid,
@@ -3628,8 +3626,7 @@ pub async fn check_debouncing_within_limits(
);
if allowed_amount
.filter(|&a| a > 0)
.map(|allowed_amount| current_amount + 1 >= allowed_amount)
.map(|allowed_amount| current_amount > allowed_amount)
.unwrap_or_default()
&& no_legacy_compat
{
@@ -5408,15 +5405,7 @@ async fn push_inner<'c, 'd>(
language
.as_ref()
.map(|x| {
let tag_lang = if x == &ScriptLang::Bunnative {
if job_kind == JobKind::Dependencies {
ScriptLang::Bun.as_str()
} else {
ScriptLang::Nativets.as_str()
}
} else {
x.as_str()
};
let tag_lang = x.as_worker_tag(job_kind == JobKind::Dependencies);
if per_workspace {
format!("{}-{}", tag_lang, workspace_id)
} else {
@@ -5513,11 +5502,10 @@ async fn push_inner<'c, 'd>(
&mut *tx,
)
.await
.map_err(|e| match e {
Error::NotFound(_) => e,
_ => Error::internal_err(format!(
.map_err(|e| {
Error::internal_err(format!(
"Could not get permissions directly for job {job_id}: {e:#}"
)),
))
})?
}
};

File diff suppressed because it is too large Load Diff

View File

@@ -56,7 +56,6 @@ impl PrewarmedIsolate {
js_code: String,
ann: NativeAnnotation,
arg_names: Vec<String>,
entrypoint: Option<String>,
) -> Self {
let (args_tx, args_rx) = tokio::sync::oneshot::channel::<String>();
let (result_tx, result_rx) = tokio::sync::oneshot::channel::<PrewarmedResult>();
@@ -99,7 +98,7 @@ impl PrewarmedIsolate {
});
let exec_result = tokio::select! {
r = execute_main(&mut js_runtime, entrypoint.as_deref(), false, None) => r,
r = execute_main(&mut js_runtime, None, false, None) => r,
_ = memory_limit_rx.recv() => {
Err(ExecuteError::Script("Memory limit reached, killing isolate".to_string()))
}

View File

@@ -10,8 +10,8 @@ path = "src/lib.rs"
[features]
default = []
private = ["windmill-api/private"]
enterprise = ["windmill-api/enterprise"]
private = []
enterprise = []
python = ["windmill-common/python"]
deno_core = ["dep:windmill-runtime-nativets"]
agent_worker_server = ["dep:windmill-api-agent-workers"]

Some files were not shown because too many files have changed in this diff Show More