Compare commits

..

1 Commits

Author SHA1 Message Date
Ruben Fiszel
56b9bc64f7 fix: add 5s pool acquire_timeout to prevent connection starvation cascade
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-11 22:47:51 +00:00
202 changed files with 3664 additions and 9600 deletions

View File

@@ -119,18 +119,6 @@ jobs:
with:
cache-workspaces: backend
toolchain: 1.93.0
- name: Fix stale v8 build cache
working-directory: ./backend
run: |
# Cargo cache may preserve v8 build fingerprints without the actual
# librusty_v8.a library. Since fingerprints look valid, cargo skips
# build.rs re-run, causing "could not find native static library rusty_v8".
for profile in debug release; do
if [ -d "target/$profile/.fingerprint" ] && [ ! -f "target/$profile/gn_out/obj/librusty_v8.a" ]; then
echo "Cleaning stale v8 build artifacts in target/$profile"
rm -rf "target/$profile/build/v8-"* "target/$profile/.fingerprint/v8-"*
fi
done
- name: cargo check
timeout-minutes: 16
working-directory: ./backend

View File

@@ -89,18 +89,6 @@ jobs:
with:
cache-workspaces: backend
toolchain: 1.93.0
- name: Fix stale v8 build cache
working-directory: ./backend
run: |
# Cargo cache may preserve v8 build fingerprints without the actual
# librusty_v8.a library. Since fingerprints look valid, cargo skips
# build.rs re-run, causing "could not find native static library rusty_v8".
for profile in debug release; do
if [ -d "target/$profile/.fingerprint" ] && [ ! -f "target/$profile/gn_out/obj/librusty_v8.a" ]; then
echo "Cleaning stale v8 build artifacts in target/$profile"
rm -rf "target/$profile/build/v8-"* "target/$profile/.fingerprint/v8-"*
fi
done
- name: Read EE repo commit hash
run: |
echo "ee_repo_ref=$(cat ./ee-repo-ref.txt)" >> "$GITHUB_ENV"

View File

@@ -1,37 +0,0 @@
name: Check system prompts freshness
on:
push:
paths:
- "system_prompts/**"
- "typescript-client/**"
- "python-client/wmill/wmill/client.py"
- "openflow.openapi.yaml"
- "backend/windmill-api/openapi.yaml"
- "cli/src/main.ts"
- "cli/src/commands/**"
pull_request:
paths:
- "system_prompts/**"
- "typescript-client/**"
- "python-client/wmill/wmill/client.py"
- "openflow.openapi.yaml"
- "backend/windmill-api/openapi.yaml"
- "cli/src/main.ts"
- "cli/src/commands/**"
jobs:
check-freshness:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: pip install pyyaml
- name: Check auto-generated files are up-to-date
run: bash system_prompts/check-freshness.sh

View File

@@ -1,209 +0,0 @@
name: Git Sync Integration Tests
on:
workflow_dispatch:
push:
branches: [main]
paths:
- "backend/windmill-git-sync/**"
- "backend/windmill-api-integration-tests/tests/git_sync*"
- "backend/ee-repo-ref.txt"
- "integration_tests/test/git_sync_test.py"
- ".github/workflows/git-sync-test.yml"
pull_request:
types: [opened, synchronize, reopened]
paths:
- "backend/windmill-git-sync/**"
- "backend/windmill-api-integration-tests/tests/git_sync*"
- "backend/ee-repo-ref.txt"
- "integration_tests/test/git_sync_test.py"
- ".github/workflows/git-sync-test.yml"
concurrency:
group: git-sync-test-${{ github.ref }}
cancel-in-progress: true
jobs:
check-relevance:
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.check.outputs.should_run }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check if git sync related files changed
id: check
env:
WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
BASE=${{ github.event.pull_request.base.sha }}
else
BASE=${{ github.event.before }}
fi
CHANGED_FILES=$(git diff --name-only "$BASE"..HEAD 2>/dev/null || echo "")
echo "Changed files:"
echo "$CHANGED_FILES"
# Direct git sync file changes — always relevant
if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Relevant: direct git sync file changes"
exit 0
fi
# If ee-repo-ref.txt changed, check if the EE diff touches windmill-git-sync/
if echo "$CHANGED_FILES" | grep -q '^backend/ee-repo-ref.txt$'; then
NEW_REF=$(cat backend/ee-repo-ref.txt)
OLD_REF=$(git show "$BASE:backend/ee-repo-ref.txt" 2>/dev/null || echo "")
if [ -n "$OLD_REF" ] && [ "$OLD_REF" != "$NEW_REF" ]; then
# Clone EE repo and check diff
git clone --bare "https://x-access-token:${WINDMILL_EE_PRIVATE_ACCESS}@github.com/windmill-labs/windmill-ee-private.git" /tmp/ee-repo 2>/dev/null
EE_CHANGED=$(git -C /tmp/ee-repo diff --name-only "$OLD_REF".."$NEW_REF" 2>/dev/null || echo "")
echo "EE changed files:"
echo "$EE_CHANGED"
if echo "$EE_CHANGED" | grep -q '^windmill-git-sync/'; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Relevant: EE git sync files changed"
exit 0
fi
fi
fi
echo "should_run=false" >> "$GITHUB_OUTPUT"
echo "No git sync relevant changes detected, skipping tests"
git_sync_e2e:
needs: [check-relevance]
if: needs.check-relevance.outputs.should_run == 'true'
runs-on: ubicloud-standard-16
services:
postgres:
image: postgres:14
ports:
- 5432:5432
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
fetch-depth: 0
- name: Read EE repo commit hash
run: |
echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
- uses: actions/checkout@v4
with:
repository: windmill-labs/windmill-ee-private
path: ./windmill-ee-private
ref: ${{ env.ee_repo_ref }}
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
- name: Substitute EE code
run: |
cd backend && ./substitute_ee_code.sh --copy --dir ./windmill-ee-private
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.93.0
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.10
- uses: denoland/setup-deno@v2
with:
deno-version: v2.x
- uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install wmill CLI
run: |
cd cli && bash gen_wm_client.sh && bun install
mkdir -p "$HOME/.local/bin"
printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill"
chmod +x "$HOME/.local/bin/wmill"
echo "$HOME/.local/bin" >> $GITHUB_PATH
- name: Build Windmill
working-directory: ./backend
env:
SQLX_OFFLINE: true
CARGO_BUILD_JOBS: 12
RUSTFLAGS: ""
run: |
cargo build --features enterprise,private,license,zip
- name: Start Gitea
run: |
docker run -d --name gitea \
-e GITEA__database__DB_TYPE=sqlite3 \
-e GITEA__security__INSTALL_LOCK=true \
-e GITEA__server__HTTP_PORT=3000 \
-e GITEA__server__ROOT_URL=http://localhost:3000 \
-e GITEA__service__DISABLE_REGISTRATION=false \
-p 3000:3000 \
gitea/gitea:1.22-rootless
echo "Waiting for Gitea to be ready..."
for i in $(seq 1 30); do
if curl -sf http://localhost:3000/api/v1/version > /dev/null 2>&1; then
echo "Gitea is ready"
break
fi
sleep 2
done
curl -sf http://localhost:3000/api/v1/version > /dev/null || { echo "Gitea failed to start"; exit 1; }
- name: Start Windmill
working-directory: ./backend
env:
DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
DENO_PATH: deno
BUN_PATH: bun
NODE_BIN_PATH: node
run: |
./target/debug/windmill &
echo "Waiting for Windmill to be ready..."
for i in $(seq 1 60); do
if curl -sf http://localhost:8000/api/version > /dev/null 2>&1; then
echo "Windmill is ready"
break
fi
sleep 2
done
curl -sf http://localhost:8000/api/version > /dev/null || { echo "Windmill failed to start"; exit 1; }
- name: Run git sync E2E tests
timeout-minutes: 10
env:
GITEA_DOCKER_URL: http://localhost:3000
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
run: |
python3 -m venv .venv
.venv/bin/pip install -r integration_tests/requirements.txt
cd integration_tests && ../.venv/bin/python -m unittest -v test.git_sync_test
- name: Archive logs
uses: actions/upload-artifact@v4
if: always()
with:
name: Git Sync Integration Tests Logs
path: |
integration_tests/logs

1
.gitignore vendored
View File

@@ -27,4 +27,3 @@ typescript-client/node_modules
frontend/.svelte-kit
backend/chrome_profiler.json
.fast-check/
__pycache__/

View File

@@ -9,14 +9,14 @@ workspace:
startupEnvs:
CARGO_FEATURES: "quickjs"
WM_CLONE_DB: false
USE_RUST_PLUGIN: false
lifecycleHooks:
postCreate: bash ./scripts/post-create.sh
preRemove: bash ./scripts/pre-remove.sh
auto_name:
provider: claude
model: haiku
model: gemini-2.5-flash-lite
# Each service defines a port env var that webmux injects into pane and agent
# process environments when creating a worktree. Ports are auto-assigned:
@@ -32,7 +32,7 @@ services:
portStep: 10
profiles:
full:
default:
runtime: host
yolo: true
envPassthrough: []
@@ -44,9 +44,7 @@ profiles:
- Pane 2: frontend (npm run dev)
To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend).
When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}.
To connect to the database, use this connection string: ${DATABASE_URL}
Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check.
IMPORTANT: Read docs/autonomous-mode.md before starting any work.
panes:
- id: agent
kind: agent
@@ -54,23 +52,14 @@ profiles:
- id: backend
kind: command
split: right
command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/backend" && PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}"
- id: frontend
kind: command
split: bottom
command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
agentOnly:
runtime: host
yolo: true
envPassthrough: []
systemPrompt: >
IMPORTANT: Read docs/autonomous-mode.md before starting any work.
panes:
- id: agent
kind: agent
focus: true
command: ROOT="$(git rev-parse --show-toplevel)"; [ -f "$ROOT/.env.local" ] && source "$ROOT/.env.local"; cd "$ROOT/frontend" && npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0
integrations:
github:
linkedRepos: []
linear:
enabled: true

View File

@@ -1,27 +1,5 @@
# Changelog
## [1.655.0](https://github.com/windmill-labs/windmill/compare/v1.654.0...v1.655.0) (2026-03-12)
### Features
* add auto_commit option to Kafka triggers with advanced UI badges ([#8317](https://github.com/windmill-labs/windmill/issues/8317)) ([ec20d76](https://github.com/windmill-labs/windmill/commit/ec20d76216492086842c4f5e4e3b36727a5631e9))
* partition audit log table by day with configurable retention ([#8292](https://github.com/windmill-labs/windmill/issues/8292)) ([2aef01d](https://github.com/windmill-labs/windmill/commit/2aef01d18c0723aedcc626f4f3991195620774ab))
* support minimal telemetry mode ([#8243](https://github.com/windmill-labs/windmill/issues/8243)) ([fe1519f](https://github.com/windmill-labs/windmill/commit/fe1519f1284aadd67d5dce46cf0cb52ab351f789))
### Bug Fixes
* **cli:** instruct agent to tell user about generate-metadata and sync push instead of running them ([#8318](https://github.com/windmill-labs/windmill/issues/8318)) ([7fb729c](https://github.com/windmill-labs/windmill/commit/7fb729cc8483a2e6966a8e8995678929f4d451a0))
* fix saved inputs popover infinite loop ([#8311](https://github.com/windmill-labs/windmill/issues/8311)) ([425a75e](https://github.com/windmill-labs/windmill/commit/425a75e030b15fe65676169f9069fbb7da19828e))
* native mode now properly sets DB pool size and sleep queue ([#8332](https://github.com/windmill-labs/windmill/issues/8332)) ([d8b4132](https://github.com/windmill-labs/windmill/commit/d8b4132b9ae90af759c6655f4f69479f6738e60a))
* prevent zombie jobs from looping forever ([#8313](https://github.com/windmill-labs/windmill/issues/8313)) ([48bc3e2](https://github.com/windmill-labs/windmill/commit/48bc3e244558dccb1f08f455b299600861788b0d))
* set min_connections(0) to prevent sqlx pool spin loop ([#8334](https://github.com/windmill-labs/windmill/issues/8334)) ([bf4340f](https://github.com/windmill-labs/windmill/commit/bf4340f40c1eb9cacee4c32e07ba44f2c92bf7c4))
* show diff editor content for resources without a language ([#8331](https://github.com/windmill-labs/windmill/issues/8331)) ([cbc7e78](https://github.com/windmill-labs/windmill/commit/cbc7e78f8a60bff1d8730a6183cdbc9125d8e2b1))
* skip python preinstall on native workers ([#8329](https://github.com/windmill-labs/windmill/issues/8329)) ([4306c9e](https://github.com/windmill-labs/windmill/commit/4306c9e4fef317e298a76924edb4f20aa7ced105))
* skip token expiry notifications for debugger and mcp-oauth tokens ([#8316](https://github.com/windmill-labs/windmill/issues/8316)) ([8667329](https://github.com/windmill-labs/windmill/commit/86673291100fd16aaf216ed33ca9b648b8a2b7a5))
* use !inline ref for scripts inside flows (preproc, error, ai tool) ([#8319](https://github.com/windmill-labs/windmill/issues/8319)) ([ca8a627](https://github.com/windmill-labs/windmill/commit/ca8a6274bc81ad49fa0c6166694ae4d65a4048cb))
## [1.654.0](https://github.com/windmill-labs/windmill/compare/v1.653.0...v1.654.0) (2026-03-10)

View File

@@ -14,7 +14,7 @@ Open-source platform for internal tools, workflows, API integrations, background
- **Validation**: `docs/validation.md` — what checks to run based on what you changed
- **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow
- **Backend patterns**: use the `rust-backend` skill when writing Rust code
- **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill.
- **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code
- **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc`
- **Brand/UI guidelines**: `frontend/brand-guidelines.md`
@@ -33,7 +33,6 @@ Open-source platform for internal tools, workflows, API integrations, background
Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state.
**Bad:**
```svelte
let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props()
```
@@ -41,7 +40,6 @@ let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props()
**Correct alternatives:**
1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site:
```svelte
let { my_prop = $bindable() }: { my_prop?: string } = $props()
let effective_value = $derived(my_prop ?? default_value)

View File

@@ -268,11 +268,11 @@ RUN bun install -g windmill-cli \
RUN curl -fsSL https://claude.ai/install.sh | bash \
&& cp /root/.local/share/claude/versions/* /usr/bin/claude
COPY --from=php:8.3.30-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.9.5 /usr/bin/composer /usr/bin/composer
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
# add the docker client to call docker from a worker if enabled
COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
ENV RUSTUP_HOME="/tmp/windmill/cache/rustup"
ENV CARGO_HOME="/tmp/windmill/cache/cargo"

View File

@@ -1,41 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, topic, partition, \"offset\" FROM kafka_pending_commits\n WHERE workspace_id = $1 AND kafka_trigger_path = $2\n ORDER BY id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "topic",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "partition",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "offset",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false
]
},
"hash": "038d2fde90fa9e99e30d15161777fa3ab402e33edfca46daa95b52e525424586"
}

View File

@@ -46,11 +46,11 @@
]
},
"nullable": [
true,
true,
true,
true,
true,
false,
false,
false,
false,
false,
true,
true
]

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n (elem->>'installation_id')::bigint as installation_id,\n elem->>'account_id' as account_id,\n elem->>'github_base_url' as github_base_url\n FROM workspace_settings,\n LATERAL jsonb_array_elements(git_app_installations) AS elem\n WHERE workspace_id = $1\n ",
"query": "\n SELECT\n (elem->>'installation_id')::bigint as installation_id,\n elem->>'account_id' as account_id\n FROM workspace_settings,\n LATERAL jsonb_array_elements(git_app_installations) AS elem\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
@@ -12,11 +12,6 @@
"ordinal": 1,
"name": "account_id",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "github_base_url",
"type_info": "Text"
}
],
"parameters": {
@@ -25,10 +20,9 @@
]
},
"nullable": [
null,
null,
null
]
},
"hash": "ae7adc583cdd3f876164ed60569ed531b05eaa17fccc599306eb1a96a65ee761"
"hash": "0ee14619dd81df460b2b8cc6df2b89646279f77469c35deffca8e17a11d7f6c8"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n auto_offset_reset = $5,\n auto_commit = $6,\n script_path = $7,\n path = $8,\n is_flow = $9,\n edited_by = $10,\n email = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n auto_offset_reset = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,7 +10,6 @@
"VarcharArray",
"JsonbArray",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Bool",
@@ -25,5 +24,5 @@
},
"nullable": []
},
"hash": "072e5ab78f929c6b7264f98c1588cb24cc635836276ee6faa2438f494bfbce04"
"hash": "12631fecee6aa11a45cf5c8d101c0dd8de50ac9b57e68198f637038d344fdd46"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT reset_offset FROM kafka_trigger WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "reset_offset",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "1df610a583e86edb70c374fd66c68554a6a4291426c09dd5b04fd832f9d31208"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM variable WHERE path = $1 AND workspace_id = $2 RETURNING path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "3317484a9c09c07c2c9db9debaecc4a4d518093ab48e79365dbb808068e0b8ff"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT (config->>'native_mode')::boolean FROM config WHERE name = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "bool",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "36b95bc7956eb7bba7cd6fa9cd829980a0bf4970b919cabad1daab16627404fc"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id, script_path,\n is_flow, workspace_id, edited_by, email, auto_commit\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"VarcharArray",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "45fc21026fa76e5d69f00a68a7be81abb3ec627578f2d14f0ce33896dc6ab4cf"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET git_app_installations = (\n SELECT jsonb_agg(\n CASE\n WHEN (elem->>'installation_id')::bigint = $2 THEN $1::jsonb\n ELSE elem\n END\n )\n FROM jsonb_array_elements(git_app_installations) AS elem\n )\n WHERE workspace_id = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Int8",
"Text"
]
},
"nullable": []
},
"hash": "48b394bd9ca63d33a7ea97113b0096bd0777da52c05e23262572089e0c3c6c46"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT auto_commit FROM kafka_trigger WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "auto_commit",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "4b2a29b3ef7ec4802d81ec4b706623b991c938e40d0db25290b03dc0577c2740"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16\n )\n ",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n auto_offset_reset,\n script_path,\n is_flow,\n mode,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -12,7 +12,6 @@
"VarcharArray",
"JsonbArray",
"Varchar",
"Bool",
"Varchar",
"Bool",
{
@@ -36,5 +35,5 @@
},
"nullable": []
},
"hash": "5dd6315ec270c268e905262e4b0a920837354d91a0ae16b1236c1267da71765f"
"hash": "4b5a711986017654bdd495893a16ddc6ab09c98cd8723865cbd341404bc6a02f"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT auto_commit\n FROM kafka_trigger\n WHERE workspace_id = $1 AND path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "auto_commit",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "50807b807bb901a380926798be655c13a18dfd26e237a8218d3006e2898b5aa3"
}

View File

@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT kafka_resource_path, topics, group_id, mode AS \"mode: String\",\n auto_offset_reset, auto_commit, reset_offset\n FROM kafka_trigger\n WHERE workspace_id = $1 AND path = $2\n ",
"query": "\n SELECT kafka_resource_path, topics, group_id, mode AS \"mode: String\"\n FROM kafka_trigger\n WHERE workspace_id = $1 AND path = $2\n ",
"describe": {
"columns": [
{
@@ -33,21 +33,6 @@
}
}
}
},
{
"ordinal": 4,
"name": "auto_offset_reset",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "auto_commit",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "reset_offset",
"type_info": "Bool"
}
],
"parameters": {
@@ -57,14 +42,11 @@
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "4cf4be7a981173d3f242887d9313c7e60d23e9827f23c0de5b546ed56697d54a"
"hash": "7e3bfb33fb771aec39b43a7550091ce7c9b1261b52d10f4a7f3273fed3c916df"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM kafka_pending_commits WHERE id = ANY($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8Array"
]
},
"nullable": []
},
"hash": "80bad96cbec6b5eca57a6380e7515565490a271050dcc4b5aac2b730ae3a55b9"
}

View File

@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n (elem->>'installation_id')::bigint as installation_id,\n elem->>'github_base_url' as github_base_url\n FROM workspace_settings,\n LATERAL jsonb_array_elements(git_app_installations) AS elem\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "installation_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "github_base_url",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
null
]
},
"hash": "ad5fc9212a123a8328397496ef3b5eea1780226698e419ac59ba55012296913d"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET git_app_installations = (\n SELECT jsonb_agg(\n CASE\n WHEN (elem->>'installation_id')::bigint = $2 THEN $1::jsonb\n ELSE elem\n END\n )\n FROM jsonb_array_elements(git_app_installations) AS elem\n )\n WHERE workspace_id = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Int8",
"Text"
]
},
"nullable": []
},
"hash": "b4d48c820bf41619bffa8f62367e98369e1d93514e1618723a34bf96080d4ebc"
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n (elem->>'installation_id')::bigint as installation_id\n FROM workspace_settings,\n LATERAL jsonb_array_elements(git_app_installations) AS elem\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "installation_id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "be00ac55e8668a0ed3befda7d8595c7cda0cba0b119d4fdb8a0dea1b28a1d560"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM resource WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "bf2aeb9a1e649106d2a084c1d628690a44573c1869a206474811215714ba97c2"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE kafka_trigger SET reset_offset = true, server_id = NULL WHERE workspace_id = $1 AND path = $2 RETURNING true",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "c2f38c9e09aac73d10e8f327715927c07832badb2c9145d5996b829163bdf7d9"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE kafka_trigger SET reset_offset = false WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "ef15599f532fab2cbb487542ffec047cf3b7ce22ce868db1b1a63e6c10d0d12b"
}

View File

@@ -1,18 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO kafka_pending_commits (workspace_id, kafka_trigger_path, topic, partition, \"offset\")\n VALUES ($1, $2, $3, $4, $5)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Int4",
"Int8"
]
},
"nullable": []
},
"hash": "f67e5c96eb9cb35953d4c3e83e0fcbb5b647737e0366529a2f418218b1a74679"
}

206
backend/Cargo.lock generated
View File

@@ -9383,9 +9383,9 @@ checksum = "80adb31078122c880307e9cdfd4e3361e6545c319f9b9dcafcb03acd3b51a575"
[[package]]
name = "once_cell"
version = "1.21.4"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "once_cell_polyfill"
@@ -9460,9 +9460,9 @@ dependencies = [
[[package]]
name = "openssl"
version = "0.10.76"
version = "0.10.75"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328"
dependencies = [
"bitflags 2.9.4",
"cfg-if",
@@ -9507,9 +9507,9 @@ dependencies = [
[[package]]
name = "openssl-sys"
version = "0.9.112"
version = "0.9.111"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
dependencies = [
"cc",
"libc",
@@ -10641,9 +10641,9 @@ dependencies = [
[[package]]
name = "quick_cache"
version = "0.6.19"
version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "530e84778a55de0f52645a51d4e3b9554978acd6a1e7cd50b6a6784692b3029e"
checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3"
dependencies = [
"ahash 0.8.12",
"equivalent",
@@ -13854,9 +13854,9 @@ dependencies = [
[[package]]
name = "tempfile"
version = "3.27.0"
version = "3.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
dependencies = [
"fastrand",
"getrandom 0.4.2",
@@ -15741,7 +15741,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15808,7 +15808,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15821,7 +15821,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"argon2",
@@ -15941,7 +15941,6 @@ dependencies = [
"windmill-parser-py",
"windmill-parser-py-imports",
"windmill-parser-sql",
"windmill-parser-sql-asset",
"windmill-parser-ts",
"windmill-queue",
"windmill-store",
@@ -15962,7 +15961,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15985,7 +15984,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -15998,7 +15997,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16024,7 +16023,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -16034,7 +16033,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16051,7 +16050,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"base64 0.22.1",
@@ -16074,7 +16073,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16097,7 +16096,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16113,7 +16112,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16133,7 +16132,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16153,7 +16152,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16167,7 +16166,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16187,7 +16186,6 @@ dependencies = [
"windmill-api-auth",
"windmill-api-client",
"windmill-common",
"windmill-git-sync",
"windmill-native-triggers",
"windmill-test-utils",
"windmill-worker",
@@ -16195,7 +16193,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16220,7 +16218,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"flate2",
@@ -16238,7 +16236,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16259,7 +16257,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16279,7 +16277,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16309,7 +16307,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16336,7 +16334,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"lazy_static",
"serde",
@@ -16348,7 +16346,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"argon2",
"axum 0.7.9",
@@ -16371,7 +16369,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16385,7 +16383,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"axum 0.7.9",
"chrono",
@@ -16416,7 +16414,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"chrono",
"lazy_static",
@@ -16430,7 +16428,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -16449,7 +16447,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"aes-gcm",
"anyhow",
@@ -16548,7 +16546,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -16567,7 +16565,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"regex",
"serde",
@@ -16582,7 +16580,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -16606,7 +16604,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"futures",
@@ -16623,7 +16621,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -16639,7 +16637,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16660,7 +16658,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16691,7 +16689,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-oauth2",
@@ -16715,7 +16713,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-stream",
@@ -16749,7 +16747,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"futures",
@@ -16767,7 +16765,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -16776,7 +16774,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16788,7 +16786,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16800,7 +16798,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"gosyn",
@@ -16812,7 +16810,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16824,7 +16822,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16836,7 +16834,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -16847,7 +16845,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16858,7 +16856,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16866,22 +16864,12 @@ dependencies = [
"rustpython-parser",
"serde_json",
"windmill-parser",
]
[[package]]
name = "windmill-parser-py-asset"
version = "1.655.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"windmill-parser",
"windmill-parser-sql-asset",
"windmill-parser-sql",
]
[[package]]
name = "windmill-parser-py-imports"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16905,7 +16893,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16919,7 +16907,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16936,7 +16924,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16944,17 +16932,6 @@ dependencies = [
"regex-lite",
"serde",
"serde_json",
"windmill-parser",
"windmill-types",
]
[[package]]
name = "windmill-parser-sql-asset"
version = "1.655.0"
dependencies = [
"anyhow",
"serde",
"serde_json",
"sqlparser 0.59.0",
"windmill-parser",
"windmill-types",
@@ -16962,7 +16939,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16976,27 +16953,12 @@ dependencies = [
"triomphe",
"wasm-bindgen",
"windmill-parser",
]
[[package]]
name = "windmill-parser-ts-asset"
version = "1.655.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"swc_ecma_visit",
"triomphe",
"wasm-bindgen",
"windmill-parser",
"windmill-parser-sql-asset",
"windmill-parser-sql",
]
[[package]]
name = "windmill-parser-wac"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -17012,7 +16974,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"serde",
@@ -17023,7 +16985,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -17060,7 +17022,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"const_format",
@@ -17098,7 +17060,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -17109,7 +17071,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -17138,7 +17100,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"axum 0.7.9",
@@ -17161,7 +17123,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17194,7 +17156,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17214,7 +17176,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17248,7 +17210,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17283,7 +17245,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17306,7 +17268,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17330,7 +17292,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-nats",
@@ -17354,7 +17316,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17389,7 +17351,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17417,7 +17379,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-trait",
@@ -17440,7 +17402,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"bitflags 2.9.4",
@@ -17458,7 +17420,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -17564,7 +17526,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.655.0"
version = "1.654.0"
dependencies = [
"bytes",
"futures",

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.655.0"
version = "1.654.0"
authors.workspace = true
edition.workspace = true
@@ -59,7 +59,6 @@ members = [
"./windmill-oauth",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-ts-asset",
"./parsers/windmill-parser-go",
"./parsers/windmill-parser-rust",
"./parsers/windmill-parser-csharp",
@@ -68,11 +67,8 @@ members = [
"./parsers/windmill-parser-ruby",
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-asset",
"./parsers/windmill-parser-py-imports",
"./parsers/windmill-parser-wac",
"./parsers/windmill-parser-sql",
"./parsers/windmill-parser-sql-asset",
"./parsers/windmill-sql-datatype-parser-wasm",
"./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu",
"./windmill-worker-volumes",
@@ -82,7 +78,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.655.0"
version = "1.654.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -324,9 +320,7 @@ windmill-api-workers = { path = "./windmill-api-workers" }
windmill-store = { path = "./windmill-store" }
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-ts-asset = { path = "./parsers/windmill-parser-ts-asset" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
windmill-parser-py-asset = { path = "./parsers/windmill-parser-py-asset" }
windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" }
windmill-parser-go = { path = "./parsers/windmill-parser-go" }
windmill-parser-rust = { path = "./parsers/windmill-parser-rust" }
@@ -337,7 +331,6 @@ windmill-parser-ruby = { path = "./parsers/windmill-parser-ruby" }
windmill-parser-nu = { path = "./parsers/windmill-parser-nu" }
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
windmill-parser-sql-asset = { path = "./parsers/windmill-parser-sql-asset" }
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-parser-wac = { path = "./parsers/windmill-parser-wac" }

View File

@@ -1 +1 @@
c74c86b78a66b976fd9968b21f77903723e668ec
cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64

View File

@@ -1,2 +0,0 @@
DROP TABLE IF EXISTS kafka_pending_commits;
ALTER TABLE kafka_trigger DROP COLUMN auto_commit;

View File

@@ -1,14 +0,0 @@
ALTER TABLE kafka_trigger ADD COLUMN auto_commit BOOLEAN NOT NULL DEFAULT TRUE;
CREATE TABLE kafka_pending_commits (
id BIGSERIAL PRIMARY KEY,
workspace_id VARCHAR(50) NOT NULL,
kafka_trigger_path VARCHAR(255) NOT NULL,
topic VARCHAR(255) NOT NULL,
partition INTEGER NOT NULL,
"offset" BIGINT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
FOREIGN KEY (workspace_id, kafka_trigger_path) REFERENCES kafka_trigger(workspace_id, path) ON DELETE CASCADE
);
CREATE INDEX idx_kafka_pending_commits_trigger ON kafka_pending_commits (workspace_id, kafka_trigger_path);

View File

@@ -1,16 +0,0 @@
[package]
name = "windmill-parser-py-asset"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_py_asset"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
anyhow.workspace = true

View File

@@ -10,6 +10,7 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
rustpython-parser.workspace = true
itertools.workspace = true
serde_json.workspace = true

View File

@@ -215,7 +215,7 @@ impl AssetsFinder {
_ => return Err(()),
};
// We use the SQL parser to detect RW, specific tables, etc.
let sql_assets = windmill_parser_sql_asset::parse_wmill_sdk_sql_assets(
let sql_assets = windmill_parser_sql::parse_wmill_sdk_sql_assets(
*kind,
path,
schema.as_deref(),

View File

@@ -21,8 +21,11 @@ use rustpython_parser::{
Parse,
};
pub mod asset_parser;
pub mod pydantic_parser;
pub use asset_parser::parse_assets;
const FUNCTION_CALL: &str = "<function call>";
/// Cheap string-based check to see if code might contain Pydantic models or dataclasses.

View File

@@ -1,17 +0,0 @@
[package]
name = "windmill-parser-sql-asset"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_sql_asset"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-types.workspace = true
anyhow.workspace = true
serde_json.workspace = true
serde.workspace = true
sqlparser = { version = "0.59.0", features = ["visitor"] }

View File

@@ -1,4 +0,0 @@
mod asset_parser;
mod asset_parser_utils;
pub use asset_parser::parse_assets;
pub use asset_parser_utils::parse_wmill_sdk_sql_assets;

View File

@@ -20,4 +20,5 @@ windmill-types.workspace = true
anyhow.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
serde.workspace = true
serde.workspace = true
sqlparser = { version = "0.59.0", features = ["visitor"] }

View File

@@ -20,6 +20,11 @@ pub use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ};
pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__";
pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__";
mod asset_parser;
mod asset_parser_utils;
pub use asset_parser::parse_assets;
pub use asset_parser_utils::parse_wmill_sdk_sql_assets;
pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_mysql_file(&code)?;
if let Some(x) = parsed {

View File

@@ -1,23 +0,0 @@
[package]
name = "windmill-parser-ts-asset"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_ts_asset"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
wasm-bindgen.workspace = true
serde-wasm-bindgen.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
swc_common.workspace = true
triomphe.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
swc_ecma_visit.workspace = true
anyhow.workspace = true

View File

@@ -15,6 +15,7 @@ serde-wasm-bindgen.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
swc_common.workspace = true
triomphe.workspace = true
swc_ecma_parser.workspace = true

View File

@@ -259,7 +259,7 @@ impl Visit for AssetsFinder {
});
// We use the SQL parser to detect RW, specific tables, etc.
let sql_assets = windmill_parser_sql_asset::parse_wmill_sdk_sql_assets(
let sql_assets = windmill_parser_sql::parse_wmill_sdk_sql_assets(
*kind,
asset_name,
schema.as_deref(),

View File

@@ -211,6 +211,8 @@ pub enum TypeDecl {
Interface(TsInterfaceDecl),
Alias(TsTypeAliasDecl),
}
pub mod asset_parser;
pub use asset_parser::parse_assets;
/// skip_params is a micro optimization for when we just want to find the main
/// function without parsing all the params.

View File

@@ -39,7 +39,6 @@ nu-parser = [ "dep:windmill-parser-nu"]
java-parser = [ "dep:windmill-parser-java"]
ruby-parser = [ "dep:windmill-parser-ruby"]
wac-parser = [ "dep:windmill-parser-wac"]
asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"]
[dependencies]
anyhow.workspace = true
@@ -58,9 +57,6 @@ windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
windmill-parser-ruby = { workspace = true, optional = true }
windmill-parser-wac = { workspace = true, optional = true }
windmill-parser-ts-asset = { workspace = true, optional = true }
windmill-parser-py-asset = { workspace = true, optional = true }
windmill-parser-sql-asset = { workspace = true, optional = true }
wasm-bindgen.workspace = true
serde_json.workspace = true

View File

@@ -61,11 +61,6 @@ const targets = [
desc: "Workflow-as-Code",
features: "wac-parser",
env: "default",
}, {
ident: "asset",
desc: "Asset parsers (TS, Python, SQL) with SQL AST",
features: "asset-parser",
env: "default",
},
# ^^^ Add new entry here ^^^
];

View File

@@ -33,6 +33,3 @@ popd
pushd "pkg-java" && npm publish ${args}
popd
pushd "pkg-asset" && npm publish ${args}
popd

View File

@@ -187,28 +187,28 @@ pub fn parse_ruby(code: &str) -> String {
wrap_sig(windmill_parser_ruby::parse_ruby_signature(code))
}
#[cfg(feature = "asset-parser")]
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_assets_sql(code: &str) -> String {
match windmill_parser_sql_asset::parse_assets(code) {
match windmill_parser_sql::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "asset-parser")]
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_assets_ts(code: &str) -> String {
match windmill_parser_ts_asset::parse_assets(code) {
match windmill_parser_ts::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "asset-parser")]
#[cfg(feature = "py-parser")]
#[wasm_bindgen]
pub fn parse_assets_py(code: &str) -> String {
match windmill_parser_py_asset::parse_assets(code) {
match windmill_parser_py::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}

View File

@@ -20,7 +20,6 @@ pub async fn connect_db(
server_mode: bool,
indexer_mode: bool,
worker_mode: bool,
num_workers: i32,
#[cfg(feature = "private")] mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
use anyhow::Context;
@@ -35,7 +34,13 @@ pub async fn connect_db(
} else if indexer_mode {
DEFAULT_MAX_CONNECTIONS_INDEXER
} else {
DEFAULT_MAX_CONNECTIONS_WORKER + (num_workers.max(1) as u32) - 1
DEFAULT_MAX_CONNECTIONS_WORKER
+ std::env::var("NUM_WORKERS")
.ok()
.map(|x| x.parse().ok())
.flatten()
.unwrap_or(1)
- 1
}
}
};
@@ -98,8 +103,9 @@ pub async fn connect(
use sqlx::Executor;
use std::time::Duration;
let mut pool_options = sqlx::postgres::PgPoolOptions::new()
.min_connections(0)
.min_connections((max_connections / 5).clamp(1, max_connections))
.max_connections(max_connections)
.acquire_timeout(Duration::from_secs(5))
.max_lifetime(Duration::from_secs(30 * 60)); // 30 mins
if worker_mode {
pool_options = pool_options.idle_timeout(Duration::from_secs(60));

View File

@@ -243,14 +243,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
create_dir_all(&*HUB_CACHE_DIR)?;
create_dir_all(&*BUN_BUNDLE_CACHE_DIR)?;
// Ensure the latest git sync script is always cached, regardless of hubPaths.json contents
let mut all_paths: Vec<String> = paths.into_values().collect();
let latest_git_sync = windmill_common::workspaces::LATEST_GIT_SYNC_SCRIPT_PATH.to_string();
if !all_paths.contains(&latest_git_sync) {
all_paths.push(latest_git_sync);
}
for path in &all_paths {
for path in paths.values() {
tracing::info!("Caching hub script at {path}");
let res = get_hub_script_content_and_requirements(Some(path), None).await?;
if res
@@ -701,7 +694,6 @@ async fn windmill_main() -> anyhow::Result<()> {
let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer || mode == Mode::MCP {
0
} else if is_native_mode_from_env() {
NATIVE_MODE_RESOLVED.store(true, std::sync::atomic::Ordering::Relaxed);
println!("Native mode enabled: forcing NUM_WORKERS=8");
8
} else {
@@ -874,30 +866,6 @@ async fn windmill_main() -> anyhow::Result<()> {
}
}
// Resolve native mode early (before connect_db) so connection pool size accounts for it.
// native_mode can come from env OR from the DB worker group config.
if worker_mode && !is_native_mode_from_env() {
if let Some(db) = conn.as_sql() {
let native_from_db: bool = sqlx::query_scalar!(
"SELECT (config->>'native_mode')::boolean FROM config WHERE name = $1",
format!("worker__{}", *windmill_common::worker::WORKER_GROUP)
)
.fetch_optional(db)
.await
.ok()
.flatten()
.flatten()
.unwrap_or(false);
if native_from_db {
NATIVE_MODE_RESOLVED.store(true, std::sync::atomic::Ordering::Relaxed);
num_workers = 8;
tracing::info!(
"Native mode detected from worker config (early): forcing NUM_WORKERS=8"
);
}
}
}
let conn = if mode == Mode::Agent {
conn
} else {
@@ -910,7 +878,6 @@ async fn windmill_main() -> anyhow::Result<()> {
server_mode,
indexer_mode,
worker_mode,
num_workers,
#[cfg(feature = "private")]
killpill_rx.resubscribe(),
)
@@ -1015,6 +982,16 @@ Windmill Community Edition {GIT_VERSION}
)
.await;
// native_mode may also be set via DB worker group config (not just env).
// NATIVE_MODE_RESOLVED is updated by load_worker_config during initial_load.
if worker_mode
&& !is_native_mode_from_env()
&& NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed)
{
num_workers = 8;
tracing::info!("Native mode detected from worker config: forcing NUM_WORKERS=8");
}
monitor_db(
&conn,
&base_internal_url,
@@ -1907,7 +1884,7 @@ pub async fn run_workers(
tracing::info!(
"Starting {num_workers} workers and SLEEP_QUEUE={}ms",
windmill_worker::sleep_queue()
*windmill_worker::SLEEP_QUEUE
);
for i in 1..(num_workers + 1) {

View File

@@ -251,8 +251,9 @@ pub async fn initial_load(
.map(|x| x.tags.clone())
.unwrap_or_default();
// we only check from env as native_mode is not stored in the token
// NATIVE_MODE_RESOLVED is already set in main.rs during startup
let native_mode = windmill_common::worker::is_native_mode_from_env();
windmill_common::worker::NATIVE_MODE_RESOLVED
.store(native_mode, std::sync::atomic::Ordering::Relaxed);
*config = WorkerConfig {
worker_tags,
env_vars: load_env_vars(
@@ -879,19 +880,10 @@ struct TokenRow {
workspace_id: Option<String>,
}
/// When updating this filter, also update:
/// - `register_token_expiry_notification` in windmill-api-auth/src/lib.rs
/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte
fn is_user_token(label: Option<&str>) -> bool {
match label {
None => true,
Some(l) => {
l != "session"
&& !l.starts_with("ephemeral")
&& !l.starts_with("Ephemeral")
&& l != "debugger-token"
&& !l.starts_with("mcp-oauth-")
}
Some(l) => l != "session" && !l.starts_with("ephemeral") && !l.starts_with("Ephemeral"),
}
}

View File

@@ -109,9 +109,7 @@ job_result_stream_v2: job_id(uuid), workspace_id(text), stream(text), idx(int)
job_settings: job_id(uuid), runnable_settings(bigint)
job_stats: workspace_id(char), job_id(uuid), metric_id(char), metric_name(char), metric_kind(metric_kind), scalar_int(int), scalar_float(float), timestamps(ts), timeseries_int(int[]), timeseries_float(float[])
FK: (workspace_id) -> workspace(id)
kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), topic(char), partition(int), offset(bigint), created_at(ts)
FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path)
kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool)
kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[])
log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool)
magic_link: email(char), token(char), expiration(ts)
mcp_oauth_client: mcp_server_url(text), client_id(text), client_secret(text), client_secret_expires_at(ts), token_endpoint(text), created_at(ts)

View File

@@ -581,9 +581,6 @@ pub async fn create_token_internal(
}
/// Insert a pending expiry notification row for user tokens that have an expiration.
/// When updating this filter, also update:
/// - `is_user_token` in src/monitor.rs
/// - `isUserToken` in frontend/src/lib/components/settings/TokensTable.svelte
pub async fn register_token_expiry_notification(
tx: &mut sqlx::PgConnection,
token: &str,
@@ -592,12 +589,7 @@ pub async fn register_token_expiry_notification(
) {
let Some(expiration) = expiration else { return };
if label == Some("session")
|| label.is_some_and(|l| {
l.starts_with("ephemeral")
|| l.starts_with("Ephemeral")
|| l == "debugger-token"
|| l.starts_with("mcp-oauth-")
})
|| label.is_some_and(|l| l.starts_with("ephemeral") || l.starts_with("Ephemeral"))
{
return;
}

View File

@@ -129,12 +129,11 @@ async fn update_config(
#[cfg(not(feature = "enterprise"))]
let config = if name.starts_with("worker__") {
// In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode
// In CE, only allow setting worker_tags, cache_clear, and init_bash
serde_json::json!({
"worker_tags": config.get("worker_tags"),
"cache_clear": config.get("cache_clear"),
"init_bash": config.get("init_bash"),
"native_mode": config.get("native_mode")
"init_bash": config.get("init_bash")
})
} else {
config

View File

@@ -10,8 +10,8 @@ path = "src/lib.rs"
[features]
default = []
private = ["windmill-test-utils/private", "dep:aws-config", "dep:aws-credential-types", "dep:aws-sdk-sqs", "windmill-git-sync/private"]
enterprise = ["windmill-test-utils/enterprise", "dep:base64", "windmill-git-sync/enterprise"]
private = ["windmill-test-utils/private", "dep:aws-config", "dep:aws-credential-types", "dep:aws-sdk-sqs"]
enterprise = ["windmill-test-utils/enterprise", "dep:base64"]
deno_core = ["windmill-test-utils/deno_core"]
mcp = []
run_inline = ["dep:windmill-worker", "windmill-test-utils/run_inline", "windmill-test-utils/duckdb"]
@@ -22,7 +22,6 @@ windmill-api-client.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-native-triggers = { workspace = true, features = ["native_trigger"] }
windmill-api-auth.workspace = true
windmill-git-sync.workspace = true
windmill-worker = { workspace = true, optional = true }
sqlx.workspace = true
serde_json.workspace = true

View File

@@ -251,8 +251,7 @@ async fn test_websocket_e2e(db: Pool<Postgres>) -> anyhow::Result<()> {
"test-workspace",
"test-user",
"test@windmill.dev",
&[json!({"type": "RawMessage", "content": "hello from e2e test"})]
as &[serde_json::Value],
&[json!({"type": "RawMessage", "content": "hello from e2e test"})] as &[serde_json::Value],
)
.execute(&db)
.await?;
@@ -303,11 +302,9 @@ async fn test_postgres_e2e(db: Pool<Postgres>) -> anyhow::Result<()> {
sqlx::query("CREATE TABLE test_trigger_table (id serial PRIMARY KEY, data text)")
.execute(&db)
.await?;
sqlx::query(&format!(
"CREATE PUBLICATION {pub_name} FOR TABLE test_trigger_table"
))
.execute(&db)
.await?;
sqlx::query(&format!("CREATE PUBLICATION {pub_name} FOR TABLE test_trigger_table"))
.execute(&db)
.await?;
sqlx::query(&format!(
"SELECT pg_create_logical_replication_slot('{slot_name}', 'pgoutput')"
))
@@ -316,9 +313,10 @@ async fn test_postgres_e2e(db: Pool<Postgres>) -> anyhow::Result<()> {
// Extract the test DB name from the pool so the resource points here,
// not at the main windmill database.
let test_db_name: String = sqlx::query_scalar("SELECT current_database()")
.fetch_one(&db)
.await?;
let test_db_name: String =
sqlx::query_scalar("SELECT current_database()")
.fetch_one(&db)
.await?;
insert_resource(
&db,

View File

@@ -214,9 +214,12 @@ async fn test_capture_delete(db: Pool<Postgres>) -> anyhow::Result<()> {
.execute(&db)
.await?;
let count = sqlx::query_scalar!("SELECT COUNT(*) FROM capture WHERE id = $1", id,)
.fetch_one(&db)
.await?;
let count = sqlx::query_scalar!(
"SELECT COUNT(*) FROM capture WHERE id = $1",
id,
)
.fetch_one(&db)
.await?;
assert_eq!(count, Some(0));
@@ -382,10 +385,7 @@ async fn test_capture_api_list_captures(db: Pool<Postgres>) -> anyhow::Result<()
.send()
.await?;
assert!(
response.status().is_success(),
"list captures should succeed"
);
assert!(response.status().is_success(), "list captures should succeed");
let captures: Vec<CaptureResponse> = response.json().await?;
assert_eq!(captures.len(), 3);
@@ -480,9 +480,12 @@ async fn test_capture_api_delete(db: Pool<Postgres>) -> anyhow::Result<()> {
assert!(response.status().is_success(), "delete should succeed");
let count = sqlx::query_scalar!("SELECT COUNT(*) FROM capture WHERE id = $1", id,)
.fetch_one(&db)
.await?;
let count = sqlx::query_scalar!(
"SELECT COUNT(*) FROM capture WHERE id = $1",
id,
)
.fetch_one(&db)
.await?;
assert_eq!(count, Some(0));
@@ -930,8 +933,7 @@ async fn test_kafka_trigger_insert(db: Pool<Postgres>) -> anyhow::Result<()> {
let trigger = sqlx::query!(
r#"
SELECT kafka_resource_path, topics, group_id, mode AS "mode: String",
auto_offset_reset, auto_commit, reset_offset
SELECT kafka_resource_path, topics, group_id, mode AS "mode: String"
FROM kafka_trigger
WHERE workspace_id = $1 AND path = $2
"#,
@@ -945,50 +947,6 @@ async fn test_kafka_trigger_insert(db: Pool<Postgres>) -> anyhow::Result<()> {
assert_eq!(trigger.topics, vec!["topic-a", "topic-b"]);
assert_eq!(trigger.group_id, "my-consumer-group");
assert_eq!(trigger.mode, "enabled");
assert_eq!(trigger.auto_offset_reset, "latest");
assert_eq!(trigger.auto_commit, true);
assert_eq!(trigger.reset_offset, false);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_kafka_trigger_insert_auto_commit_disabled(db: Pool<Postgres>) -> anyhow::Result<()> {
sqlx::query!(
r#"
INSERT INTO kafka_trigger (
path, kafka_resource_path, topics, group_id, script_path,
is_flow, workspace_id, edited_by, email, auto_commit
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
"#,
"f/test/kafka_trigger_no_commit",
"u/admin/kafka_resource",
&["topic-c"] as &[&str],
"my-consumer-group-2",
"f/test/kafka_handler",
false,
"test-workspace",
"test-user",
"test@windmill.dev",
false,
)
.execute(&db)
.await?;
let trigger = sqlx::query!(
r#"
SELECT auto_commit
FROM kafka_trigger
WHERE workspace_id = $1 AND path = $2
"#,
"test-workspace",
"f/test/kafka_trigger_no_commit",
)
.fetch_one(&db)
.await?;
assert_eq!(trigger.auto_commit, false);
Ok(())
}

View File

@@ -570,7 +570,6 @@ pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Resu
&HTTP_CLIENT,
&db,
windmill_common::stats_oss::SendStatsReason::Manual,
false,
)
.await?;
@@ -583,7 +582,6 @@ pub async fn get_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Resul
let stats = windmill_common::stats_oss::get_stats_payload(
&db,
&windmill_common::stats_oss::SendStatsReason::Manual,
false,
)
.await?;
let encrypted = windmill_common::stats_oss::encrypt_stats(&stats)?;

View File

@@ -63,7 +63,6 @@ windmill-object-store.workspace = true
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-sql-asset.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-py = { workspace = true, optional = true }
windmill-parser-py-imports = { workspace = true, optional = true }

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.655.0
version: 1.654.0
title: Windmill API
contact:
@@ -1939,58 +1939,6 @@ paths:
"200":
description: Successfully imported the installation
/w/{workspace}/github_app/ghes_installation_callback:
post:
summary: GHES installation callback
description: Register a self-managed GitHub App installation from GitHub Enterprise Server
operationId: ghesInstallationCallback
tags:
- Git Sync
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- installation_id
properties:
installation_id:
type: integer
format: int64
description: The GitHub App installation ID from GHES
responses:
"200":
description: GHES installation registered successfully
/github_app/ghes_config:
get:
summary: Get GHES app config
description: Returns the GitHub Enterprise Server app configuration (without private key) for constructing the installation URL
operationId: getGhesConfig
tags:
- Git Sync
responses:
"200":
description: GHES app configuration
content:
application/json:
schema:
type: object
properties:
base_url:
type: string
app_slug:
type: string
client_id:
type: string
required:
- base_url
- app_slug
- client_id
/users/accept_invite:
post:
summary: accept invite to workspace
@@ -12069,39 +12017,6 @@ paths:
"200":
description: kafka trigger offsets reset successfully
/w/{workspace}/kafka_triggers/commit_offsets/{path}:
post:
summary: commit kafka offsets for a specific trigger
operationId: commitKafkaOffsets
tags:
- kafka_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: offsets to commit
required: true
content:
application/json:
schema:
type: object
properties:
topic:
type: string
partition:
type: integer
format: int32
offset:
type: integer
format: int64
required:
- topic
- partition
- offset
responses:
"200":
description: kafka offsets committed successfully
/w/{workspace}/nats_triggers/create:
post:
summary: create nats trigger
@@ -22183,10 +22098,6 @@ components:
- earliest
default: latest
description: "Initial offset behavior when consumer group has no committed offset. 'latest' starts from new messages only, 'earliest' starts from the beginning."
auto_commit:
type: boolean
default: true
description: "When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint."
server_id:
type: string
description: ID of the server currently handling this trigger (internal)
@@ -22254,10 +22165,6 @@ components:
- earliest
default: latest
description: "Initial offset behavior when consumer group has no committed offset."
auto_commit:
type: boolean
default: true
description: "When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint."
mode:
$ref: "#/components/schemas/TriggerMode"
error_handler_path:
@@ -22317,10 +22224,6 @@ components:
- earliest
default: latest
description: "Initial offset behavior when consumer group has no committed offset."
auto_commit:
type: boolean
default: true
description: "When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint."
path:
type: string
description: The unique path identifier for this trigger
@@ -23629,6 +23532,7 @@ components:
items:
$ref: "#/components/schemas/GitSyncObjectType"
required:
- script_path
- git_repo_resource_path
MetricMetadata:

View File

@@ -1012,7 +1012,6 @@ async fn http_payload(
.to_v2_preprocessor_args(
&http_trigger_config.route_path,
&route_path,
"",
&params,
headers,
query,

View File

@@ -4833,7 +4833,7 @@ fn register_potential_assets_on_inline_execution(
preview: &PreviewInline,
) {
let assets = if preview.language == ScriptLang::DuckDb {
Some(windmill_parser_sql_asset::parse_assets(&preview.content).map(|a| a.assets))
Some(windmill_parser_sql::parse_assets(&preview.content).map(|a| a.assets))
} else if preview.language == ScriptLang::Postgresql {
let datatable = preview
.args
@@ -4851,7 +4851,7 @@ fn register_potential_assets_on_inline_execution(
(None, None)
};
let content = content.as_deref().unwrap_or(&preview.content);
windmill_parser_sql_asset::parse_wmill_sdk_sql_assets(
windmill_parser_sql::parse_wmill_sdk_sql_assets(
AssetKind::DataTable,
datatable,
schema.as_deref(),

View File

@@ -468,7 +468,6 @@ async fn route_job(
.to_args_from_format(
&trigger.route_path,
&called_path,
&trigger.path,
&params,
runnable_format,
trigger.wrap_body,

View File

@@ -68,7 +68,6 @@ struct HttpTriggerPreprocessorEvent<'a> {
kind: String,
route: &'a str,
path: &'a str,
trigger_path: &'a str,
body: Box<RawValue>,
raw_string: Option<String>,
params: &'a HashMap<String, String>,
@@ -118,7 +117,6 @@ impl HttpTriggerArgs {
self,
route_path: &str,
called_path: &str,
trigger_path: &str,
params: &HashMap<String, String>,
format: RunnableFormat,
wrap_body: bool,
@@ -128,14 +126,7 @@ impl HttpTriggerArgs {
match format {
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => {
// we don't care about wrap_body in v2
self.to_v2_preprocessor_args(
route_path,
called_path,
trigger_path,
params,
headers,
query,
)
self.to_v2_preprocessor_args(route_path, called_path, params, headers, query)
}
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => self
.to_v1_preprocessor_args(
@@ -186,7 +177,6 @@ impl HttpTriggerArgs {
self,
route_path: &str,
called_path: &str,
trigger_path: &str,
params: &HashMap<String, String>,
headers: HashMap<String, Box<RawValue>>,
query: HashMap<String, Box<RawValue>>,
@@ -203,7 +193,6 @@ impl HttpTriggerArgs {
method: (&self.0.metadata.method).try_into()?,
route: route_path,
path: called_path,
trigger_path,
params,
}),
);

View File

@@ -21,14 +21,15 @@ pub fn prepend_token_to_github_url(
) -> crate::error::Result<String> {
let url = Url::parse(github_url)?;
let host = url.host_str().ok_or_else(|| {
crate::error::Error::BadRequest("Invalid GitHub URL: no host".to_string())
})?;
if url.host_str() != Some("github.com") {
return Err(crate::error::Error::BadRequest(
"Invalid: not a github URL".to_string(),
));
}
Ok(format!(
"https://x-access-token:{}@{}{}",
"https://x-access-token:{}@github.com{}",
installation_token,
host,
url.path()
))
}

View File

@@ -58,7 +58,6 @@ pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy";
pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route";
pub const SECRET_BACKEND_SETTING: &str = "secret_backend";
pub const MIN_KEEP_ALIVE_VERSION_SETTING: &str = "min_keep_alive_version";
pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app";
pub const ENV_SETTINGS: &[&str] = &[
"DISABLE_NSJAIL",

View File

@@ -21,9 +21,9 @@ pub struct TantivyIndexerSettings {
impl Default for TantivyIndexerSettings {
fn default() -> Self {
TantivyIndexerSettings {
writer_memory_budget: 150_000_000,
commit_job_max_batch_size: 10_000,
commit_log_max_batch_size: 5_000,
writer_memory_budget: 300_000_000,
commit_job_max_batch_size: 50_000,
commit_log_max_batch_size: 10_000,
refresh_index_period: 300,
refresh_log_index_period: 300,
max_indexed_job_log_size: 1_000_000,

View File

@@ -32,7 +32,6 @@ pub async fn send_stats(
_http_client: &reqwest::Client,
_db: &DB,
_reason: SendStatsReason,
_minimal: bool,
) -> Result<()> {
// stats details are closed source
Ok(())
@@ -57,11 +56,7 @@ pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>(
pub struct Stats {}
#[cfg(not(feature = "private"))]
pub async fn get_stats_payload(
_db: &DB,
_reason: &SendStatsReason,
_minimal: bool,
) -> Result<Stats> {
pub async fn get_stats_payload(_db: &DB, _reason: &SendStatsReason) -> Result<Stats> {
// stats details are closed source
Ok(Stats {})
}

View File

@@ -149,16 +149,11 @@ pub enum ObjectType {
WorkspaceDependencies,
}
pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28160/sync-script-to-git-repo-windmill";
#[derive(Serialize, Deserialize, Debug)]
pub struct GitRepositorySettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub exclude_types_override: Option<Vec<ObjectType>>,
/// None means auto-managed: always use LATEST_GIT_SYNC_SCRIPT_PATH.
/// Some(path) means pinned to a specific script.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub script_path: Option<String>,
pub script_path: String,
pub git_repo_resource_path: String,
pub use_individual_branch: Option<bool>,
pub group_by_folder: Option<bool>,
@@ -169,26 +164,23 @@ pub struct GitRepositorySettings {
}
impl GitRepositorySettings {
pub fn effective_script_path(&self) -> &str {
self.script_path
.as_deref()
.unwrap_or(LATEST_GIT_SYNC_SCRIPT_PATH)
}
pub fn is_script_meets_min_version(&self, min_version: u32) -> error::Result<bool> {
let path = self.effective_script_path();
// example: "hub/28102/sync-script-to-git-repo-windmill"
let current = path
let current = self
.script_path
.split("/") // -> ["hub" "28102" "sync-script-to-git-repo-windmill"]
.skip(1) // omit "hub"
.next() // get numeric id
.ok_or(Error::InternalErr(format!(
"cannot get script version id from: {}",
path
&self.script_path
)))?
.parse()
.unwrap_or_else(|e| {
tracing::warn!("cannot get script version id from: {}. e: {e}", path);
tracing::warn!(
"cannot get script version id from: {}. e: {e}",
&self.script_path
);
u32::MAX
});

View File

@@ -891,12 +891,12 @@ async fn delete_resource(
.fetch_optional(&mut *tx)
.await?;
not_found_if_none(deleted_path, "Resource", &path)?;
let deleted_linked_variable = sqlx::query_scalar!(
"DELETE FROM variable WHERE path = $1 AND workspace_id = $2 RETURNING path",
sqlx::query!(
"DELETE FROM variable WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(&mut *tx)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
@@ -924,34 +924,9 @@ async fn delete_resource(
webhook.send_message(
w_id.clone(),
WebhookMessage::DeleteResource { workspace: w_id.clone(), path: path.to_owned() },
WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() },
);
if deleted_linked_variable.is_some() {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Variable {
path: path.to_string(),
parent_path: Some(path.to_string()),
},
Some(format!(
"Variable '{}' deleted (linked resource deleted)",
path
)),
true,
None,
)
.await?;
webhook.send_message(
w_id.clone(),
WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() },
);
}
Ok(format!("resource {} deleted", path))
}

View File

@@ -536,12 +536,12 @@ async fn delete_variable(
)
.execute(&mut *tx)
.await?;
let deleted_linked_resource = sqlx::query_scalar!(
"DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path",
sqlx::query!(
"DELETE FROM resource WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(&mut *tx)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
@@ -575,34 +575,9 @@ async fn delete_variable(
webhook.send_message(
w_id.clone(),
WebhookMessage::DeleteVariable { workspace: w_id.clone(), path: path.to_owned() },
WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() },
);
if deleted_linked_resource.is_some() {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Resource {
path: path.to_string(),
parent_path: Some(path.to_string()),
},
Some(format!(
"Resource '{}' deleted (linked variable deleted)",
path
)),
true,
None,
)
.await?;
webhook.send_message(
w_id.clone(),
WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() },
);
}
Ok(format!("variable {} deleted", path))
}

View File

@@ -322,7 +322,7 @@ impl Listener for WebsocketTrigger {
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
payload: Self::Payload,
mut trigger_info: HashMap<String, Box<RawValue>>,
trigger_info: HashMap<String, Box<RawValue>>,
extra: Option<Self::Extra>,
) -> Result<()> {
let ListeningTrigger {
@@ -338,7 +338,6 @@ impl Listener for WebsocketTrigger {
let WebsocketConfig { url, .. } = trigger_config;
trigger_info.insert("trigger_path".to_string(), to_raw_value(path));
let args = WebsocketTrigger::build_job_args(
&script_path,
*is_flow,

View File

@@ -21,7 +21,6 @@ use windmill_common::{
jobs::JobTriggerKind,
triggers::{TriggerKind, TriggerMetadata},
utils::report_critical_error,
worker::to_raw_value,
DB, INSTANCE_NAME,
};
@@ -468,13 +467,9 @@ pub trait Listener: TriggerCrud + TriggerJobArgs {
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
payload: Self::Payload,
mut trigger_info: HashMap<String, Box<RawValue>>,
trigger_info: HashMap<String, Box<RawValue>>,
_extra: Option<Self::Extra>,
) -> Result<()> {
trigger_info.insert(
"trigger_path".to_string(),
to_raw_value(&listening_trigger.path),
);
let args = Self::build_job_args(
&listening_trigger.script_path,
listening_trigger.is_flow,
@@ -557,11 +552,6 @@ pub trait Listener: TriggerCrud + TriggerJobArgs {
return Ok(());
}
let mut trigger_info = trigger_info;
trigger_info.insert(
"trigger_path".to_string(),
to_raw_value(&listening_trigger.path),
);
let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info);
if let Err(err) = insert_capture_payload(
db,

View File

@@ -422,14 +422,11 @@ pub fn start_background_processor(
}
async fn send_job_completed(job_completed_tx: JobCompletedSender, jc: JobCompleted) {
if let Err(e) = job_completed_tx
job_completed_tx
.send_job(jc, true)
.with_context(windmill_common::otel_oss::otel_ctx())
.await
{
tracing::error!("send job completed failed, triggering worker shutdown: {e:#}");
job_completed_tx.send_worker_killpill();
}
.expect("send job completed")
}
pub async fn process_result(

View File

@@ -303,7 +303,7 @@ pub struct PowershellRepo {
lazy_static::lazy_static! {
static ref SLEEP_QUEUE_BASE: u64 = std::env::var("SLEEP_QUEUE")
pub static ref SLEEP_QUEUE: u64 = std::env::var("SLEEP_QUEUE")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or_else(|| {
@@ -647,14 +647,6 @@ lazy_static::lazy_static! {
pub static ref FLOW_RUNNER_RUNNING: Mutex<bool> = Mutex::new(false);
}
pub fn sleep_queue() -> u64 {
if NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) {
300
} else {
*SLEEP_QUEUE_BASE
}
}
type Envs = Vec<(String, String)>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -895,19 +887,6 @@ impl JobCompletedSender {
pub fn is_sql(&self) -> bool {
matches!(self, Self::Sql(_))
}
pub fn set_worker_killpill(&mut self, killpill_tx: KillpillSender) {
if let Self::Sql(sql) = self {
sql.worker_killpill_tx = Some(killpill_tx);
}
}
pub fn send_worker_killpill(&self) {
if let Self::Sql(SqlJobCompletedSender { worker_killpill_tx: Some(killpill_tx), .. }) = self
{
killpill_tx.send();
}
}
}
#[derive(Clone)]
@@ -915,7 +894,6 @@ pub struct SqlJobCompletedSender {
sender: flume::Sender<SendResult>,
unbounded_sender: flume::Sender<SendResult>,
killpill_tx: broadcast::Sender<()>,
worker_killpill_tx: Option<KillpillSender>,
}
pub struct JobCompletedReceiver {
@@ -940,12 +918,7 @@ impl JobCompletedSender {
let (unbounded_sender, unbounded_rx) = flume::unbounded::<SendResult>();
let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10);
(
Self::Sql(SqlJobCompletedSender {
sender,
unbounded_sender,
killpill_tx,
worker_killpill_tx: None,
}),
Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }),
JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx },
)
}
@@ -1400,7 +1373,7 @@ fn start_interactive_worker_shell(
{
Duration::from_secs(WORKER_SHELL_NAP_TIME_DURATION)
}
_ => Duration::from_millis(sleep_queue() * 10),
_ => Duration::from_millis(*SLEEP_QUEUE * 10),
};
tokio::select! {
_ = tokio::time::sleep(nap_time) => {
@@ -1413,7 +1386,7 @@ fn start_interactive_worker_shell(
Err(err) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
tokio::time::sleep(Duration::from_millis(sleep_queue() * 20)).await;
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 20)).await;
}
};
}
@@ -1459,7 +1432,7 @@ pub async fn run_worker(
tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir");
#[cfg(feature = "python")]
if !NATIVE_MODE_RESOLVED.load(std::sync::atomic::Ordering::Relaxed) {
{
let (conn, worker_name, hostname, worker_dir) = (
conn.clone(),
worker_name.clone(),
@@ -1741,8 +1714,7 @@ pub async fn run_worker(
let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<SameWorkerPayload>(5);
let (mut job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 10);
job_completed_tx.set_worker_killpill(killpill_tx.clone());
let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 10);
let same_worker_queue_size = Arc::new(AtomicU16::new(0));
let same_worker_tx = SameWorkerSender(same_worker_tx, same_worker_queue_size.clone());
@@ -2727,7 +2699,7 @@ pub async fn run_worker(
None
};
tokio::time::sleep(Duration::from_millis(sleep_queue())).await;
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await;
#[cfg(feature = "benchmark")]
{
@@ -2748,7 +2720,7 @@ pub async fn run_worker(
}
Err(err) => {
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
tokio::time::sleep(Duration::from_millis(sleep_queue() * 5)).await;
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE * 5)).await;
}
};
}

View File

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.655.0";
export const VERSION = "v1.654.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({

View File

@@ -275,9 +275,6 @@ const command = new Command()
"Default TypeScript runtime (bun or deno)"
)
.action(async (opts: any, appFolder: string | undefined) => {
log.warn(
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
);
const { generateLocksCommand } = await import("./app_metadata.ts");
await generateLocksCommand(opts, appFolder);
});

View File

@@ -157,7 +157,7 @@ export async function generateAppLocksInternal(
return remote_path;
}
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
if (Object.keys(filteredDeps).length > 0) {
log.info(
(await blueColor())(
`Found workspace dependencies (${workspaceDependenciesLanguages
@@ -180,11 +180,9 @@ export async function generateAppLocksInternal(
}
if (changedScripts.length > 0) {
if (!noStaleMessage) {
log.info(
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
);
}
log.info(
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
);
if (rawApp) {
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER);
@@ -232,7 +230,7 @@ export async function generateAppLocksInternal(
yamlStringify(appFile as Record<string, any>, yamlOptions)
);
}
} else if (!noStaleMessage) {
} else {
log.info(colors.gray(`No scripts changed in ${appFolder}`));
}
}
@@ -248,9 +246,7 @@ export async function generateAppLocksInternal(
for (const [scriptPath, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(appFolder, hash, scriptPath);
}
if (!noStaleMessage) {
log.info(colors.green(`App ${remote_path} lockfiles updated`));
}
log.info(colors.green(`App ${remote_path} lockfiles updated`));
}
/**
@@ -771,7 +767,7 @@ export async function inferRunnableSchemaFromFile(
}
}
export function getAppFolders(elems: Record<string, any>, extension: string) {
function getAppFolders(elems: Record<string, any>, extension: string) {
return Object.keys(elems)
.filter((p) => p.endsWith(SEP + extension))
.map((p) => p.substring(0, p.length - (SEP + extension).length));

View File

@@ -56,20 +56,13 @@ export async function pushFlow(
}
const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile;
const fileReader = async (path: string) => await readFile(localPath + path, "utf-8");
await replaceInlineScripts(
localFlow.value.modules,
fileReader,
async (path: string) => await readFile(localPath + path, "utf-8"),
log,
localPath,
SEP
);
if (localFlow.value.failure_module) {
await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, localPath, SEP);
}
if (localFlow.value.preprocessor_module) {
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, localPath, SEP);
}
if (flow) {
if (isSuperset(localFlow, flow)) {
@@ -259,20 +252,13 @@ async function preview(
const localFlow = (await yamlParseFile(flowPath + "flow.yaml")) as FlowFile;
// Replace inline scripts with their actual content
const fileReader = async (path: string) => await readFile(flowPath + path, "utf-8");
await replaceInlineScripts(
localFlow.value.modules,
fileReader,
async (path: string) => await readFile(flowPath + path, "utf-8"),
log,
flowPath,
SEP
);
if (localFlow.value.failure_module) {
await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, flowPath, SEP);
}
if (localFlow.value.preprocessor_module) {
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, flowPath, SEP);
}
const input = opts.data ? await resolve(opts.data) : {};
@@ -308,15 +294,12 @@ async function preview(
}
}
export async function generateLocks(
async function generateLocks(
opts: GlobalOptions & {
yes?: boolean;
} & SyncOptions,
folder: string | undefined
) {
log.warn(
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);

View File

@@ -19,7 +19,6 @@ import {
} from "../../utils/metadata.ts";
import { ScriptLanguage } from "../../utils/script_common.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { newPathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
import { exts } from "../script/script.ts";
@@ -98,7 +97,7 @@ export async function generateFlowLockInternal(
return remote_path;
}
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
if (Object.keys(filteredDeps).length > 0) {
log.info(
(await blueColor())(
`Found workspace dependencies (${workspaceDependenciesLanguages
@@ -121,24 +120,15 @@ export async function generateFlowLockInternal(
}
}
if (!noStaleMessage) {
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
}
const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8");
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
await replaceInlineScripts(
flowValue.value.modules,
fileReader,
async (path: string) => await readFile(folder + SEP + path, "utf-8"),
log,
folder + SEP!,
SEP,
changedScripts
);
if (flowValue.value.failure_module) {
await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, changedScripts);
}
if (flowValue.value.preprocessor_module) {
await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, changedScripts);
}
//removeChangedLocks
flowValue.value = await updateFlow(
@@ -148,20 +138,12 @@ export async function generateFlowLockInternal(
filteredDeps
);
const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun");
const inlineScripts = extractInlineScriptsForFlows(
flowValue.value.modules,
{},
SEP,
opts.defaultTs,
lockAssigner
opts.defaultTs
);
if (flowValue.value.failure_module) {
inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.failure_module], {}, SEP, opts.defaultTs, lockAssigner));
}
if (flowValue.value.preprocessor_module) {
inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.preprocessor_module], {}, SEP, opts.defaultTs, lockAssigner));
}
inlineScripts.forEach((s) => {
writeIfChanged(process.cwd() + SEP + folder + SEP + s.path, s.content);
});
@@ -182,9 +164,7 @@ export async function generateFlowLockInternal(
for (const [path, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(folder, hash, path);
}
if (!noStaleMessage) {
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
}
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
}
/**
@@ -196,15 +176,7 @@ async function filterWorkspaceDependenciesForFlow(
rawWorkspaceDependencies: Record<string, string>,
folder: string
): Promise<Record<string, string>> {
const clonedValue = structuredClone(flowValue);
const depAssigner = newPathAssigner("bun");
const inlineScripts = extractInlineScriptsForFlows(clonedValue.modules, {}, SEP, undefined, depAssigner);
if (clonedValue.failure_module) {
inlineScripts.push(...extractInlineScriptsForFlows([clonedValue.failure_module], {}, SEP, undefined, depAssigner));
}
if (clonedValue.preprocessor_module) {
inlineScripts.push(...extractInlineScriptsForFlows([clonedValue.preprocessor_module], {}, SEP, undefined, depAssigner));
}
const inlineScripts = extractInlineScriptsForFlows(structuredClone(flowValue.modules), {}, SEP, undefined);
// Filter out lock files and map to common interface
const scripts = inlineScripts

View File

@@ -1,332 +0,0 @@
import { Command } from "@cliffy/command";
import { Confirm } from "@cliffy/prompt/confirm";
import { colors } from "@cliffy/ansi/colors";
import { sep as SEP } from "node:path";
import { GlobalOptions } from "../../types.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
import * as log from "../../core/log.ts";
import {
generateScriptMetadataInternal,
getRawWorkspaceDependencies,
} from "../../utils/metadata.ts";
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
import { generateAppLocksInternal, getAppFolders } from "../app/app_metadata.ts";
import {
elementsToMap,
FSFSElement,
ignoreF,
} from "../sync/sync.ts";
import { exts } from "../script/script.ts";
import { isFlowPath, isAppPath } from "../../utils/resource_folders.ts";
import { listSyncCodebases } from "../../utils/codebase.ts";
interface StaleItem {
type: "script" | "flow" | "app";
path: string;
folder: string;
isRawApp?: boolean;
}
async function generateMetadata(
opts: GlobalOptions & {
yes?: boolean;
lockOnly?: boolean;
schemaOnly?: boolean;
dryRun?: boolean;
skipScripts?: boolean;
skipFlows?: boolean;
skipApps?: boolean;
} & SyncOptions,
folder?: string
) {
if (folder === "") {
folder = undefined;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
const codebases = await listSyncCodebases(opts);
const ignore = await ignoreF(opts);
const staleItems: StaleItem[] = [];
// --schema-only implies skipping flows and apps (they only have locks, no schemas)
const skipScripts = opts.skipScripts ?? false;
const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false;
const skipApps = opts.skipApps ?? opts.schemaOnly ?? false;
const checking: string[] = [];
if (!skipScripts) checking.push("scripts");
if (!skipFlows) checking.push("flows");
if (!skipApps) checking.push("apps");
if (checking.length === 0) {
log.info(colors.yellow("Nothing to check (all types skipped)"));
return;
}
log.info(colors.gray(`Checking ${checking.join(", ")}...`));
// === Collect stale scripts ===
if (!skipScripts) {
// TODO: run elementsToMap only once but for all runnable types.
const scriptElems = await elementsToMap(
await FSFSElement(process.cwd(), codebases, false),
(p, isD) => {
return (
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
ignore(p, isD) ||
isFlowPath(p) ||
isAppPath(p)
);
},
false,
{}
);
for (const e of Object.keys(scriptElems)) {
const candidate = await generateScriptMetadataInternal(
e,
workspace,
opts,
true, // dryRun
true, // noStaleMessage
rawWorkspaceDependencies,
codebases,
false
);
if (candidate) {
staleItems.push({ type: "script", path: candidate, folder: e });
}
}
}
// === Collect stale flows ===
if (!skipFlows) {
const flowElems = Object.keys(
await elementsToMap(
await FSFSElement(process.cwd(), [], true),
(p, isD) => {
return (
ignore(p, isD) ||
(!isD &&
!p.endsWith(SEP + "flow.yaml") &&
!p.endsWith(SEP + "flow.json"))
);
},
false,
{}
)
).map((x) => x.substring(0, x.lastIndexOf(SEP)));
for (const folder of flowElems) {
const candidate = await generateFlowLockInternal(
folder,
true, // dryRun
workspace,
opts,
false,
true // noStaleMessage
);
if (candidate) {
staleItems.push({ type: "flow", path: candidate, folder });
}
}
}
// === Collect stale apps ===
if (!skipApps) {
const elems = await elementsToMap(
await FSFSElement(process.cwd(), [], true),
(p, isD) => {
return (
ignore(p, isD) ||
(!isD &&
!p.endsWith(SEP + "raw_app.yaml") &&
!p.endsWith(SEP + "app.yaml"))
);
},
false,
{}
);
const rawAppFolders = getAppFolders(elems, "raw_app.yaml");
const appFolders = getAppFolders(elems, "app.yaml");
for (const appFolder of rawAppFolders) {
const candidate = await generateAppLocksInternal(
appFolder,
true, // rawApp
true, // dryRun
workspace,
opts,
false,
true // noStaleMessage
);
if (candidate) {
staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: true });
}
}
for (const appFolder of appFolders) {
const candidate = await generateAppLocksInternal(
appFolder,
false, // rawApp
true, // dryRun
workspace,
opts,
false,
true // noStaleMessage
);
if (candidate) {
staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: false });
}
}
}
// === Filter by folder if specified ===
let filteredItems = staleItems;
if (folder) {
// Strip trailing separator to match deprecated flow/app handler behavior
// (see generateFlowLockInternal line 64-66, generateAppLocksInternal line 109-110)
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
}
filteredItems = staleItems.filter((item) => item.folder === folder || item.folder.startsWith(folder + SEP));
}
// === Show stale items and confirm ===
if (filteredItems.length === 0) {
log.info(colors.green("All metadata up-to-date"));
return;
}
// Group items by type for display
const scripts = filteredItems.filter((i) => i.type === "script");
const flows = filteredItems.filter((i) => i.type === "flow");
const apps = filteredItems.filter((i) => i.type === "app");
log.info("");
log.info(`Found ${filteredItems.length} item(s) with stale metadata:`);
if (scripts.length > 0) {
log.info(colors.gray(` Scripts (${scripts.length}):`));
for (const item of scripts) {
log.info(colors.yellow(` ${item.path}`));
}
}
if (flows.length > 0) {
log.info(colors.gray(` Flows (${flows.length}):`));
for (const item of flows) {
log.info(colors.yellow(` ${item.path}`));
}
}
if (apps.length > 0) {
log.info(colors.gray(` Apps (${apps.length}):`));
for (const item of apps) {
log.info(colors.yellow(` ${item.path}`));
}
}
if (opts.dryRun) {
return;
}
log.info("");
if (
!opts.yes &&
!(await Confirm.prompt({
message: "Update metadata?",
default: true,
}))
) {
return;
}
log.info("");
// === Process all stale items with progress counter ===
const total = filteredItems.length;
const maxWidth = `[${total}/${total}]`.length;
let current = 0;
const formatProgress = (n: number) => {
const bracket = `[${n}/${total}]`;
return colors.gray(bracket.padEnd(maxWidth, " "));
};
// Process scripts
for (const item of scripts) {
current++;
log.info(`${formatProgress(current)} script ${colors.cyan(item.path)}`);
await generateScriptMetadataInternal(
item.folder,
workspace,
opts,
false, // dryRun
true, // noStaleMessage - we handle output
rawWorkspaceDependencies,
codebases,
false
);
}
// Process flows
for (const item of flows) {
current++;
log.info(`${formatProgress(current)} flow ${colors.cyan(item.path)}`);
await generateFlowLockInternal(
item.folder,
false, // dryRun
workspace,
opts,
false,
true // noStaleMessage - we handle output
);
}
// Process apps
for (const item of apps) {
current++;
log.info(`${formatProgress(current)} app ${colors.cyan(item.path)}`);
await generateAppLocksInternal(
item.folder,
item.isRawApp!, // rawApp
false, // dryRun
workspace,
opts,
false,
true // noStaleMessage - we handle output
);
}
log.info("");
log.info(colors.green(`Done. Updated ${total} item(s).`));
}
const command = new Command()
.description("Generate metadata (locks, schemas) for all scripts, flows, and apps")
.arguments("[folder:string]")
.option("--yes", "Skip confirmation prompt")
.option("--dry-run", "Show what would be updated without making changes")
.option("--lock-only", "Re-generate only the lock files")
.option("--schema-only", "Re-generate only script schemas (skips flows and apps)")
.option("--skip-scripts", "Skip processing scripts")
.option("--skip-flows", "Skip processing flows")
.option("--skip-apps", "Skip processing apps")
.option(
"-i --includes <patterns:file[]>",
"Comma separated patterns to specify which files to include"
)
.option(
"-e --excludes <patterns:file[]>",
"Comma separated patterns to specify which files to exclude"
)
.action(generateMetadata as any);
export default command;

View File

@@ -978,7 +978,7 @@ export type GlobalDeps = Map<
Record<string, string>
>;
export async function generateMetadata(
async function generateMetadata(
opts: GlobalOptions & {
lockOnly?: boolean;
schemaOnly?: boolean;
@@ -986,9 +986,6 @@ export async function generateMetadata(
} & SyncOptions,
scriptPath: string | undefined
) {
log.warn(
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
);
log.info(
"This command only works for workspace scripts, for flows inline scripts use `wmill flow generate-locks`"
);

View File

@@ -592,35 +592,14 @@ function ZipFSElement(
}
let inlineScripts;
try {
const assigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() });
inlineScripts = extractInlineScriptsForFlows(
flow.value.modules as any,
{},
SEP,
defaultTs,
assigner,
undefined, // pathAssigner - let it create one
{ skipInlineScriptSuffix: getNonDottedPaths() },
);
if (flow.value.failure_module) {
inlineScripts.push(...extractInlineScriptsForFlows(
[flow.value.failure_module],
{},
SEP,
defaultTs,
assigner,
{ skipInlineScriptSuffix: getNonDottedPaths() },
));
}
if (flow.value.preprocessor_module) {
inlineScripts.push(...extractInlineScriptsForFlows(
[flow.value.preprocessor_module],
{},
SEP,
defaultTs,
assigner,
{ skipInlineScriptSuffix: getNonDottedPaths() },
));
}
} catch (error) {
log.error(
`Failed to extract inline scripts for flow at path: ${p}`,

File diff suppressed because it is too large Load Diff

View File

@@ -39,7 +39,6 @@ import queues from "./commands/queues/queues.ts";
import dependencies from "./commands/dependencies/dependencies.ts";
import init from "./commands/init/init.ts";
import jobs from "./commands/jobs/jobs.ts";
import generateMetadata from "./commands/generate-metadata/generate-metadata.ts";
import docs from "./commands/docs/docs.ts";
import { fetchVersion } from "./core/context.ts";
@@ -68,7 +67,7 @@ export {
workspaceAdd,
};
export const VERSION = "1.655.0";
export const VERSION = "1.654.0";
// Re-exported from constants.ts to maintain backwards compatibility
export { WM_FORK_PREFIX } from "./core/constants.ts";
@@ -130,7 +129,6 @@ const command = new Command()
.command("queues", queues)
.command("dependencies", dependencies)
.command("jobs", jobs)
.command("generate-metadata", generateMetadata)
.command("docs", docs)
.command("version --version", "Show version information")
.action(async (opts: any) => {

View File

@@ -35,7 +35,7 @@ function loadParser(pkgName: string): Promise<any> {
const wasmPath = _require.resolve(
`${pkgName}/windmill_parser_wasm_bg.wasm`
);
await mod.default({ module_or_path: readFileSync(wasmPath) });
await mod.default(readFileSync(wasmPath));
return mod;
})();
_parserCache.set(pkgName, p);
@@ -223,7 +223,7 @@ export async function generateScriptMetadataInternal(
return `${remotePath} (${language})`;
}
if (!justUpdateMetadataLock && !noStaleMessage) {
if (!justUpdateMetadataLock) {
log.info(colors.gray(`Generating metadata for ${scriptPath}`));
}

View File

@@ -1,498 +0,0 @@
/**
* Unit tests for failure_module and preprocessor_module inline script
* extraction (pull) and replacement (push).
*
* These tests verify that rawscript content in failure_module and
* preprocessor_module is correctly extracted to !inline references
* and resolved back, matching the existing behavior for regular modules.
*/
import { expect, test, describe } from "bun:test";
import { extractInlineScripts, extractCurrentMapping } from "../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { replaceInlineScripts } from "../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { newPathAssigner } from "../windmill-utils-internal/src/path-utils/path-assigner.ts";
import type { FlowModule } from "../windmill-utils-internal/src/gen/types.gen.ts";
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function makeRawscriptModule(
id: string,
content: string,
language: "bun" | "python3" | "deno" = "bun",
lock?: string,
): FlowModule {
return {
id,
value: {
type: "rawscript" as const,
content,
language,
lock: lock,
input_transforms: {},
},
};
}
const noopLogger = {
info: () => {},
error: () => {},
};
// ---------------------------------------------------------------------------
// extractInlineScripts — PULL direction
// ---------------------------------------------------------------------------
describe("extractInlineScripts for failure_module / preprocessor_module", () => {
test("extracts rawscript from failure_module wrapped in array", () => {
const failureModule = makeRawscriptModule(
"failure",
'export function main() { throw new Error("handler"); }',
"bun",
);
const scripts = extractInlineScripts([failureModule], {}, "/", "bun");
expect(scripts.length).toBeGreaterThanOrEqual(1);
const script = scripts.find((s) => !s.is_lock);
expect(script).toBeDefined();
expect(script!.content).toBe(
'export function main() { throw new Error("handler"); }',
);
// The module content should have been replaced with an !inline reference
expect(failureModule.value.content).toStartWith("!inline ");
});
test("extracts rawscript from preprocessor_module wrapped in array", () => {
const preprocessorModule = makeRawscriptModule(
"preprocessor",
"export function main() { return {}; }",
"python3",
);
const scripts = extractInlineScripts(
[preprocessorModule],
{},
"/",
"bun",
);
expect(scripts.length).toBeGreaterThanOrEqual(1);
const script = scripts.find((s) => !s.is_lock);
expect(script).toBeDefined();
expect(script!.content).toBe("export function main() { return {}; }");
expect(script!.language).toBe("python3");
expect(preprocessorModule.value.content).toStartWith("!inline ");
});
test("extracts lock alongside content", () => {
const mod = makeRawscriptModule(
"failure",
"console.log('hi')",
"bun",
"some-lock-content",
);
const scripts = extractInlineScripts([mod], {}, "/", "bun");
const contentScript = scripts.find((s) => !s.is_lock);
const lockScript = scripts.find((s) => s.is_lock);
expect(contentScript).toBeDefined();
expect(lockScript).toBeDefined();
expect(lockScript!.content).toBe("some-lock-content");
expect((mod.value as any).lock).toStartWith("!inline ");
});
test("shared pathAssigner prevents collisions when summaries match", () => {
// If a regular module and failure_module share the same summary,
// a shared PathAssigner deduplicates via its internal counter.
const regular = makeRawscriptModule("a", "code_a", "bun");
regular.summary = "my step";
const failure = makeRawscriptModule("failure", "code_failure", "bun");
failure.summary = "my step"; // same summary — would collide without shared assigner
const assigner = newPathAssigner("bun");
const scripts1 = extractInlineScripts([regular], {}, "/", "bun", assigner);
const scripts2 = extractInlineScripts([failure], {}, "/", "bun", assigner);
const allPaths = [...scripts1, ...scripts2]
.filter((s) => !s.is_lock)
.map((s) => s.path);
// All paths should be unique despite identical summaries
expect(allPaths.length).toBe(2);
expect(new Set(allPaths).size).toBe(2);
});
test("without shared pathAssigner, identical summaries produce duplicate paths", () => {
// Demonstrates the problem that sharing a PathAssigner solves.
const regular = makeRawscriptModule("a", "code_a", "bun");
regular.summary = "my step";
const failure = makeRawscriptModule("failure", "code_failure", "bun");
failure.summary = "my step";
// Separate assigners — each starts with a fresh counter
const scripts1 = extractInlineScripts([regular], {}, "/", "bun");
const scripts2 = extractInlineScripts([failure], {}, "/", "bun");
const allPaths = [...scripts1, ...scripts2]
.filter((s) => !s.is_lock)
.map((s) => s.path);
// Without a shared assigner, the paths collide
expect(allPaths.length).toBe(2);
expect(new Set(allPaths).size).toBe(1); // both got the same path
});
test("skips non-rawscript failure_module (identity type)", () => {
const identityModule: FlowModule = {
id: "failure",
value: { type: "identity" as any },
};
const scripts = extractInlineScripts([identityModule], {}, "/", "bun");
expect(scripts).toEqual([]);
});
});
// ---------------------------------------------------------------------------
// replaceInlineScripts — PUSH direction
// ---------------------------------------------------------------------------
describe("replaceInlineScripts for failure_module / preprocessor_module", () => {
test("resolves !inline reference back to file content", async () => {
const failureModule = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
);
const files: Record<string, string> = {
"failure.inline_script.ts": 'export function main() { return "error handled"; }',
};
await replaceInlineScripts(
[failureModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(failureModule.value.content).toBe(
'export function main() { return "error handled"; }',
);
});
test("resolves !inline reference for preprocessor_module", async () => {
const preprocessorModule = makeRawscriptModule(
"preprocessor",
"!inline preprocessor.inline_script.py",
"python3",
);
const files: Record<string, string> = {
"preprocessor.inline_script.py": "def main(): return {}",
};
await replaceInlineScripts(
[preprocessorModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(preprocessorModule.value.content).toBe("def main(): return {}");
});
test("resolves !inline lock reference", async () => {
const mod = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
"!inline failure.inline_script.lock",
);
const files: Record<string, string> = {
"failure.inline_script.ts": "code here",
"failure.inline_script.lock": "lock-data-here",
};
await replaceInlineScripts(
[mod],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(mod.value.content).toBe("code here");
expect((mod.value as any).lock).toBe("lock-data-here");
});
test("leaves non-inline content untouched", async () => {
const mod = makeRawscriptModule(
"failure",
"export function main() { return 1; }",
"bun",
);
await replaceInlineScripts(
[mod],
async () => {
throw new Error("should not be called");
},
noopLogger,
"/tmp/test/",
"/",
);
expect(mod.value.content).toBe("export function main() { return 1; }");
});
});
// ---------------------------------------------------------------------------
// Round-trip: extract then replace
// ---------------------------------------------------------------------------
describe("round-trip extract → replace for failure_module / preprocessor_module", () => {
test("failure_module content survives extract + replace", async () => {
const originalContent = 'export function main(error: any) {\n console.error(error);\n return { handled: true };\n}';
const failureModule = makeRawscriptModule(
"failure",
originalContent,
"bun",
);
// PULL: extract inline scripts (mutates module in place)
const extracted = extractInlineScripts([failureModule], {}, "/", "bun");
expect(failureModule.value.content).toStartWith("!inline ");
// Build a virtual filesystem from extracted scripts
const files: Record<string, string> = {};
for (const s of extracted) {
files[s.path] = s.content;
}
// PUSH: replace inline references back
await replaceInlineScripts(
[failureModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(failureModule.value.content).toBe(originalContent);
});
test("preprocessor_module content survives extract + replace", async () => {
const originalContent = "def main():\n return {\"preprocessed\": True}";
const preprocessorModule = makeRawscriptModule(
"preprocessor",
originalContent,
"python3",
);
const extracted = extractInlineScripts(
[preprocessorModule],
{},
"/",
"bun",
);
expect(preprocessorModule.value.content).toStartWith("!inline ");
const files: Record<string, string> = {};
for (const s of extracted) {
files[s.path] = s.content;
}
await replaceInlineScripts(
[preprocessorModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(preprocessorModule.value.content).toBe(originalContent);
});
test("failure_module with lock survives extract + replace", async () => {
const originalContent = "export function main() { return 42; }";
const originalLock = "package-lock-contents-here";
const mod = makeRawscriptModule(
"failure",
originalContent,
"bun",
originalLock,
);
const extracted = extractInlineScripts([mod], {}, "/", "bun");
const files: Record<string, string> = {};
for (const s of extracted) {
files[s.path] = s.content;
}
await replaceInlineScripts(
[mod],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(mod.value.content).toBe(originalContent);
expect((mod.value as any).lock).toBe(originalLock);
});
test("full flow with modules + failure_module + preprocessor_module round-trips", async () => {
const regularContent = "export function main() { return 'step1'; }";
const failureContent = "export function main(e: any) { return e; }";
const preprocessorContent = "def main():\n pass";
const modules = [makeRawscriptModule("a", regularContent, "bun")];
const failureModule = makeRawscriptModule("failure", failureContent, "bun");
const preprocessorModule = makeRawscriptModule("preprocessor", preprocessorContent, "python3");
// Extract all (mimicking sync.ts pull logic)
const allExtracted = [
...extractInlineScripts(modules, {}, "/", "bun"),
...extractInlineScripts([failureModule], {}, "/", "bun"),
...extractInlineScripts([preprocessorModule], {}, "/", "bun"),
];
// All modules should now have !inline references
expect(modules[0].value.content).toStartWith("!inline ");
expect(failureModule.value.content).toStartWith("!inline ");
expect(preprocessorModule.value.content).toStartWith("!inline ");
// All paths should be unique
const paths = allExtracted.filter((s) => !s.is_lock).map((s) => s.path);
expect(new Set(paths).size).toBe(paths.length);
// Build filesystem
const files: Record<string, string> = {};
for (const s of allExtracted) {
files[s.path] = s.content;
}
const fileReader = async (path: string) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
};
// Replace all (mimicking flow.ts push logic)
await replaceInlineScripts(modules, fileReader, noopLogger, "/tmp/", "/");
await replaceInlineScripts([failureModule], fileReader, noopLogger, "/tmp/", "/");
await replaceInlineScripts([preprocessorModule], fileReader, noopLogger, "/tmp/", "/");
expect(modules[0].value.content).toBe(regularContent);
expect(failureModule.value.content).toBe(failureContent);
expect(preprocessorModule.value.content).toBe(preprocessorContent);
});
});
// ---------------------------------------------------------------------------
// extractCurrentMapping
// ---------------------------------------------------------------------------
describe("extractCurrentMapping for failure_module / preprocessor_module", () => {
test("extracts mapping from failure_module via optional param", () => {
const failureModule: FlowModule = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
);
const mapping = extractCurrentMapping(
undefined,
{},
failureModule,
undefined,
);
expect(mapping["failure"]).toBe("failure.inline_script.ts");
});
test("extracts mapping from preprocessor_module via optional param", () => {
const preprocessorModule: FlowModule = makeRawscriptModule(
"preprocessor",
"!inline preprocessor.inline_script.py",
"python3",
);
const mapping = extractCurrentMapping(
undefined,
{},
undefined,
preprocessorModule,
);
expect(mapping["preprocessor"]).toBe("preprocessor.inline_script.py");
});
test("extracts mapping from modules + failure + preprocessor combined", () => {
const modules: FlowModule[] = [
makeRawscriptModule("a", "!inline a.inline_script.ts", "bun"),
];
const failureModule = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
);
const preprocessorModule = makeRawscriptModule(
"preprocessor",
"!inline preprocessor.inline_script.py",
"python3",
);
const mapping = extractCurrentMapping(
modules,
{},
failureModule,
preprocessorModule,
);
expect(mapping["a"]).toBe("a.inline_script.ts");
expect(mapping["failure"]).toBe("failure.inline_script.ts");
expect(mapping["preprocessor"]).toBe("preprocessor.inline_script.py");
});
test("ignores non-inline content in failure_module", () => {
const failureModule = makeRawscriptModule(
"failure",
"export function main() {}",
"bun",
);
const mapping = extractCurrentMapping(
undefined,
{},
failureModule,
undefined,
);
expect(mapping["failure"]).toBeUndefined();
});
});

View File

@@ -3,13 +3,6 @@
*
* Tests the sync pull and push functionality with a simulated filesystem
* containing every kind of Windmill resource type.
*
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
* @see test_fixtures.ts - Shared local fixtures (prefer using this module for new tests)
* @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.)
*
* This file contains: Local fixtures (should migrate to test_fixtures.ts) + createRemoteScript
* If you add new helpers, update cross-links in the files above.
*/
import { expect, test, describe } from "bun:test";
@@ -44,13 +37,10 @@ import { newPathAssigner } from "../windmill-utils-internal/src/path-utils/path-
// =============================================================================
// Test Fixtures - Every Type of Windmill Resource
// See file header for cross-links to related helpers.
// Consider migrating these to test_fixtures.ts for reuse across tests.
// =============================================================================
/**
* Creates a mock script file structure.
* See file header for cross-links to related helpers.
* Creates a mock script file structure
*/
function createScriptFixture(
name: string,
@@ -99,8 +89,7 @@ kind: script
}
/**
* Creates a mock flow file structure.
* See file header for cross-links to related helpers.
* Creates a mock flow file structure
*/
function createFlowFixture(name: string): Record<string, { path: string; content: string }> {
const flowSuffix = getFolderSuffix("flow");
@@ -134,8 +123,7 @@ schema:
}
/**
* Creates a mock app file structure.
* See file header for cross-links to related helpers.
* Creates a mock app file structure
*/
function createAppFixture(name: string): Record<string, { path: string; content: string }> {
const appSuffix = getFolderSuffix("app");
@@ -163,8 +151,7 @@ policy:
}
/**
* Creates a mock raw_app file structure.
* See file header for cross-links to related helpers.
* Creates a mock raw_app file structure
*/
function createRawAppFixture(name: string): Record<string, { path: string; content: string }> {
const rawAppSuffix = getFolderSuffix("raw_app");
@@ -1933,7 +1920,7 @@ excludes: []
import type { TestBackend } from "./test_backend.ts";
/** Create a script on the remote via API. See file header for cross-links. */
/** Create a script on the remote via API */
async function createRemoteScript(
backend: TestBackend,
scriptPath: string,

View File

@@ -19,13 +19,6 @@
* // ...
* });
* });
*
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
* @see test_fixtures.ts - Local file fixtures (createLocalScript, createLocalFlow, etc.)
* @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based)
*
* This file contains: API-based creation helpers (createTestApp, createTestResource, etc.)
* If you add new helpers, update cross-links in the files above.
*/
import { CargoBackend, CargoBackendConfig } from "./cargo_backend.ts";
@@ -116,7 +109,6 @@ class CargoBackendAdapter implements TestBackend {
return this.backend.apiRequest(path, options);
}
/** Seeds test data via API calls. See file header for cross-links to related helpers. */
async seedTestData(): Promise<void> {
// Create test folder first
await this.createTestFolder("test");
@@ -132,7 +124,6 @@ class CargoBackendAdapter implements TestBackend {
await this.createTestApp("f/test/test_dashboard");
}
/** See file header for cross-links to related helpers. */
private async createTestApp(path: string): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, {
method: "POST",
@@ -165,7 +156,6 @@ class CargoBackendAdapter implements TestBackend {
}
}
/** See file header for cross-links to related helpers. */
private async createTestFolder(name: string): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/folders/create`, {
method: "POST",
@@ -182,7 +172,6 @@ class CargoBackendAdapter implements TestBackend {
}
}
/** See file header for cross-links to related helpers. */
private async createTestGroup(name: string): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/groups/create`, {
method: "POST",
@@ -200,7 +189,6 @@ class CargoBackendAdapter implements TestBackend {
}
/** See file header for cross-links to related helpers. */
private async createTestResource(path: string, description: string): Promise<void> {
// First ensure the folder exists
const folderPath = path.split("/").slice(0, 2).join("/"); // e.g., "f/test"
@@ -240,7 +228,6 @@ class CargoBackendAdapter implements TestBackend {
}
}
/** See file header for cross-links to related helpers. */
private async createTestVariable(path: string, value: string): Promise<void> {
const response = await this.backend.apiRequest(
`/api/w/${this.workspace}/variables/create`,

View File

@@ -1,531 +0,0 @@
/**
* Test Fixtures
*
* Shared helpers for creating test data (scripts, flows, apps, raw apps) in tests.
*
* Two types of helpers:
* - Fixture functions: Return data structures with paths and contents (no disk I/O)
* - Local creation functions: Create fixtures AND write them to disk
*
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
* @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.)
* @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based)
*
* This file contains: Shared local fixtures (createLocalScript, createLocalFlow, etc.)
* If you add new helpers, update cross-links in the files above.
*
* @example
* // Using fixtures (data only)
* const fixture = createScriptFixture("my_script", "bun");
*
* // Using local creation (writes to disk)
* await createLocalScript(tempDir, "f/test", "my_script", "bun");
*
* @keywords createLocal, local script, local flow, local app, raw app, fixture, test data
*/
import { writeFile, mkdir } from "node:fs/promises";
import {
getFolderSuffix,
getMetadataFileName,
} from "../src/utils/resource_folders.ts";
// =============================================================================
// Fixture Types
// =============================================================================
export interface FileFixture {
path: string;
content: string;
}
export interface ScriptFixture {
contentFile: FileFixture;
metadataFile: FileFixture;
}
export interface FlowFixture {
metadata: FileFixture;
inlineScript: FileFixture;
}
export interface AppFixture {
metadata: FileFixture;
}
export interface RawAppFixture {
metadata: FileFixture;
indexHtml: FileFixture;
indexJs: FileFixture;
[key: string]: FileFixture;
}
// =============================================================================
// Script Fixtures
// =============================================================================
/**
* Creates a script fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* Use this when you need fine-grained control over the script structure.
* For simple cases, use {@link createLocalScript} instead.
*
* @param name - Script name (without extension)
* @param language - Script language
* @param content - Optional custom script content
* @returns Script fixture with content and metadata files
*
* @example
* const fixture = createScriptFixture("my_script", "bun");
* const fixture = createScriptFixture("custom", "python3", "def main(): return 42");
*
* @keywords script fixture, create script, local script
*/
export function createScriptFixture(
name: string,
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
content?: string
): ScriptFixture {
const extensions: Record<string, string> = {
python3: ".py",
deno: ".ts",
bun: ".ts",
bash: ".sh",
go: ".go",
postgresql: ".sql",
};
const ext = extensions[language];
const defaultContent: Record<string, string> = {
python3: `def main():\n return "Hello from ${name}"`,
deno: `export async function main() {\n return "Hello from ${name}";\n}`,
bun: `export async function main() {\n return "Hello from ${name}";\n}`,
bash: `#!/bin/bash\necho "Hello from ${name}"`,
go: `package inner\n\nfunc main() string {\n return "Hello from ${name}"\n}`,
postgresql: `-- ${name}\nSELECT 'Hello from ${name}';`,
};
return {
contentFile: {
path: `${name}${ext}`,
content: content ?? defaultContent[language],
},
metadataFile: {
path: `${name}.script.yaml`,
content: `summary: "${name} script"
description: "A ${language} script for testing"
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
is_template: false
lock: ""
kind: script
`,
},
};
}
// =============================================================================
// Flow Fixtures
// =============================================================================
/**
* Creates a flow fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* TODO: Add optional params: language, summary, description
*
* Use this when you need fine-grained control over the flow structure.
* For simple cases, use {@link createLocalFlow} instead.
*
* @param name - Flow name
* @param inlineScriptContent - Optional custom inline script content
* @returns Flow fixture with metadata and inline script
*
* @example
* const fixture = createFlowFixture("my_flow");
*
* @keywords flow fixture, create flow, local flow
*/
export function createFlowFixture(
name: string,
inlineScriptContent?: string
): FlowFixture {
const flowSuffix = getFolderSuffix("flow");
const metadataFile = getMetadataFileName("flow", "yaml");
const scriptContent =
inlineScriptContent ??
`export async function main() {\n return "Hello from flow ${name}";\n}`;
return {
metadata: {
path: `${name}${flowSuffix}/${metadataFile}`,
content: `summary: "${name} flow"
description: "A flow for testing"
value:
modules:
- id: a
value:
type: rawscript
content: |
${scriptContent.split("\n").join("\n ")}
language: bun
input_transforms: {}
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
`,
},
inlineScript: {
path: `${name}${flowSuffix}/a.inline_script.ts`,
content: scriptContent,
},
};
}
// =============================================================================
// App Fixtures
// =============================================================================
/**
* Creates an app fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* TODO: Add optional params: inlineScriptContent, summary, grid
*
* Use this when you need fine-grained control over the app structure.
* For simple cases, use {@link createLocalApp} instead.
*
* @param name - App name
* @returns App fixture with metadata
*
* @example
* const fixture = createAppFixture("my_app");
*
* @keywords app fixture, create app, local app
*/
export function createAppFixture(name: string): AppFixture {
const appSuffix = getFolderSuffix("app");
const metadataFile = getMetadataFileName("app", "yaml");
return {
metadata: {
path: `${name}${appSuffix}/${metadataFile}`,
content: `summary: "${name} app"
value:
type: app
grid:
- id: button1
data:
type: buttoncomponent
componentInput:
type: runnable
runnable:
type: runnableByName
inlineScript:
content: |
export async function main() {
return "hello from app";
}
language: bun
hiddenInlineScripts: []
css: {}
norefreshbar: false
policy:
on_behalf_of: null
on_behalf_of_email: null
triggerables: {}
execution_mode: viewer
`,
},
};
}
// =============================================================================
// Raw App Fixtures
// =============================================================================
/**
* Creates a raw app fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* TODO: Add optional params: inlineScriptContent, htmlContent, jsContent
*
* Raw apps are React/frontend apps with separate inline scripts.
* Use this when you need fine-grained control over the raw app structure.
* For simple cases, use {@link createLocalRawApp} instead.
*
* @param name - Raw app name
* @returns Raw app fixture with metadata and frontend files
*
* @example
* const fixture = createRawAppFixture("my_raw_app");
*
* @keywords raw app fixture, create raw app, local raw app, react app
*/
export function createRawAppFixture(name: string): RawAppFixture {
const rawAppSuffix = getFolderSuffix("raw_app");
const metadataFile = getMetadataFileName("raw_app", "yaml");
return {
metadata: {
path: `${name}${rawAppSuffix}/${metadataFile}`,
content: `summary: "${name} raw app"
policy:
execution_mode: publisher
triggerables: {}
`,
},
indexHtml: {
path: `${name}${rawAppSuffix}/index.html`,
content: `<!DOCTYPE html>
<html>
<head><title>${name}</title></head>
<body><div id="root"></div></body>
</html>`,
},
indexJs: {
path: `${name}${rawAppSuffix}/index.tsx`,
content: `import React from 'react'
import { createRoot } from 'react-dom/client'
const App = () => <div><h1>${name}</h1></div>
const root = createRoot(document.getElementById('root')!)
root.render(<App/>)
`,
},
packageJson: {
path: `${name}${rawAppSuffix}/package.json`,
content: `{
"dependencies": {
"react": "19.0.0",
"react-dom": "19.0.0"
}
}`,
},
inlineScript: {
path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.ts`,
content: `export async function main(x: string) {
return x
}
`,
},
inlineScriptLock: {
path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.lock`,
content: ``,
},
};
}
// =============================================================================
// Local Creation Functions (Fixture + Write to Disk)
// =============================================================================
/**
* Creates a script on the local filesystem.
* See file header for cross-links to related helpers.
*
* This is a convenience function that creates a script fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - Script name (without extension)
* @param language - Script language (default: "bun")
* @param content - Optional custom script content
*
* @example
* await createLocalScript(tempDir, "f/test", "my_script");
* await createLocalScript(tempDir, "f/test", "custom", "python3", "def main(): return 42");
*
* @keywords create local script, local script, write script, script on disk
*/
export async function createLocalScript(
tempDir: string,
path: string,
name: string,
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
content?: string
): Promise<void> {
const fixture = createScriptFixture(name, language, content);
await mkdir(`${tempDir}/${path}`, { recursive: true });
await writeFile(
`${tempDir}/${path}/${fixture.contentFile.path}`,
fixture.contentFile.content,
"utf-8"
);
await writeFile(
`${tempDir}/${path}/${fixture.metadataFile.path}`,
fixture.metadataFile.content,
"utf-8"
);
}
/**
* Creates a flow on the local filesystem.
* See file header for cross-links to related helpers.
*
* This is a convenience function that creates a flow fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - Flow name
* @param inlineScriptContent - Optional custom inline script content
*
* @example
* await createLocalFlow(tempDir, "f/test", "my_flow");
*
* @keywords create local flow, local flow, write flow, flow on disk
*/
export async function createLocalFlow(
tempDir: string,
path: string,
name: string,
inlineScriptContent?: string
): Promise<void> {
const fixture = createFlowFixture(name, inlineScriptContent);
const flowDir = `${tempDir}/${path}/${name}${getFolderSuffix("flow")}`;
await mkdir(flowDir, { recursive: true });
for (const file of Object.values(fixture)) {
const fullPath = `${tempDir}/${path}/${file.path}`;
await writeFile(fullPath, file.content, "utf-8");
}
}
/**
* Creates an app on the local filesystem.
* See file header for cross-links to related helpers.
*
* This is a convenience function that creates an app fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - App name
*
* @example
* await createLocalApp(tempDir, "f/test", "my_app");
*
* @keywords create local app, local app, write app, app on disk
*/
export async function createLocalApp(
tempDir: string,
path: string,
name: string
): Promise<void> {
const fixture = createAppFixture(name);
const appDir = `${tempDir}/${path}/${name}${getFolderSuffix("app")}`;
await mkdir(appDir, { recursive: true });
for (const file of Object.values(fixture)) {
const fullPath = `${tempDir}/${path}/${file.path}`;
await writeFile(fullPath, file.content, "utf-8");
}
}
/**
* Creates a raw app on the local filesystem.
* See file header for cross-links to related helpers.
*
* Raw apps are React/frontend apps with separate inline scripts.
* This is a convenience function that creates a raw app fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - Raw app name
*
* @example
* await createLocalRawApp(tempDir, "f/test", "my_raw_app");
*
* @keywords create local raw app, local raw app, write raw app, raw app on disk, react app
*/
export async function createLocalRawApp(
tempDir: string,
path: string,
name: string
): Promise<void> {
const fixture = createRawAppFixture(name);
const rawAppSuffix = getFolderSuffix("raw_app");
const appDir = `${tempDir}/${path}/${name}${rawAppSuffix}`;
await mkdir(`${appDir}/inline_scripts`, { recursive: true });
for (const file of Object.values(fixture)) {
const fullPath = `${tempDir}/${path}/${file.path}`;
const dir = fullPath.substring(0, fullPath.lastIndexOf("/"));
await mkdir(dir, { recursive: true });
await writeFile(fullPath, file.content, "utf-8");
}
}
// =============================================================================
// Resource Fixtures (Variables, Resources, Schedules, etc.)
// =============================================================================
/**
* Creates a resource fixture.
*
* @keywords resource fixture, create resource
*/
export function createResourceFixture(
name: string,
resourceType: string,
value: Record<string, unknown>
): FileFixture {
return {
path: `${name}.resource.yaml`,
content: `resource_type: "${resourceType}"
value:
${Object.entries(value)
.map(([k, v]) => ` ${k}: ${JSON.stringify(v)}`)
.join("\n")}
`,
};
}
/**
* Creates a variable fixture.
*
* @keywords variable fixture, create variable
*/
export function createVariableFixture(
name: string,
value: string,
isSecret: boolean = false
): FileFixture {
return {
path: `${name}.variable.yaml`,
content: `value: "${value}"
is_secret: ${isSecret}
description: "Variable ${name} for testing"
`,
};
}
/**
* Creates a schedule fixture.
*
* @keywords schedule fixture, create schedule
*/
export function createScheduleFixture(
name: string,
scriptPath: string,
schedule: string = "0 * * * *"
): FileFixture {
return {
path: `${name}.schedule.yaml`,
content: `path: "${name}"
schedule: "${schedule}"
script_path: "${scriptPath}"
is_flow: false
args: {}
enabled: true
timezone: "UTC"
`,
};
}

View File

@@ -1,451 +0,0 @@
/**
* Unified generate-metadata Command Tests
*
* Tests the new unified `generate-metadata` command that processes
* scripts, flows, and apps together.
*/
import { expect, test, describe } from "bun:test";
import { withTestBackend } from "./test_backend.ts";
import { addWorkspace } from "../workspace.ts";
import { writeFile } from "node:fs/promises";
import {
createLocalScript,
createLocalFlow,
createLocalApp,
createLocalRawApp,
} from "./test_fixtures.ts";
/**
* Helper to set up a workspace with wmill.yaml
*/
async function setupWorkspace(backend: any, tempDir: string, workspaceName: string) {
const testWorkspace = {
remote: backend.baseUrl,
workspaceId: backend.workspace,
name: workspaceName,
token: backend.token
};
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes:
- "**"
excludes: []`, "utf-8");
}
// =============================================================================
// Main test: processes scripts, flows, and apps together
// =============================================================================
test("generate-metadata: processes scripts, flows, and apps together", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "unified_all_test");
// Create one of each type
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
await createLocalApp(tempDir, "f/test", "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"unified_all_test"
);
expect(result.code).toEqual(0);
// Should find stale items
expect(result.stdout).toContain("Found");
expect(result.stdout).toContain("stale metadata");
});
});
// =============================================================================
// Flag tests
// =============================================================================
describe("generate-metadata flags", () => {
test("--includes filters to specific paths", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "includes_test");
// Create two scripts in different folders
await createLocalScript(tempDir, "f/included", "script_a");
await createLocalScript(tempDir, "f/excluded", "script_b");
// Run with --includes to only process f/included
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "-i", "f/included/**"],
tempDir,
"includes_test"
);
expect(result.code).toEqual(0);
// Should only mention the included script
const output = result.stdout + result.stderr;
expect(output).toContain("script_a");
expect(output).not.toContain("script_b");
});
});
test("--excludes filters out specific paths", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "excludes_test");
// Create two scripts
await createLocalScript(tempDir, "f/keep", "script_keep");
await createLocalScript(tempDir, "f/skip", "script_skip");
// Run with --excludes to skip f/skip
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "-e", "f/skip/**"],
tempDir,
"excludes_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("script_keep");
expect(output).not.toContain("script_skip");
});
});
test("--dry-run shows stale items without updating", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "dry_run_test");
await createLocalScript(tempDir, "f/test", "my_script");
// Run with --dry-run
const result = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"dry_run_test"
);
expect(result.code).toEqual(0);
// Should show stale items (Scripts section header)
expect(result.stdout).toContain("Scripts");
expect(result.stdout).toContain("my_script");
// Should NOT show "Done" (didn't actually update)
expect(result.stdout).not.toContain("Done");
// Run again without --dry-run to verify it would still be stale
const result2 = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"dry_run_test"
);
expect(result2.stdout).toContain("Scripts");
});
});
test("--lock-only only regenerates locks", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "lock_only_test");
await createLocalScript(tempDir, "f/test", "my_script");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--lock-only"],
tempDir,
"lock_only_test"
);
expect(result.code).toEqual(0);
});
});
test("--schema-only only processes scripts (skips flows and apps)", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "schema_only_test");
// Create one of each type
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
await createLocalApp(tempDir, "f/test", "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--schema-only"],
tempDir,
"schema_only_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should show "Checking scripts..." only
expect(output).toContain("Checking scripts...");
// Should find the script (Scripts section header)
expect(output).toContain("Scripts");
// Should NOT find flows or apps
expect(output).not.toContain("Flows");
expect(output).not.toContain("Apps");
});
});
test("--skip-scripts skips scripts", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_scripts_test");
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--skip-scripts"],
tempDir,
"skip_scripts_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should NOT contain script
expect(output).not.toContain("Scripts");
// Should contain flow
expect(output).toContain("Flows");
});
});
test("--skip-flows skips flows", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_flows_test");
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--skip-flows"],
tempDir,
"skip_flows_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should contain script
expect(output).toContain("Scripts");
// Should NOT contain flow
expect(output).not.toContain("Flows");
});
});
test("--skip-apps skips apps", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_apps_test");
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalApp(tempDir, "f/test", "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--skip-apps"],
tempDir,
"skip_apps_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should contain script
expect(output).toContain("Scripts");
// Should NOT contain app
expect(output).not.toContain("Apps");
});
});
test("shows 'All metadata up-to-date' when nothing to update", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "uptodate_test");
// Create a script and run generate-metadata twice
await createLocalScript(tempDir, "f/test", "my_script");
// First run - generates metadata
await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"uptodate_test"
);
// Second run - should be up-to-date
const result = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"uptodate_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("up-to-date");
});
});
test("skipping all types shows warning", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_all_test");
await createLocalScript(tempDir, "f/test", "my_script");
const result = await backend.runCLICommand(
["generate-metadata", "--skip-scripts", "--skip-flows", "--skip-apps"],
tempDir,
"skip_all_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("Nothing to check");
});
});
});
// =============================================================================
// Folder argument tests
// =============================================================================
describe("generate-metadata folder argument", () => {
test("filters to specific script folder", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_script_test");
// Create scripts in different folders
await createLocalScript(tempDir, "f/included", "script_a");
await createLocalScript(tempDir, "f/excluded", "script_b");
// Run with folder argument
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/included/script_a.ts"],
tempDir,
"folder_script_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("script_a");
expect(output).not.toContain("script_b");
});
});
test("filters to specific flow folder", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_flow_test");
// Create flows in different folders
await createLocalFlow(tempDir, "f/included", "flow_a");
await createLocalFlow(tempDir, "f/excluded", "flow_b");
// Run with folder argument (flow folder path - uses .flow suffix by default)
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/included/flow_a.flow"],
tempDir,
"folder_flow_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("flow_a");
expect(output).not.toContain("flow_b");
});
});
test("filters to specific app folder", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_app_test");
// Create apps in different folders
await createLocalApp(tempDir, "f/included", "app_a");
await createLocalApp(tempDir, "f/excluded", "app_b");
// Run with folder argument (app folder path - uses .app suffix by default)
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/included/app_a.app"],
tempDir,
"folder_app_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("app_a");
expect(output).not.toContain("app_b");
});
});
test("shows up-to-date when folder has no stale items", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_uptodate_test");
await createLocalScript(tempDir, "f/test", "my_script");
// First run to generate metadata
await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"folder_uptodate_test"
);
// Second run with folder - should be up-to-date
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/test/my_script.ts"],
tempDir,
"folder_uptodate_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("up-to-date");
});
});
test("trailing slash is stripped (matches deprecated behavior)", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "trailing_slash_test");
await createLocalScript(tempDir, "f/test", "my_script");
// Run with trailing slash
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/test/my_script.ts/"],
tempDir,
"trailing_slash_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("my_script");
});
});
test("parent folder matches all children", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "parent_folder_test");
// Create scripts in nested folders
await createLocalScript(tempDir, "f/parent", "script_a");
await createLocalScript(tempDir, "f/parent/child", "script_b");
await createLocalScript(tempDir, "f/other", "script_c");
// Run with parent folder - should match both scripts in f/parent tree
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/parent"],
tempDir,
"parent_folder_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("script_a");
expect(output).toContain("script_b");
expect(output).not.toContain("script_c");
});
});
test("non-existent folder shows up-to-date", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "nonexistent_folder_test");
await createLocalScript(tempDir, "f/exists", "my_script");
// Run with non-existent folder
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/does_not_exist"],
tempDir,
"nonexistent_folder_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("up-to-date");
});
});
});

View File

@@ -150,17 +150,8 @@ export function extractInlineScripts(
*/
export function extractCurrentMapping(
modules: FlowModule[] | undefined,
mapping: Record<string, string> = {},
failureModule?: FlowModule,
preprocessorModule?: FlowModule,
mapping: Record<string, string> = {}
): Record<string, string> {
if (failureModule) {
extractCurrentMapping([failureModule], mapping);
}
if (preprocessorModule) {
extractCurrentMapping([preprocessorModule], mapping);
}
if (!modules || !Array.isArray(modules)) {
return mapping;
}

View File

@@ -67,7 +67,7 @@ RUN curl -fsSL https://claude.ai/install.sh | bash \
&& cp /root/.local/share/claude/versions/* /usr/bin/claude
# add the docker client to call docker from a worker if enabled
COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
# nsjail runtime deps and binary
RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \

View File

@@ -67,7 +67,7 @@ RUN curl -fsSL https://claude.ai/install.sh | bash \
&& cp /root/.local/share/claude/versions/* /usr/bin/claude
# add the docker client to call docker from a worker if enabled
COPY --from=docker:29-dind /usr/local/bin/docker /usr/local/bin/
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
# nsjail runtime deps and binary
RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \

Some files were not shown because too many files have changed in this diff Show More