Compare commits

..

7 Commits

Author SHA1 Message Date
Ruben Fiszel
7882e355eb tests 2025-02-24 12:25:52 +01:00
Ruben Fiszel
0b6d130efd all 2025-02-24 02:05:32 +01:00
Ruben Fiszel
31a8f24734 all 2025-02-24 02:04:30 +01:00
Ruben Fiszel
34587fea42 all 2025-02-24 02:02:20 +01:00
Ruben Fiszel
0577f999c2 all 2025-02-24 01:45:34 +01:00
Ruben Fiszel
25a49b74f9 all 2025-02-24 01:38:30 +01:00
Ruben Fiszel
1df9457bb1 all 2025-02-24 01:33:33 +01:00
480 changed files with 8822 additions and 18897 deletions

6
.github/CODEOWNERS vendored
View File

@@ -1,4 +1,4 @@
* @rubenfiszel @HugoCasa @alpetric
* @rubenfiszel
/community/ @rubenfiszel @HugoCasa @alpetric
/frontend/ @rubenfiszel @HugoCasa @alpetric
/community/ @fatonramadani @rubenfiszel
/frontend/ @fatonramadani @rubenfiszel

View File

@@ -40,7 +40,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.2.3 /usr/local/bin/bun /usr/bin/bun
ARG TARGETPLATFORM

View File

@@ -16,7 +16,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.85.0
toolchain: 1.83.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
@@ -40,7 +40,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.85.0
toolchain: 1.83.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
@@ -77,7 +77,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.85.0
toolchain: 1.83.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend
@@ -117,7 +117,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.85.0
toolchain: 1.83.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend

View File

@@ -51,7 +51,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
toolchain: 1.85.0
toolchain: 1.83.0
- uses: Swatinem/rust-cache@v2
with:
workspaces: backend

View File

@@ -8,317 +8,22 @@ on:
jobs:
benchmark_single:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
--shm-size=2g
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 30
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_single
path: |
*.json
benchmark_dedicated:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: dedicated
DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
options: >-
--pull always --restart unless-stopped --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 20
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
--no-warm-up -c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_dedicated
path: |
*.json
benchmark_4workers:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
windmill_1:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_2:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_3:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 20
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
--workers 4
--factor 3
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_4workers
path: |
*.json
benchmark_8workers:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
windmill_1:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_2:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_3:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_4:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_5:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_6:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_7:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 20
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
--workers 8
--factor 3
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_8workers
path: |
*.json
benchmark_graphs:
runs-on: ubicloud
needs:
- benchmark_single
- benchmark_dedicated
- benchmark_4workers
- benchmark_8workers
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- uses: actions/checkout@v4
with:
ref: benchmarks
- name: Download benchmark results
uses: actions/download-artifact@v4
with:
merge-multiple: true
- name: graphs
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json
- name: Push changes
run: |
ls -la
pwd
git add .
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git commit -m "Update benchmarks"
git push
docker run -d --network=host -e POSTGRES_PASSWORD=changeme -e POSTGRES_USER=postgres -e POSTGRES_DB=windmill -e POSTGRES_INITDB_ARGS="-c log_duration=on -c log_statement=all -c log_min_duration_statement=0 -c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB -c shared_preload_libraries=auto_explain -c auto_explain.log_min_duration=5 -c auto_explain.log_analyze=on -c auto_explain.log_timing=on -c auto_explain.log_buffers=on -c auto_explain.log_verbose=on \
-c log_statement=all \
-c log_min_duration_statement=0 \
-c shared_buffers=2GB \
-c work_mem=32MB \
-c effective_cache_size=4GB" \
postgres
sleep 5
docker run -d -it --network=host -e DATABASE_URL=postgres://postgres:changeme@localhost/windmill ghcr.io/windmill-labs/windmill:main
sleep 10
deno run --unstable -A -r https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json

View File

@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}

View File

@@ -30,12 +30,6 @@ jobs:
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: 1.85.0
override: true
- name: Substitute EE code
shell: bash
run: |
@@ -51,7 +45,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
- name: Rename binary with corresponding architecture
run: |

View File

@@ -67,7 +67,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=embedding,parquet,openidconnect,deno_core,license,http_trigger,zip,oauth2,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
${{ steps.meta-public.outputs.tags }}

View File

@@ -95,12 +95,13 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=embedding,parquet,openidconnect,jemalloc,deno_core,license,http_trigger,zip,oauth2,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
labels: |
${{ steps.meta-public.outputs.labels }}
org.opencontainers.image.licenses=AGPLv3
build_ee:
runs-on: ubicloud
@@ -157,7 +158,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}

View File

@@ -32,12 +32,6 @@ jobs:
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
- name: Setup Rust
uses: actions-rs/toolchain@v1
with:
toolchain: 1.85.0
override: true
- name: Substitute EE code
shell: bash
run: |
@@ -53,7 +47,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,mqtt_trigger,websocket,python,smtp,csharp,static_frontend,rust
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
- name: Rename binary with corresponding architecture
run: |

View File

@@ -1,202 +1,5 @@
# Changelog
## [1.477.1](https://github.com/windmill-labs/windmill/compare/v1.477.0...v1.477.1) (2025-03-13)
### Bug Fixes
* fix rusttls panic ([6a6b760](https://github.com/windmill-labs/windmill/commit/6a6b760e321fae949a02c1b0e0c32b0beaa8693b))
## [1.477.0](https://github.com/windmill-labs/windmill/compare/v1.476.0...v1.477.0) (2025-03-12)
### Features
* add search by args on input history directly ([593dc30](https://github.com/windmill-labs/windmill/commit/593dc30bc81ab407bd119963a6befaa4fbc16eae))
### Bug Fixes
* add setValue support for tables ([ec52476](https://github.com/windmill-labs/windmill/commit/ec5247645d425a35b5adf0aaed40713d08439b11))
* improve oneOf arg input reactivity to value changes ([a695621](https://github.com/windmill-labs/windmill/commit/a6956215eca8d1180b3c999519f9fa2ef43b5ab0))
* pg_listeners have no timeout ([52f55ff](https://github.com/windmill-labs/windmill/commit/52f55ff1f11adf9157ca0f0fe356fa17d65ea20a))
* prevent monitoring task to die without sending killpill ([#5472](https://github.com/windmill-labs/windmill/issues/5472)) ([d58ca9b](https://github.com/windmill-labs/windmill/commit/d58ca9b395cb151b05c43d910b0081988b3291ae))
* tutorial's step 6 not working (button.click is not a function) ([#5474](https://github.com/windmill-labs/windmill/issues/5474)) ([00e1841](https://github.com/windmill-labs/windmill/commit/00e18419f5db8ef19ad92c6ac290812084cd1ecd))
* update bun to 1.2.4 ([8e0963e](https://github.com/windmill-labs/windmill/commit/8e0963eec8a86b6d8593995c803dfbdd2c96bfc1))
## [1.476.0](https://github.com/windmill-labs/windmill/compare/v1.475.1...v1.476.0) (2025-03-11)
### Features
* option to prefix http route with workspace id ([#5461](https://github.com/windmill-labs/windmill/issues/5461)) ([61a5cea](https://github.com/windmill-labs/windmill/commit/61a5ceaba38787dc146a36b443bbd3f78e26102b))
### Bug Fixes
* cache for querying scripts correclty handles ScriptMetadata ([#5466](https://github.com/windmill-labs/windmill/issues/5466)) ([6dd2502](https://github.com/windmill-labs/windmill/commit/6dd2502d70dffcadee4427164db02607cd109c61))
* codebases compatible with git sync ([#5470](https://github.com/windmill-labs/windmill/issues/5470)) ([bd7586a](https://github.com/windmill-labs/windmill/commit/bd7586a5eec5516fe291070303fa6516d8adc8de))
## [1.475.1](https://github.com/windmill-labs/windmill/compare/v1.475.0...v1.475.1) (2025-03-11)
### Bug Fixes
* improve arginput sql and object viewer args change ([2a8a756](https://github.com/windmill-labs/windmill/commit/2a8a756b3f0a0e69145421eee87251956d85403b))
* improve flow status viewer iteration picker behavior with very large forloops ([78d9664](https://github.com/windmill-labs/windmill/commit/78d9664ad89212196ef32c0a02114092331bfe63))
## [1.475.0](https://github.com/windmill-labs/windmill/compare/v1.474.0...v1.475.0) (2025-03-06)
### Features
* **backend:** option to invalidate all sessions on logout ([#5419](https://github.com/windmill-labs/windmill/issues/5419)) ([e9044f0](https://github.com/windmill-labs/windmill/commit/e9044f0b9b1647e0fc74e5e3cce39a4fc2718672))
* deploy triggers to prod/staging workspace ([#5429](https://github.com/windmill-labs/windmill/issues/5429)) ([b210ae3](https://github.com/windmill-labs/windmill/commit/b210ae36f7c1fd8860241aa5908cefc0ac956b7b))
* **frontend:** improve flow suspend status display ([#5425](https://github.com/windmill-labs/windmill/issues/5425)) ([a845733](https://github.com/windmill-labs/windmill/commit/a8457337cec870e9e3d6a053f4fbe89f58229401))
* **frontend:** pick image from workspace storage bucket ([#5382](https://github.com/windmill-labs/windmill/issues/5382)) ([8dbe0fa](https://github.com/windmill-labs/windmill/commit/8dbe0fa6446a34bd60484f1b5ac828ff9f892735))
* kafka mTLS ([#5449](https://github.com/windmill-labs/windmill/issues/5449)) ([371c892](https://github.com/windmill-labs/windmill/commit/371c892f9aa43fe2757b4d23ee076781b237f11f))
* MQTT triggers ([#5277](https://github.com/windmill-labs/windmill/issues/5277)) ([5c39037](https://github.com/windmill-labs/windmill/commit/5c39037aea35f9bdf780f1946abbb384533ee547))
### Bug Fixes
* **cli:** fix wmill user create-token with email and password ([a16cab0](https://github.com/windmill-labs/windmill/commit/a16cab0923f3be46f181dffc209a10c711873f86))
* **frontend:** fix many s3 file picker bugs ([#5428](https://github.com/windmill-labs/windmill/issues/5428)) ([4fabc2a](https://github.com/windmill-labs/windmill/commit/4fabc2a8256b5088b1af61baf81355cd556d23e2))
* **frontend:** improve capture payload preview ([#5417](https://github.com/windmill-labs/windmill/issues/5417)) ([fd56a63](https://github.com/windmill-labs/windmill/commit/fd56a639d21366bea5129409d805feefb22659c1))
* improve objectviewer performance ([2444f4f](https://github.com/windmill-labs/windmill/commit/2444f4f23e5212fa4d2ff8542d864118d4b4feb9))
* s3 file picker delete + better s3 path handling ([#5454](https://github.com/windmill-labs/windmill/issues/5454)) ([ae618c7](https://github.com/windmill-labs/windmill/commit/ae618c79dff08e8e9fe1f4ce69665ad7faeac006))
## [1.474.0](https://github.com/windmill-labs/windmill/compare/v1.473.1...v1.474.0) (2025-03-04)
### Features
* add template script for all triggers ([#5424](https://github.com/windmill-labs/windmill/issues/5424)) ([0a9d8c6](https://github.com/windmill-labs/windmill/commit/0a9d8c6b8b95c0e1093aef89b2a1956dbe59bbee))
* **frontend:** global recompute helper function ([#5408](https://github.com/windmill-labs/windmill/issues/5408)) ([b961efa](https://github.com/windmill-labs/windmill/commit/b961efa8691f806d437ba2cb303ad1a50fa618c4))
* more controls on setting token duration ([#5421](https://github.com/windmill-labs/windmill/issues/5421)) ([534a824](https://github.com/windmill-labs/windmill/commit/534a8249d60f7de6b542c3b5b5ee0b5eda360f22))
### Bug Fixes
* do not depend on public schema anymore ([90b00f5](https://github.com/windmill-labs/windmill/commit/90b00f55011288115e00b7a30b3e8e91bc0b7f4b))
* **python:** windows worker fails to install 3.10 ([#5409](https://github.com/windmill-labs/windmill/issues/5409)) ([ebb58e0](https://github.com/windmill-labs/windmill/commit/ebb58e0dc7cc104e0bbfd90cf7f37e40ffd0bbf5))
## [1.473.1](https://github.com/windmill-labs/windmill/compare/v1.473.0...v1.473.1) (2025-03-03)
### Bug Fixes
* **backend:** copilot info exists_ai_resource ([#5415](https://github.com/windmill-labs/windmill/issues/5415)) ([844edd1](https://github.com/windmill-labs/windmill/commit/844edd1117bc5fdcc108adf53aec3974a7c66384))
* improve cancel performance ([fba9e7e](https://github.com/windmill-labs/windmill/commit/fba9e7ef03b91d3e2c78ff833ddcc5207b7436d2))
## [1.473.0](https://github.com/windmill-labs/windmill/compare/v1.472.1...v1.473.0) (2025-03-03)
### Features
* app s3 input anonymous delete ([#5401](https://github.com/windmill-labs/windmill/issues/5401)) ([46c7845](https://github.com/windmill-labs/windmill/commit/46c784574add176cb75d3627c9d3f55b6fb945f8))
* track workspace runnables used in flows ([#5369](https://github.com/windmill-labs/windmill/issues/5369)) ([7bf9e25](https://github.com/windmill-labs/windmill/commit/7bf9e25ede82486115eae71865202f85aa931a8d))
### Bug Fixes
* improve db loads by adding index on audit ([e1ff001](https://github.com/windmill-labs/windmill/commit/e1ff00117ca5b66dd0b9365e63b9ff7dc277bc2c))
* migrations do not refer to public schema anymore ([#5400](https://github.com/windmill-labs/windmill/issues/5400)) ([3063001](https://github.com/windmill-labs/windmill/commit/3063001491b49a4b6d0cd5985818b32aa4d3f16f))
* remove typings_extensions from python sdk ([04ffbf8](https://github.com/windmill-labs/windmill/commit/04ffbf8c266a06c3efcebcbbaee767f0ab0771e2))
## [1.472.1](https://github.com/windmill-labs/windmill/compare/v1.472.0...v1.472.1) (2025-02-26)
### Bug Fixes
* disable bundling using env var ([#5396](https://github.com/windmill-labs/windmill/issues/5396)) ([cb559d6](https://github.com/windmill-labs/windmill/commit/cb559d6083553c400e18e6077002c4891289a8a2))
## [1.472.0](https://github.com/windmill-labs/windmill/compare/v1.471.1...v1.472.0) (2025-02-26)
### Bug Fixes
* downgrade v8 to fix some rare panics ([5569e4d](https://github.com/windmill-labs/windmill/commit/5569e4d4953a01f2ad03ea8b71e695e833964bea))
* **frontend:** markdown shows single backtick in single line code block ([#5391](https://github.com/windmill-labs/windmill/issues/5391)) ([7f290bb](https://github.com/windmill-labs/windmill/commit/7f290bbf6a33e2811dbe2bd8ee905c0fa8e8db3b))
* migrate toggle to melt (4/4) ([#5329](https://github.com/windmill-labs/windmill/issues/5329)) ([69fc8a9](https://github.com/windmill-labs/windmill/commit/69fc8a98ae78bc01dc3d97f9732ee28864b323dd))
## [1.471.1](https://github.com/windmill-labs/windmill/compare/v1.471.0...v1.471.1) (2025-02-26)
### Bug Fixes
* update to rust 1.86.0 ([3ada264](https://github.com/windmill-labs/windmill/commit/3ada264c4ad49f666c3a053eb48c7df294bf085b))
## [1.471.0](https://github.com/windmill-labs/windmill/compare/v1.470.1...v1.471.0) (2025-02-26)
### Features
* add support for claude sonnet 3.7 thinking ([#5387](https://github.com/windmill-labs/windmill/issues/5387)) ([487d84b](https://github.com/windmill-labs/windmill/commit/487d84bd7fdc39a2401df4108fca6183189cf38a))
### Bug Fixes
* **frontend:** improve pagination handling and filter refreshes ([#5378](https://github.com/windmill-labs/windmill/issues/5378)) ([a85ebfb](https://github.com/windmill-labs/windmill/commit/a85ebfbbf48590812c0931ad93179c322f819849))
## [1.470.1](https://github.com/windmill-labs/windmill/compare/v1.470.0...v1.470.1) (2025-02-26)
### Bug Fixes
* multiple app initializations fixes ([630e54f](https://github.com/windmill-labs/windmill/commit/630e54f65c950ec0073b3cdac9974cb666c1ab3f))
## [1.470.0](https://github.com/windmill-labs/windmill/compare/v1.469.0...v1.470.0) (2025-02-26)
### Features
* **frontend:** set default app refesh interval ([#5380](https://github.com/windmill-labs/windmill/issues/5380)) ([478d3fb](https://github.com/windmill-labs/windmill/commit/478d3fbf4a7e52d19fcb5cf8d601b2eeb3487716))
### Bug Fixes
* multiple app initializations fixes ([24b6003](https://github.com/windmill-labs/windmill/commit/24b600378025632aecb2ca898b63d6032e08eb2e))
## [1.469.0](https://github.com/windmill-labs/windmill/compare/v1.468.0...v1.469.0) (2025-02-25)
### Features
* limit the number of times a job can be restarted (3) after loss of pings ([c8a9596](https://github.com/windmill-labs/windmill/commit/c8a959691c37350def37fe3eb9f24c6f7789960d))
* python-client now support mocked api via `WM_MOCKED_API_FILE` env ([#5372](https://github.com/windmill-labs/windmill/issues/5372)) ([50607c7](https://github.com/windmill-labs/windmill/commit/50607c7625e4a48fb397cff167b41bb6602716c0))
### Bug Fixes
* improve flow editor for vscode extension ([44b26d2](https://github.com/windmill-labs/windmill/commit/44b26d2ccec0c9dd65d1f53b057d031f841d7dba))
* improve infinite grid behavior ([56d1da7](https://github.com/windmill-labs/windmill/commit/56d1da78fd3424ae5b4abbb009c7437ea98765ef))
## [1.468.0](https://github.com/windmill-labs/windmill/compare/v1.467.1...v1.468.0) (2025-02-24)
### Features
* add audit logs scope filter in admins workspace ([#5352](https://github.com/windmill-labs/windmill/issues/5352)) ([b3e00b7](https://github.com/windmill-labs/windmill/commit/b3e00b7fdc3ad4c689fc30216accbed05822794c))
* add support for | None and Optional in python ([#5361](https://github.com/windmill-labs/windmill/issues/5361)) ([9736355](https://github.com/windmill-labs/windmill/commit/9736355d5f82615100212698c5537997e5a0de39))
* make flow lock deployment error visible in UI ([b8e6d0d](https://github.com/windmill-labs/windmill/commit/b8e6d0da79ca57b115e7cb0ccff9f5623b23f3f3))
### Bug Fixes
* add LOCALAPPDATA env variable to python execution on windows ([8806870](https://github.com/windmill-labs/windmill/commit/8806870b1bf67c2f77beaf04d986cf172c7b4bf4))
* fix confirmation modal check on deploy ([3028325](https://github.com/windmill-labs/windmill/commit/3028325615e2f7e5ee3d1b6278580121880db14f))
* **frontend:** make html app component content selectable ([#5359](https://github.com/windmill-labs/windmill/issues/5359)) ([f1c5b77](https://github.com/windmill-labs/windmill/commit/f1c5b77d7af8433905937d274b97c2d5cd6c1316))
* handle better forced value propagation in apps ([7c842c8](https://github.com/windmill-labs/windmill/commit/7c842c88bf5225b6bc39857109b1b1ba5f99d708))
* handle better optional chaining operator ([d45c1f6](https://github.com/windmill-labs/windmill/commit/d45c1f69d48a5ad93f4399ce0150bbff6fd4fa6b))
* improve app markdown rendering ([96597d3](https://github.com/windmill-labs/windmill/commit/96597d3d6b3d31298e5582a55e11e1d48edbf175))
* improve cancel/back behavior on editors ([0565981](https://github.com/windmill-labs/windmill/commit/05659816e722effcba27e71f855e819c606f8756))
* improve custom component rendering ([4ee4ff7](https://github.com/windmill-labs/windmill/commit/4ee4ff78d389d61c63952c84dee967113c783c45))
* improve webhook settings cache invalidation ([0456272](https://github.com/windmill-labs/windmill/commit/0456272e3f36996c5f223fc332b150c7a64c2f05))
* update bun t.1.43->1.2.3 ([4e477d1](https://github.com/windmill-labs/windmill/commit/4e477d1f589343980d7bd2953909ff6a6be30739))
* update deno 2.1.2->2.2.1 ([b102ff4](https://github.com/windmill-labs/windmill/commit/b102ff4a4643e2f06d44d493f2f776b44ae721cc))
## [1.467.1](https://github.com/windmill-labs/windmill/compare/v1.467.0...v1.467.1) (2025-02-22)

View File

@@ -1,5 +1,5 @@
ARG DEBIAN_IMAGE=debian:bookworm-slim
ARG RUST_IMAGE=rust:1.85-slim-bookworm
ARG RUST_IMAGE=rust:1.83-slim-bookworm
FROM ${RUST_IMAGE} AS rust_base
@@ -191,7 +191,7 @@ COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.2.4 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.2.3 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer

20
LICENSE
View File

@@ -1,6 +1,6 @@
Source code in this repository is variously licensed under the Apache License
Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
./LICENSE-AGPL) and a proprietary license for certain enterprise features.
./LICENSE-AGPL)
Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
specified. Every file is under License AGPL unless otherwise specified or
@@ -12,23 +12,11 @@ and commercial license. The files under frontend/ are AGPLv3 Licensed, except
any snippets of code that require a positive license check to be activated.
Those snippets and files are under a proprietary and commercial license. Private
and public forks MUST not include any of the above proprietary and commercial
code. The files under python-client/ deno-client/ go-client/ powershell-client/
are Apache 2.0 Licensed. The openapi files, including the OpenFlow spec is
code. Windmill Labs, Inc. provide tools to clean the codebase from those
snippets upon demand. The files under python-client/ deno-client/ go-client/ powershell-client/ are
Apache 2.0 Licensed.
The binary compilable from source code in this repository without the
"enterprise" feature flag is open-source under the AGPLv3 License terms and
conditions.
The "Community Edition" of Windmill available in the docker images hosted under
ghcr.io/windmill-labs/windmill and the github binary releases contains the files
under the AGPLv3 and Apache 2 sources but also includes proprietary and
non-public code and features which are not open source and under the following
terms: Windmill Labs, Inc. grants a right to use all the features of the
"Community Edition" for free without restrictions other than the limits and
quotas set in the software and a right to distribute the community edition as is
but not to sell, resell, serve as a managed service, modify or wrap under any
form without an explicit agreement.
The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
All third party components incorporated into the Windmill Software are licensed
under the original license provided by the owner of the applicable component.

150
README.md
View File

@@ -110,8 +110,8 @@ You can build your entire infra on top of Windmill!
```typescript
//import any dependency from npm
import * as wmill from "windmill-client";
import * as cowsay from "cowsay@1.5.0";
import * as wmill from "windmill-client"
import * as cowsay from 'cowsay@1.5.0';
// fill the type, or use the +Resource type to get a type-safe reference to a resource
type Postgresql = {
@@ -146,9 +146,7 @@ export async function main(
## CLI
We have a powerful CLI to interact with the windmill platform and sync your
scripts from local files, GitHub repos and to run scripts and flows on the
instance from local commands. See
We have a powerful CLI to interact with the windmill platform and sync your scripts from local files, GitHub repos and to run scripts and flows on the instance from local commands. See
[more details](https://www.windmill.dev/docs/advanced/cli).
![CLI Screencast](./cli/vhs/output/setup.gif)
@@ -170,8 +168,7 @@ Code extension: <https://www.windmill.dev/docs/cli_local_dev/vscode-extension>.
Architecture:
- Stateless API backend.
- Workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
interested).
Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if interested).
- Frontend in Svelte.
- Scripts executions are sandboxed using Google's
[nsjail](https://github.com/google/nsjail).
@@ -287,37 +284,22 @@ edition.
### Commercial license
See the [LICENSE](https://github.com/windmill-labs/windmill/blob/main/LICENSE)
file for the full license text.
The "Community Edition" of Windmill available in the docker images hosted under
ghcr.io/windmill-labs/windmill and the github binary releases contains the files
under the AGPLv3 and Apache 2 sources but also includes proprietary and
non-public code and features which are not open source and under the following
terms: Windmill Labs, Inc. grants a right to use all the features of the
"Community Edition" for free without restrictions other than the limits and
quotas set in the software and a right to distribute the community edition as is
but not to sell, resell, serve Windmill as a managed service, modify or wrap
under any form without an explicit agreement.
The binary compilable from source code in this repository without the
"enterprise" feature flag is open-source under the
[LICENSE-AGPLv3](https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL)
License terms and conditions.
To self-host Windmill, you must respect the terms of the
[AGPLv3 license](https://www.gnu.org/licenses/agpl-3.0.en.html) which you do not
need to worry about for personal uses. For business uses, you should be fine if
you do not re-expose Windmill in any way to your users and are comfortable with
AGPLv3.
To
[re-expose directly any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling)
as a feature of your product, with the exception of iframed public Windmill
"apps", or to build a feature on top of "Windmill Community Edition" that you
sell commercially or embed in a distributable product or binary, you must get a
commercial license. Contact us at <sales@windmill.dev> if you have any
questions. To do the same from the binary compiled from the source code in this
repository without the "enterprise" feature flag, you must comply with the
AGPLv3 license terms and conditions or get a commercial license from Windmill
Labs, Inc.
[re-expose any Windmill parts to your users](https://www.windmill.dev/docs/misc/white_labelling)
as a feature of your product, or to build a feature on top of Windmill, to
comply with AGPLv3 your product must be AGPLv3 or you must get a commercial
license. Contact us at <ruben@windmill.dev> if you have any doubts.
To use Windmill "Community Edition" as is internally in your organization, or to
use its APIs as is, you do NOT need a commercial license.
In addition, a commercial license grants you a dedicated engineer to transition
your current infrastructure to Windmill, support with tight SLA, and our global
cache sync for high-performance/no dependency cache miss of cluster from 10+
nodes to 200+ nodes.
### Integrations
@@ -332,76 +314,70 @@ you to have it being synced automatically everyday.
## Environment Variables
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ----------------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
| DATABASE_URL | | The Postgres database url. | All |
| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker |
| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server, agent | All |
| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server |
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | | The path to the python binary if wanting to not have it managed by uv. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| GOPROXY | | The GOPROXY env variable to use | Worker |
| NETRC | | The netrc content to use a private go registry | Worker |
| PY_CONCURRENT_DOWNLOADS | 20 | Sets the maximum number of in-flight concurrent python downloads that windmill will perform at any given time. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
| DISABLE_RESPONSE_LOGS | false | Disable response logs | Server |
| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmins can create new workspaces | Server |
| Environment Variable name | Default | Description | Api Server/Worker/All |
| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
| DATABASE_URL | | The Postgres database url. | All |
| WORKER_GROUP | default | The worker group the worker belongs to and get its configuration pulled from | Worker |
| MODE | standalone | The mode if the binary. Possible values: standalone, worker, server, agent | All |
| METRICS_ADDR | None | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance. Is overriden by the instance settings if any. | Server |
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | | The path to the python binary if wanting to not have it managed by uv. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| GOPROXY | | The GOPROXY env variable to use | Worker |
| NETRC | | The netrc content to use a private go registry | Worker | | Worker |
| PY_CONCURRENT_DOWNLOADS | 20 | Sets the maximum number of in-flight concurrent python downloads that windmill will perform at any given time. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
| DISABLE_RESPONSE_LOGS | false | Disable response logs | Server |
| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmins can create new workspaces | Server |
## Run a local dev setup
See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
running options.
Using [Nix](./frontend/README_DEV.md#nix).
### only Frontend
This will use the backend of <https://app.windmill.dev> but your own frontend
with hot-code reloading. Note that you will need to use a username / password
login due to CSRF checks using a different auth provider.
with hot-code reloading. Note that you will need to use a username / password login due to CSRF checks using a different auth provider.
In the `frontend/` directory:
1. install the dependencies with `npm install` (or `pnpm install` or `yarn`)
2. generate the windmill client:
```
npm run generate-backend-client
## on mac use
npm run generate-backend-client-mac
```
```
npm run generate-backend-client
## on mac use
npm run generate-backend-client-mac
```
3. Run your dev server with `npm run dev`
4. Et voilà, windmill should be available at `http://localhost/`
### Backend + Frontend
See the [./frontend/README_DEV.md](./frontend/README_DEV.md) file for all
running options.
1. Create a Postgres Database for Windmill and create an admin role inside your
Postgres setup. The easiest way to get a working db is to run
```
Postgres setup.
The easiest way to get a working db is to run
```
cargo install sqlx-cli
env DATABASE_URL=<YOUR_DATABASE_URL> sqlx migrate run
```
```
This will also avoid compile time issue with sqlx's `query!` macro
2. Install [nsjail](https://github.com/google/nsjail) and have it accessible in
your PATH
@@ -411,15 +387,15 @@ running options.
5. Install the [lld linker](https://lld.llvm.org/)
6. Go to `frontend/`:
1. `npm install`, `npm run generate-backend-client` then `npm run dev`
2. You might need to set some extra heap space for the node runtime
`export NODE_OPTIONS="--max-old-space-size=4096"`
3. In another shell `npm run build` otherwise the backend will not find the
`frontend/build` folder and will not compile.
2. You might need to set some extra heap space for the node runtime `export NODE_OPTIONS="--max-old-space-size=4096"`
3. In another shell `npm run build` otherwise the backend will not find the `frontend/build` folder and will not compile.
4. In another shell `sudo caddy run --config Caddyfile`
7. Go to `backend/`:
`env DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
8. Et voilà, windmill should be available at `http://localhost/`
## Contributors
<a href="https://github.com/windmill-labs/windmill/graphs/contributors">
@@ -428,4 +404,4 @@ running options.
## Copyright
Windmill Labs, Inc 2023
Windmill Labs, Inc 2023

View File

@@ -20,8 +20,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -130,28 +130,28 @@
},
{
"ordinal": 25,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 26,
"name": "code_completion_model",
"type_info": "Varchar"
},
{
"ordinal": 27,
"name": "teams_command_script",
"type_info": "Text"
},
{
"ordinal": 28,
"ordinal": 26,
"name": "teams_team_id",
"type_info": "Text"
},
{
"ordinal": 29,
"ordinal": 27,
"name": "teams_team_name",
"type_info": "Text"
},
{
"ordinal": 28,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 29,
"name": "code_completion_model",
"type_info": "Varchar"
}
],
"parameters": {
@@ -185,10 +185,10 @@
true,
true,
true,
true,
true,
true,
false,
true,
true,
true,
true
]
},

View File

@@ -1,24 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n path, \n script_path, \n is_flow, \n route_path, \n workspace_id, \n is_async, \n requires_auth, \n edited_by, \n email, \n static_asset_config AS \"static_asset_config: _\",\n workspaced_route,\n is_static_website\n FROM \n http_trigger \n WHERE \n http_method = $1\n ",
"query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger WHERE http_method = $1",
"describe": {
"columns": [
{
@@ -55,11 +55,6 @@
},
{
"ordinal": 10,
"name": "workspaced_route",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "is_static_website",
"type_info": "Bool"
}
@@ -93,9 +88,8 @@
false,
false,
true,
false,
false
]
},
"hash": "01b87cdd552c2d5a3cd475eb705110394fe6c8abcc1fa93291fac6f929043d20"
"hash": "1961d15ae075072bd5f677c95f9b4dac7f747585d98aa8bcf60dcfb4c8124028"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n script_path, \n is_flow, \n http_method as \"http_method: _\", \n edited_by, \n email, \n edited_at, \n extra_perms, \n is_async, \n requires_auth, \n static_asset_config as \"static_asset_config: _\", \n is_static_website\n FROM \n http_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth, static_asset_config as \"static_asset_config: _\", is_static_website\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
@@ -25,21 +25,16 @@
},
{
"ordinal": 4,
"name": "workspaced_route",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 6,
"ordinal": 5,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 7,
"ordinal": 6,
"name": "http_method: _",
"type_info": {
"Custom": {
@@ -57,42 +52,42 @@
}
},
{
"ordinal": 8,
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 9,
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 10,
"ordinal": 9,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 11,
"ordinal": 10,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 12,
"ordinal": 11,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 13,
"ordinal": 12,
"name": "requires_auth",
"type_info": "Bool"
},
{
"ordinal": 14,
"ordinal": 13,
"name": "static_asset_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 15,
"ordinal": 14,
"name": "is_static_website",
"type_info": "Bool"
}
@@ -117,10 +112,9 @@
false,
false,
false,
false,
true,
false
]
},
"hash": "4353a7de6b4de1df6e19d95a4db8ae89c8d75034a9210233aed2f0414690e37d"
"hash": "197321abfe4667256761884970334f58c1b3edfcc1e863ec4316c9742a1ac7c8"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (job_id, logs)\n VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs\n WHERE job_logs.job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "1ab0d1ba1fbfad31ffb28a01a6c9640d0ac142aabee8d288a4f9c56ad9dbeac4"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH email_lookup AS (\n SELECT email FROM token WHERE token = $1\n )\n DELETE FROM token\n WHERE email = (SELECT email FROM email_lookup) AND label = 'session'\n RETURNING email",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "1bdf186d3b99bbd913cbf95150105470cd5f1d4ddbb147cb8ce46f9d1da5dfaf"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_workspace_runnables SET workspace_id = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "2bb2cf6accb18d3e37a63388cca52a6591e7593b1a7c3d7a6848587679a48187"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) VALUES ($1, $2, TRUE, $3) ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "35795d27c4ca69d2f145b4dba08a6ed16c25aea4584103c1b9a3651eb31bfe53"
}

View File

@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "3bbde0fa35d935ec2dd8bd1fb14cfecf48305f5f4b644b3c35355074e1ccce28"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'mqtt' AND \n server_id IS NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
"query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes FROM token WHERE email = $1 AND label != 'ephemeral-script'\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -50,5 +50,5 @@
true
]
},
"hash": "8be2919c3511575c89b882b112b987fd5724c299cb285f819a2561260404e513"
"hash": "4115a8e75ad5ed74f737f24aa64edc68e71b29ddc32a00a754678ea4f2c167e5"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929"
}

View File

@@ -1,25 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n mqtt_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b"
}

View File

@@ -1,50 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'mqtt' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "trigger_config!: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
true
]
},
"hash": "548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19"
}

View File

@@ -130,28 +130,28 @@
},
{
"ordinal": 25,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 26,
"name": "code_completion_model",
"type_info": "Varchar"
},
{
"ordinal": 27,
"name": "teams_command_script",
"type_info": "Text"
},
{
"ordinal": 28,
"ordinal": 26,
"name": "teams_team_id",
"type_info": "Text"
},
{
"ordinal": 29,
"ordinal": 27,
"name": "teams_team_name",
"type_info": "Text"
},
{
"ordinal": 28,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 29,
"name": "code_completion_model",
"type_info": "Varchar"
}
],
"parameters": {
@@ -185,10 +185,10 @@
true,
true,
true,
true,
true,
true,
false,
true,
true,
true,
true
]
},

View File

@@ -28,8 +28,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}
@@ -63,8 +62,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n http_trigger \n SET \n script_path = $1, \n path = $2, \n is_flow = $3, \n http_method = $4,\n static_asset_config = $5, \n edited_by = $6, \n email = $7, \n is_async = $8, \n requires_auth = $9, \n edited_at = now(), \n is_static_website = $10\n WHERE \n workspace_id = $11 AND \n path = $12\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "5d3401f608b27937285e418f7bd64bd112977c995bf266680c54f58ad57fd4b6"
}

View File

@@ -1,36 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL\n WHERE \n workspace_id = $12 AND \n path = $13\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"JsonbArray",
{
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "5d773db0a69c73e86ccd5fd978932e7ac64eb41a56d8ca9ff0a89c594432b531"
}

View File

@@ -20,8 +20,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) \n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2 AND ($3::TEXT IS NULL OR path != $3))",
"describe": {
"columns": [
{
@@ -33,5 +33,5 @@
null
]
},
"hash": "0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58"
"hash": "63b5f03741be97d0e8763dd070649ebb6ec02aa083d7e175c1a02a38935a4024"
}

View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "WITH zombie_jobs AS (\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM v2_job j, v2_job_runtime r\n WHERE j.id = q.id AND j.id = r.id\n AND ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n RETURNING q.id, q.workspace_id, ping\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id, workspace_id, ping FROM zombie_jobs",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "ping",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
true
]
},
"hash": "653574b381a31548d82c1f6f3f44ec826597c42310ab78f9aacd7d9448206c6a"
}

View File

@@ -1,135 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>\",\n v3_config as \"v3_config!: Option<SqlxJson<MqttV3Config>>\",\n v5_config as \"v5_config!: Option<SqlxJson<MqttV5Config>>\",\n client_version AS \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n mqtt_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "mqtt_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>",
"type_info": "JsonbArray"
},
{
"ordinal": 2,
"name": "v3_config!: Option<SqlxJson<MqttV3Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "v5_config!: Option<SqlxJson<MqttV5Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "client_version: _",
"type_info": {
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
}
},
{
"ordinal": 5,
"name": "client_id",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 13,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 16,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 17,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
true,
true,
false,
true,
false,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "65cd5a5a12eb79a2f21ed898616c5cd5c5e671ce1ed3d89639cd306c76a1d9db"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (job_id, logs)\n VALUES ($1, 'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')\n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs\n WHERE job_logs.job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "67afe352fc26dda9107c90e50e954642d877178ce2c0e73b72c3824135ef86f4"
}

View File

@@ -31,8 +31,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -1,41 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n http_trigger \n SET \n route_path = $1, \n route_path_key = $2, \n workspaced_route = $3, \n script_path = $4, \n path = $5, \n is_flow = $6, \n http_method = $7, \n static_asset_config = $8, \n edited_by = $9, \n email = $10, \n is_async = $11, \n requires_auth = $12, \n edited_at = now(), \n is_static_website = $13\n WHERE \n workspace_id = $14 AND \n path = $15\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "78e6342696052d0e1f79fd721f5942d4da686a65ab11eefdca16367d2db67963"
}

View File

@@ -1,132 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>\",\n v3_config as \"v3_config!: Option<SqlxJson<MqttV3Config>>\",\n v5_config as \"v5_config!: Option<SqlxJson<MqttV5Config>>\",\n client_version as \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM\n mqtt_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "mqtt_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "subscribe_topics!: Vec<SqlxJson<SubscribeTopic>>",
"type_info": "JsonbArray"
},
{
"ordinal": 2,
"name": "v3_config!: Option<SqlxJson<MqttV3Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "v5_config!: Option<SqlxJson<MqttV5Config>>",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "client_version: _",
"type_info": {
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
}
},
{
"ordinal": 5,
"name": "client_id",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 13,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 16,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 17,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
true,
true,
false,
true,
false,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false
]
},
"hash": "7b6e8dac5f83fcbae95056c4206bf4de1b12d6c8aab32f1f55d7f2c230c5c08e"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM token WHERE email = $1 AND label = 'session'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "7db681e86f8332c636d0f29b389860292b704788827edfebc467a92486f9e14f"
}

View File

@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, static_asset_config = $5, edited_by = $6, email = $7, is_async = $8, requires_auth = $9, edited_at = now(), is_static_website = $10\n WHERE workspace_id = $11 AND path = $12",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "89c08575afb31b70984f6b2b7dd4297af93b5b83ffdfb3ec91eba5df7ad3fd95"
}

View File

@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Bool",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n path, \n script_path, \n is_flow, \n route_path, \n workspace_id, \n is_async, \n requires_auth, \n edited_by, \n email,\n static_asset_config AS \"static_asset_config: _\",\n workspaced_route,\n is_static_website \n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n http_method = $2\n ",
"query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger WHERE workspace_id = $1 AND http_method = $2",
"describe": {
"columns": [
{
@@ -55,11 +55,6 @@
},
{
"ordinal": 10,
"name": "workspaced_route",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "is_static_website",
"type_info": "Bool"
}
@@ -94,9 +89,8 @@
false,
false,
true,
false,
false
]
},
"hash": "300b41622dbcdf908d9e173eee2133239d5806fc50bba2d3fe9d8a6e11627655"
"hash": "8ff25d890d3f7019c5c6f2b47f29f4c7b2c91094036d9b5037ef9a4ac986ee53"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ",
"query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ",
"describe": {
"columns": [
{
@@ -30,11 +30,6 @@
},
{
"ordinal": 5,
"name": "mqtt_used!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "sqs_used!",
"type_info": "Bool"
}
@@ -50,9 +45,8 @@
null,
null,
null,
null,
null
]
},
"hash": "31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5"
"hash": "93d0ee34c7b7c56ab9cae28071f8e5fdaff0e601e2760cc5653d197c51b106bb"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(\n SELECT 1 FROM http_trigger \n WHERE path = $1 AND workspace_id = $2\n )",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
null
]
},
"hash": "adb0090afd3ce918d8b80ff51d9f6104a430a11d7c5cb9447025d11506585708"
"hash": "9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb"
}

View File

@@ -1,24 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT f.path\n FROM flow_workspace_runnables fwr \n JOIN flow f \n ON fwr.flow_path = f.path AND fwr.workspace_id = f.workspace_id\n WHERE fwr.runnable_path = $1 AND fwr.runnable_is_flow = $2 AND fwr.workspace_id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Bool",
"Text"
]
},
"nullable": [
false
]
},
"hash": "9aeee333b1dbe58ba819ba3b2713242b54d77b46b5f785ae66b8104f89f43219"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE \n FROM \n mqtt_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM http_trigger \n WHERE workspace_id = $1 \n AND path = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "a269c388056eabe4b045948f451ea74ffbb4c0ed7e694f8f03d92f2a7c118af9"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n route_path_key = $1\n AND workspace_id = $2 \n AND http_method = $3 \n AND ($4::TEXT IS NULL OR path != $4)\n )\n ",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3 AND ($4::TEXT IS NULL OR path != $4))",
"describe": {
"columns": [
{
@@ -34,5 +34,5 @@
null
]
},
"hash": "679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294"
"hash": "a96ff22bc78b74d7234550a12d9fb5c555c1187b276f1353dae1b2ac0670a92a"
}

View File

@@ -1,41 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH to_update AS (\n SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter\n FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_runtime r ON r.id = j.id\n LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id\n WHERE ping < now() - ($1 || ' seconds')::interval\n AND running = true\n AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')\n AND same_worker = false\n AND (zjc.counter IS NULL OR zjc.counter <= $2)\n FOR UPDATE of q SKIP LOCKED\n ),\n zombie_jobs AS (\n UPDATE v2_job_queue q\n SET running = false, started_at = null\n FROM to_update tu\n WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)\n RETURNING q.id, q.workspace_id, ping, tu.counter\n ),\n update_ping AS (\n UPDATE v2_job_runtime r\n SET ping = null\n FROM zombie_jobs zj\n WHERE r.id = zj.id\n ),\n increment_counter AS (\n INSERT INTO zombie_job_counter (job_id, counter)\n SELECT id, 1 FROM to_update WHERE counter < $2\n ON CONFLICT (job_id) DO UPDATE \n SET counter = zombie_job_counter.counter + 1\n ),\n update_concurrency AS (\n UPDATE concurrency_counter cc\n SET job_uuids = job_uuids - zj.id::text\n FROM zombie_jobs zj\n INNER JOIN concurrency_key ck ON ck.job_id = zj.id\n WHERE cc.concurrency_id = ck.key\n )\n SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "ping",
"type_info": "Timestamptz"
},
{
"ordinal": 3,
"name": "counter",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Int4"
]
},
"nullable": [
false,
false,
true,
null
]
},
"hash": "abfc6b530565f125bd0b8ac99cd67fd22f14f8fce89e59890ba21e87fe323af5"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO token\n (token, email, label, expiration, super_admin)\n VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5)",
"query": "INSERT INTO token\n (token, email, label, expiration, super_admin)\n VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)",
"describe": {
"columns": [],
"parameters": {
@@ -14,5 +14,5 @@
},
"nullable": []
},
"hash": "8aebd7f7fd1374f1c3d5389e953ebf080df3f76ac3e6e6373a89c8d46388125d"
"hash": "b05c5f62ef4aa21d33369130cced0e9d7d128727eb58a9be7ae69cbb16bcbb27"
}

View File

@@ -0,0 +1,39 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, static_asset_config, edited_by, email, edited_at, is_static_website) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "b41cef713e822bbd89b49b1f35cc662539d5e4af1dd0b5923d8ee17b772c5677"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_workspace_runnables SET flow_path = REGEXP_REPLACE(flow_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE flow_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "b656927cd70b6667f3c72186ec04f0bf040da3af9e2eac3229264ec95b4755d8"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_audit_timestamps ON audit (timestamp DESC)",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8"
}

View File

@@ -23,8 +23,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) VALUES ($1, $2, $3, FALSE, $4) ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int8",
"Varchar"
]
},
"nullable": []
},
"hash": "c35f44f91b08fa57e29a2b4a685706f62e700695810f23108e975dfcd1fee7a3"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id AS \"id!\" FROM v2_job_queue INNER JOIN v2_job USING (id) WHERE parent_job = $1 AND v2_job.workspace_id = $2",
"query": "SELECT id AS \"id!\" FROM v2_job WHERE parent_job = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
false
]
},
"hash": "eb68469026be39048c5f42a80a2c538fbb54ad269ec81aea89e431a511245a1e"
"hash": "c5259e37703c3e48104438bad6e1f3615f4439c090a75e6fde03702a21589b25"
}

View File

@@ -20,8 +20,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, static_asset_config = $7, edited_by = $8, email = $9, is_async = $10, requires_auth = $11, edited_at = now(), is_static_website = $12\n WHERE workspace_id = $13 AND path = $14",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "ccb7aea162fa8781675d547fb82f2996dede4de49c5bd9cca2928209dc40b8f1"
}

View File

@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool",
"Text"
]
},
"nullable": [
null
]
},
"hash": "cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n workspace_id, \n workspaced_route,\n path, \n route_path, \n route_path_key, \n script_path, \n is_flow, \n edited_by, \n edited_at, \n email, \n extra_perms, \n is_async, \n requires_auth, \n http_method AS \"http_method: _\", \n static_asset_config AS \"static_asset_config: _\", \n is_static_website \n FROM http_trigger\n WHERE workspace_id = $1\n ",
"query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, edited_by, edited_at, email, extra_perms, is_async, requires_auth, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
@@ -10,66 +10,61 @@
},
{
"ordinal": 1,
"name": "workspaced_route",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"ordinal": 2,
"name": "route_path",
"type_info": "Varchar"
},
{
"ordinal": 4,
"ordinal": 3,
"name": "route_path_key",
"type_info": "Varchar"
},
{
"ordinal": 5,
"ordinal": 4,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 6,
"ordinal": 5,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 7,
"ordinal": 6,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"ordinal": 7,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 9,
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 10,
"ordinal": 9,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 11,
"ordinal": 10,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 12,
"ordinal": 11,
"name": "requires_auth",
"type_info": "Bool"
},
{
"ordinal": 13,
"ordinal": 12,
"name": "http_method: _",
"type_info": {
"Custom": {
@@ -87,12 +82,12 @@
}
},
{
"ordinal": 14,
"ordinal": 13,
"name": "static_asset_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 15,
"ordinal": 14,
"name": "is_static_website",
"type_info": "Bool"
}
@@ -116,10 +111,9 @@
false,
false,
false,
false,
true,
false
]
},
"hash": "e44ea5870a8eba1fe3d22d4dfe7734669464a56e470935ce2924723003f3f92b"
"hash": "d8aa1a5813fedbb22128112fcdf9d16769811cd5d5ad3b66362cc5bb1750ae6b"
}

View File

@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_workspace_runnables SET runnable_path = REGEXP_REPLACE(runnable_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE runnable_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "dc58e5b4715601a93b3c01a2564a4420f232867a23cacb9a62b386f129a86a4b"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM flow_workspace_runnables WHERE flow_path = $1 AND workspace_id = $2",
"query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "acbea8740b28c26942c50edcf5618cd141e68cf83a7dcae7d3c1b8a7ba94425b"
"hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188"
}

View File

@@ -28,8 +28,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -36,8 +36,7 @@
"email",
"nats",
"postgres",
"sqs",
"mqtt"
"sqs"
]
}
}

View File

@@ -1,40 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO http_trigger (\n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n script_path, \n is_flow, \n is_async, \n requires_auth, \n http_method, \n static_asset_config, \n edited_by, \n email, \n edited_at, \n is_static_website\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14\n )\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Bool",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Jsonb",
"Varchar",
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "eb4779976677308e8ad7c6427bfb45146478f70956919c01718623f0828e0383"
}

View File

@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "f632d08a8d3df691fff9f57fdf926f787287c3cd181a6853056077edba10473d"
}

View File

@@ -1,36 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"JsonbArray",
{
"Custom": {
"name": "mqtt_client_version",
"kind": {
"Enum": [
"v3",
"v5"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Varchar"
]
},
"nullable": []
},
"hash": "f82974abc7da71fac397f754fe3b3c9c84b6b438b2d9f8cbba61192e999971f4"
}

View File

@@ -1,25 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96"
}

View File

@@ -11,5 +11,5 @@
"remote.autoForwardPorts": true,
"conventionalCommits.scopes": [
"restructring triggers, decoding trigger message on work"
],
]
}

2035
backend/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.477.1"
version = "1.467.1"
authors.workspace = true
edition.workspace = true
@@ -30,7 +30,7 @@ members = [
]
[workspace.package]
version = "1.477.1"
version = "1.467.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -59,7 +59,7 @@ flow_testing = ["windmill-worker/flow_testing"]
openidconnect = ["windmill-api/openidconnect"]
cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"]
tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "enterprise", "parquet"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"]
kafka = ["windmill-api/kafka"]
@@ -74,7 +74,6 @@ mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
websocket = ["windmill-api/websocket"]
postgres_trigger = ["windmill-api/postgres_trigger"]
mqtt_trigger = ["windmill-api/mqtt_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger"]
python = ["windmill-worker/python"]
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
@@ -119,7 +118,7 @@ object_store = { workspace = true, optional = true }
quote.workspace = true
memchr.workspace = true
v8 = { workspace = true, optional = true }
rustls.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
@@ -161,7 +160,6 @@ windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-api-client = { path = "./windmill-api-client" }
rustls = "0.23.0"
memchr = "2.7.4"
axum = { version = "^0.7", features = ["multipart"] }
headers = "^0"
@@ -175,7 +173,7 @@ serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
uuid = { version = "^1", features = ["serde", "v4"] }
thiserror = "^2"
anyhow = "^1"
chrono = { version = "=0.4.39", features = ["serde"] }
chrono = { version = "0.4.35", features = ["serde"] }
chrono-tz = "^0.10.1"
tracing = "^0"
tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
@@ -213,20 +211,20 @@ itertools = "^0"
regex = "^1"
semver = "^1"
v8 = "=130.0.7" # Exact version
deno_fetch = "0.214.0"
deno_tls = "0.177.0"
deno_console = "0.190.0"
deno_url = "0.190.0"
deno_webidl = "0.190.0"
deno_web = "0.221.0"
deno_io = "0.100.0"
deno_net = "0.182.0"
deno_core = "0.336.0"
v8 = "=134.4.0" # Exact version
deno_fetch = "0.216.0"
deno_tls = "0.179.0"
deno_console = "0.192.0"
deno_url = "0.192.0"
deno_webidl = "0.192.0"
deno_web = "0.223.0"
deno_io = "0.102.0"
deno_net = "0.184.0"
deno_core = "0.338.0"
deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_permissions = "0.49.0"
deno_runtime = { version = "0.198.0", features = ["transpile"] }
deno_telemetry = "0.12.0"
deno_permissions = "0.51.0"
deno_runtime = { version = "0.200.0", features = ["transpile"] }
deno_telemetry = "0.14.0"
deno_error = "=0.5.5"
swc_common = "=0.37.5"
@@ -357,4 +355,3 @@ tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
tree-sitter = {version = "0.23.0", features = []}
tree-sitter-c-sharp = "0.23.0"
oracle = { version = "0.6.3", features = ["chrono"] }
rumqttc = { version = "0.24.0", features = ["use-native-tls"]}

View File

@@ -1,27 +0,0 @@
DO
$$
DECLARE
tbl_name text;
policy_exists boolean;
current_sch text;
tbl_names text[] := ARRAY['account', 'app', 'audit', 'capture', 'completed_job', 'flow', 'folder', 'http_trigger', 'queue', 'raw_app', 'resource', 'schedule', 'script', 'usr_to_group', 'variable'];
BEGIN
-- Get the current schema
SELECT current_schema() INTO current_sch;
FOR tbl_name IN SELECT unnest(tbl_names)
LOOP
SELECT EXISTS (
SELECT 1
FROM pg_policies
WHERE schemaname = current_sch
AND tablename = tbl_name
AND policyname = 'admin_policy'
) INTO policy_exists;
IF NOT policy_exists THEN
EXECUTE format('CREATE POLICY admin_policy ON %I.%I TO windmill_admin USING (true);', current_sch, tbl_name);
END IF;
END LOOP;
END;
$$;

View File

@@ -1 +1 @@
9a3f58425136b0f9e9c8106151537b1e271ec838
2dca712cde013e7dda96dc3162e47a148973a0dc

View File

@@ -1,4 +0,0 @@
-- Add down migration script here
DROP TRIGGER webhook_change_trigger ON workspace_settings;
DROP FUNCTION notify_webhook_change();

View File

@@ -1,15 +0,0 @@
-- Add up migration script here
CREATE OR REPLACE FUNCTION notify_webhook_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_webhook_change', NEW.workspace_id);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER webhook_change_trigger
AFTER UPDATE OF webhook ON workspace_settings
FOR EACH ROW
WHEN (OLD.webhook IS DISTINCT FROM NEW.webhook)
EXECUTE FUNCTION notify_webhook_change();

View File

@@ -1,3 +0,0 @@
-- Add down migration script here
DROP TABLE zombie_job_counter;

View File

@@ -1,7 +0,0 @@
-- Add up migration script here
CREATE TABLE IF NOT EXISTS zombie_job_counter (
job_id UUID PRIMARY KEY REFERENCES v2_job (id) ON DELETE CASCADE,
counter INTEGER NOT NULL DEFAULT 0
);

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,37 +0,0 @@
DO
$do$
DECLARE
current_schema_name TEXT;
BEGIN
-- Get the current schema for the session
SELECT current_schema() INTO current_schema_name;
-- Lock the roles table to prevent race conditions
LOCK TABLE pg_catalog.pg_roles;
EXECUTE format('GRANT USAGE ON SCHEMA %I TO windmill_user', current_schema_name);
EXECUTE format('GRANT USAGE ON SCHEMA %I TO windmill_admin', current_schema_name);
-- Grant privileges dynamically to the current schema
EXECUTE format('GRANT ALL ON ALL TABLES IN SCHEMA %I TO windmill_user', current_schema_name);
EXECUTE format('GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO windmill_user', current_schema_name);
-- Alter default privileges dynamically
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO windmill_user', current_schema_name);
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO windmill_user', current_schema_name);
-- Grant privileges dynamically to the current schema
EXECUTE format('GRANT ALL ON ALL TABLES IN SCHEMA %I TO windmill_admin', current_schema_name);
EXECUTE format('GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA %I TO windmill_admin', current_schema_name);
-- Alter default privileges dynamically
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO windmill_admin', current_schema_name);
EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO windmill_admin', current_schema_name);
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'Error granting proper permissions to windmill users: %', SQLERRM;
END
$do$;

View File

@@ -1,3 +0,0 @@
-- Add down migration script here
DROP TABLE mqtt_trigger;
DROP TYPE MQTT_CLIENT_VERSION;

View File

@@ -1,72 +0,0 @@
-- Add up migration script here
CREATE TYPE MQTT_CLIENT_VERSION AS ENUM ('v3', 'v5');
CREATE TABLE mqtt_trigger (
mqtt_resource_path VARCHAR(255) NOT NULL,
subscribe_topics JSONB[] NOT NULL,
client_version MQTT_CLIENT_VERSION DEFAULT 'v5' NOT NULL,
v5_config JSONB NULL,
v3_config JSONB NULL,
client_id VARCHAR(65535) DEFAULT NULL,
path VARCHAR(255) NOT NULL,
script_path VARCHAR(255) NOT NULL,
is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
edited_by VARCHAR(50) NOT NULL,
email VARCHAR(255) NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
extra_perms JSONB NOT NULL DEFAULT '{}',
server_id VARCHAR(50) NULL,
last_server_ping TIMESTAMPTZ NULL,
error TEXT NULL,
enabled BOOLEAN NOT NULL,
PRIMARY KEY (path, workspace_id)
);
GRANT ALL ON mqtt_trigger TO windmill_user;
GRANT ALL ON mqtt_trigger TO windmill_admin;
ALTER TABLE mqtt_trigger ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON mqtt_trigger FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_folder_extra_perms_user_select ON mqtt_trigger FOR SELECT TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_insert ON mqtt_trigger FOR INSERT TO windmill_user
WITH CHECK (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON mqtt_trigger FOR UPDATE TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_delete ON mqtt_trigger FOR DELETE TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'f' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_own ON mqtt_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'u' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON mqtt_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(mqtt_trigger.path, '/', 1) = 'g' AND SPLIT_PART(mqtt_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
CREATE POLICY see_extra_perms_user_select ON mqtt_trigger FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
CREATE POLICY see_extra_perms_user_insert ON mqtt_trigger FOR INSERT TO windmill_user
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_update ON mqtt_trigger FOR UPDATE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_delete ON mqtt_trigger FOR DELETE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups_select ON mqtt_trigger FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
CREATE POLICY see_extra_perms_groups_insert ON mqtt_trigger FOR INSERT TO windmill_user
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_update ON mqtt_trigger FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_delete ON mqtt_trigger FOR DELETE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,2 +0,0 @@
-- Add up migration script here
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'mqtt';

View File

@@ -1 +0,0 @@
DROP TABLE flow_workspace_runnables;

View File

@@ -1,12 +0,0 @@
CREATE TABLE flow_workspace_runnables (
flow_path VARCHAR(255) NOT NULL,
runnable_path VARCHAR(255) NOT NULL,
script_hash BIGINT NULL,
runnable_is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
FOREIGN KEY (workspace_id, flow_path) REFERENCES flow (workspace_id, path) ON DELETE CASCADE
);
CREATE UNIQUE INDEX flow_workspace_without_hash_unique_idx ON flow_workspace_runnables (flow_path, runnable_path, runnable_is_flow, workspace_id) WHERE script_hash IS NULL;
CREATE UNIQUE INDEX flow_workspace_with_hash_unique_idx ON flow_workspace_runnables (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id) WHERE script_hash IS NOT NULL;
CREATE INDEX flow_workspace_runnable_path_is_flow_idx ON flow_workspace_runnables (runnable_path, runnable_is_flow, workspace_id);

View File

@@ -1,3 +0,0 @@
-- Add down migration script here
ALTER TABLE http_trigger
DROP COLUMN workspaced_route;

View File

@@ -1,3 +0,0 @@
-- Add up migration script here
ALTER TABLE http_trigger
ADD COLUMN workspaced_route BOOLEAN NOT NULL DEFAULT false;

6
backend/package-lock.json generated Normal file
View File

@@ -0,0 +1,6 @@
{
"name": "backend",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}

View File

@@ -96,11 +96,11 @@ pub fn parse_python_signature(
.iter()
.enumerate()
.map(|(i, x)| {
let (mut typ, has_default) = x
let mut typ = x
.as_arg()
.annotation
.as_ref()
.map_or((Typ::Unknown, false), |e| parse_expr(e));
.map_or(Typ::Unknown, |e| parse_expr(e));
let default = if i >= def_arg_start {
params
@@ -138,7 +138,7 @@ pub fn parse_python_signature(
otyp: None,
name: x.as_arg().arg.to_string(),
typ,
has_default: has_default || default.is_some(),
has_default: default.is_some(),
default,
oidx: None,
}
@@ -158,27 +158,17 @@ pub fn parse_python_signature(
}
}
fn parse_expr(e: &Box<Expr>) -> (Typ, bool) {
fn parse_expr(e: &Box<Expr>) -> Typ {
match e.as_ref() {
Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref()), false),
Expr::Name(ExprName { id, .. }) => parse_typ(id.as_ref()),
Expr::Attribute(x) => {
if x.value
.as_name_expr()
.is_some_and(|x| x.id.as_str() == "wmill")
{
(parse_typ(x.attr.as_str()), false)
parse_typ(x.attr.as_str())
} else {
(Typ::Unknown, false)
}
}
Expr::BinOp(x) => {
if matches!(
x.right.as_ref(),
Expr::Constant(ExprConstant { value: Constant::None, .. })
) {
(parse_expr(&x.left).0, true)
} else {
(Typ::Unknown, false)
Typ::Unknown
}
}
Expr::Subscript(x) => match x.value.as_ref() {
@@ -203,15 +193,14 @@ fn parse_expr(e: &Box<Expr>) -> (Typ, bool) {
}
_ => None,
};
(Typ::Str(values), false)
Typ::Str(values)
}
"List" => (Typ::List(Box::new(parse_expr(&x.slice).0)), false),
"Optional" => (parse_expr(&x.slice).0, true),
_ => (Typ::Unknown, false),
"List" => Typ::List(Box::new(parse_expr(&x.slice))),
_ => Typ::Unknown,
},
_ => (Typ::Unknown, false),
_ => Typ::Unknown,
},
_ => (Typ::Unknown, false),
_ => Typ::Unknown,
}
}
@@ -687,53 +676,4 @@ def main(a: list, e: List[int], b: list = [1,2,3,4], c = [1,2,3,4], d = ["a", "b
Ok(())
}
#[test]
fn test_parse_python_sig_9() -> anyhow::Result<()> {
let code = r#"
from typing import Optional
def main(a: str, b: Optional[str], c: str | None): return
"#;
println!(
"{}",
serde_json::to_string(&parse_python_signature(code, None, false)?)?
);
assert_eq!(
parse_python_signature(code, None, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "a".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "b".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "c".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true,
oidx: None
},
],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
Ok(())
}
}

View File

@@ -4,7 +4,6 @@ use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use windmill_parser_bash::parse_powershell_sig;
use windmill_parser_ts::{parse_deno_signature, parse_expr_for_ids, parse_expr_for_imports};
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_deno_sig() -> anyhow::Result<()> {
let code = "
@@ -147,8 +146,6 @@ export function main(test1?: string, test2: string = \"burkina\",
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_deno_sig_implicit_types() -> anyhow::Result<()> {
let code = "
@@ -227,7 +224,6 @@ export function main(test2 = \"burkina\",
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_deno_types() -> anyhow::Result<()> {
let code = "
@@ -278,7 +274,6 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_enum_list() -> anyhow::Result<()> {
let code = "
@@ -310,7 +305,6 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_extract_ident() -> anyhow::Result<()> {
let code = "
@@ -330,7 +324,6 @@ fn test_parse_extract_ident() -> anyhow::Result<()> {
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_imports() -> anyhow::Result<()> {
let code = "
@@ -354,7 +347,6 @@ fn test_parse_imports() -> anyhow::Result<()> {
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_imports_dts() -> anyhow::Result<()> {
let code = "
@@ -367,7 +359,6 @@ export type foo = number
Ok(())
}
#[allow(dead_code)]
#[wasm_bindgen_test]
fn test_parse_powershell_sig() -> anyhow::Result<()> {
let code = "

View File

@@ -29,7 +29,6 @@ use windmill_api::HTTP_CLIENT;
use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID};
use windmill_common::{
get_database_url,
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
@@ -45,9 +44,9 @@ use windmill_common::{
},
scripts::ScriptLang,
stats_ee::schedule_stats,
utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS},
utils::{hostname, rd_string, Mode, GIT_VERSION},
worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP},
KillpillSender, METRICS_ENABLED,
DB, METRICS_ENABLED,
};
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
@@ -116,25 +115,6 @@ where
}
pub fn main() -> anyhow::Result<()> {
// https://github.com/denoland/deno/blob/main/cli/main.rs#L477
#[cfg(feature = "deno_core")]
let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
"--stack-size=1024".to_string(),
// TODO(bartlomieju): I think this can be removed as it's handled by `deno_core`
// and its settings.
// deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript
// TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast
"--no-harmony-import-assertions".to_string(),
])
.into_iter()
.skip(1)
.collect::<Vec<_>>();
#[cfg(feature = "deno_core")]
if !unrecognized_v8_flags.is_empty() {
println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags);
}
#[cfg(feature = "deno_core")]
deno_core::JsRuntime::init_platform(None, false);
create_and_run_current_thread_inner(windmill_main())
@@ -237,18 +217,65 @@ async fn windmill_main() -> anyhow::Result<()> {
std::env::set_var("RUST_LOG", "info")
}
if let Err(_e) = rustls::crypto::ring::default_provider().install_default() {
tracing::error!("Failed to install rustls crypto provider");
}
let hostname = hostname();
let mode_and_addons = MODE_AND_ADDONS.clone();
let mode = mode_and_addons.mode;
let mut enable_standalone_indexer: bool = false;
if mode == Mode::Standalone {
println!("Running in standalone mode");
}
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
println!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
println!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
println!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
eprintln!("mode not recognized, defaulting to standalone: {x}");
} else {
println!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
println!("jemalloc enabled");
@@ -366,7 +393,7 @@ async fn windmill_main() -> anyhow::Result<()> {
let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?;
let (killpill_tx, mut killpill_rx) = KillpillSender::new(2);
let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2);
let mut monitor_killpill_rx = killpill_tx.subscribe();
let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2);
let server_killpill_rx = killpill_phase2_tx.subscribe();
@@ -491,7 +518,8 @@ Windmill Community Edition {GIT_VERSION}
.expect("could not create initial server dir");
#[cfg(feature = "tantivy")]
let should_index_jobs = mode == Mode::Indexer || mode_and_addons.indexer;
let should_index_jobs =
mode == Mode::Indexer || (enable_standalone_indexer && mode == Mode::Standalone);
reload_indexer_config(&db).await;
@@ -624,7 +652,7 @@ Windmill Community Edition {GIT_VERSION}
)
.await?;
tracing::info!("All workers exited.");
killpill_tx.send();
killpill_tx.send(())?;
} else {
rx.recv().await?;
}
@@ -646,10 +674,9 @@ Windmill Community Edition {GIT_VERSION}
let tx = killpill_tx.clone();
let base_internal_url = base_internal_url.to_string();
let db_url: String = get_database_url().await?;
let h = tokio::spawn(async move {
let mut listener = retry_listen_pg(&db_url).await;
let mut listener = retry_listen_pg(&db).await;
loop {
tokio::select! {
biased;
@@ -665,6 +692,17 @@ Windmill Community Edition {GIT_VERSION}
tracing::info!("received killpill for monitor job");
break;
},
_ = tokio::time::sleep(Duration::from_secs(30)) => {
monitor_db(
&db,
&base_internal_url,
server_mode,
worker_mode,
false,
tx.clone(),
)
.await;
},
notification = listener.recv() => {
match notification {
Ok(n) => {
@@ -684,11 +722,6 @@ Windmill Community Edition {GIT_VERSION}
}
}
},
"notify_webhook_change" => {
let workspace_id = n.payload();
tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id);
windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id);
},
"notify_global_setting_change" => {
tracing::info!("Global setting change detected: {}", n.payload());
match n.payload() {
@@ -845,27 +878,14 @@ Windmill Community Edition {GIT_VERSION}
tracing::info!("received killpill for monitor job");
break;
},
new_listener = retry_listen_pg(&db_url) => {
new_listener = retry_listen_pg(&db) => {
listener = new_listener;
continue;
}
}
}
};
},
_ = tokio::time::sleep(Duration::from_secs(30)) => {
tracing::info!("monitor task started");
monitor_db(
&db,
&base_internal_url,
server_mode,
worker_mode,
false,
tx.clone(),
)
.await;
tracing::info!("monitor task finished");
},
}
}
}
});
@@ -874,7 +894,6 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!("Error waiting for monitor handle: {e:#}")
}
tracing::info!("Monitor exited");
killpill_tx.send();
Ok(()) as anyhow::Result<()>
};
@@ -931,8 +950,8 @@ Windmill Community Edition {GIT_VERSION}
Ok(())
}
async fn listen_pg(url: &str) -> Option<PgListener> {
let mut listener = match PgListener::connect(url).await {
async fn listen_pg(db: &DB) -> Option<PgListener> {
let mut listener = match PgListener::connect_with(&db).await {
Ok(l) => l,
Err(e) => {
tracing::error!(error = %e, "Could not connect to database");
@@ -941,11 +960,7 @@ async fn listen_pg(url: &str) -> Option<PgListener> {
};
if let Err(e) = listener
.listen_all(vec![
"notify_config_change",
"notify_global_setting_change",
"notify_webhook_change",
])
.listen_all(vec!["notify_config_change", "notify_global_setting_change"])
.await
{
tracing::error!(error = %e, "Could not listen to database");
@@ -955,13 +970,13 @@ async fn listen_pg(url: &str) -> Option<PgListener> {
return Some(listener);
}
async fn retry_listen_pg(url: &str) -> PgListener {
let mut listener = listen_pg(url).await;
async fn retry_listen_pg(db: &DB) -> PgListener {
let mut listener = listen_pg(db).await;
loop {
if listener.is_none() {
tracing::info!("Retrying listening to pg listen in 5 seconds");
tokio::time::sleep(Duration::from_secs(5)).await;
listener = listen_pg(url).await;
listener = listen_pg(db).await;
} else {
tracing::info!("Successfully connected to pg listen");
return listener.unwrap();
@@ -989,7 +1004,7 @@ fn display_config(envs: &[&str]) {
pub async fn run_workers(
db: Pool<Postgres>,
mut rx: tokio::sync::broadcast::Receiver<()>,
tx: KillpillSender,
tx: tokio::sync::broadcast::Sender<()>,
num_workers: i32,
base_internal_url: String,
agent_mode: bool,
@@ -1102,14 +1117,17 @@ pub async fn run_workers(
Ok(())
}
async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
if max_delay_secs == 0 {
max_delay_secs = 1;
}
async fn send_delayed_killpill(
tx: &tokio::sync::broadcast::Sender<()>,
max_delay_secs: u64,
context: &str,
) {
// Random delay to avoid all servers/workers shutting down simultaneously
let rd_delay = rand::rng().random_range(0..max_delay_secs);
tracing::info!("Scheduling {context} shutdown in {rd_delay}s");
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
tx.send();
if let Err(e) = tx.send(()) {
tracing::error!(error = %e, "Could not send killpill for {context}");
}
}

View File

@@ -34,7 +34,11 @@ use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
#[cfg(feature = "oauth2")]
use windmill_common::global_settings::OAUTH_SETTING;
use windmill_common::{
ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{
auth::JWT_SECRET,
ee::CriticalErrorChannel,
error,
flow_status::{FlowStatus, FlowStatusModule},
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
@@ -44,11 +48,23 @@ use windmill_common::{
NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
}, indexer::load_indexer_config, jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{
},
indexer::load_indexer_config,
jobs::QueuedJob,
oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
server::load_smtp_config,
tracing_init::JSON_FMT,
users::truncate_token,
utils::{now_from_db, rd_string, report_critical_error, Mode},
worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG,
SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP,
}, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS
},
BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -117,7 +133,7 @@ lazy_static::lazy_static! {
pub async fn initial_load(
db: &Pool<Postgres>,
tx: KillpillSender,
tx: tokio::sync::broadcast::Sender<()>,
worker_mode: bool,
server_mode: bool,
#[cfg(feature = "parquet")] disable_s3_store: bool,
@@ -226,40 +242,19 @@ pub async fn load_otel(db: &DB) {
if let Some(v) = v {
let deser = serde_json::from_value::<OtelSetting>(v);
if let Ok(o) = deser {
let metrics_enabled = o.metrics_enabled.unwrap_or_else(|| {
std::env::var("OTEL_METRICS_ENABLED")
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false)
});
let logs_enabled = o.logs_enabled.unwrap_or_else(|| {
std::env::var("OTEL_LOGS_ENABLED")
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false)
});
let tracing_enabled = o.tracing_enabled.unwrap_or_else(|| {
std::env::var("OTEL_TRACING_ENABLED")
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false)
});
let metrics_enabled = o.metrics_enabled.unwrap_or(false);
let logs_enabled = o.logs_enabled.unwrap_or(false);
let tracing_enabled = o.tracing_enabled.unwrap_or(false);
OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed);
OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed);
OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed);
let endpoint = if let Some(endpoint) = o.otel_exporter_otlp_endpoint {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint.clone());
Some(endpoint.clone())
} else {
std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok()
};
let headers = if let Some(headers) = o.otel_exporter_otlp_headers {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers.clone());
Some(headers.clone())
} else {
std::env::var("OTEL_EXPORTER_OTLP_HEADERS").ok()
};
if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
}
if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers);
}
if let Some(protocol) = o.otel_exporter_otlp_protocol {
std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
}
@@ -267,7 +262,7 @@ pub async fn load_otel(db: &DB) {
std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression);
}
println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})",
endpoint, headers.is_some());
o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some());
} else {
tracing::error!("Error deserializing otel settings");
}
@@ -1249,7 +1244,7 @@ pub async fn monitor_db(
server_mode: bool,
_worker_mode: bool,
initial_load: bool,
_killpill_tx: KillpillSender,
_killpill_tx: tokio::sync::broadcast::Sender<()>,
) {
let zombie_jobs_f = async {
if server_mode && !initial_load {
@@ -1425,7 +1420,7 @@ pub async fn reload_indexer_config(db: &Pool<Postgres>) {
pub async fn reload_worker_config(
db: &DB,
tx: KillpillSender,
tx: tokio::sync::broadcast::Sender<()>,
kill_if_change: bool,
) {
let config = load_worker_config(&db, tx.clone()).await;
@@ -1440,17 +1435,17 @@ pub async fn reload_worker_config(
|| (*wc).dedicated_worker != config.dedicated_worker
{
tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
let _ = tx.send();
let _ = tx.send(());
}
if (*wc).init_bash != config.init_bash {
tracing::info!("Init bash config changed, sending killpill. Expecting to be restarted by supervisor.");
let _ = tx.send();
let _ = tx.send(());
}
if (*wc).cache_clear != config.cache_clear {
tracing::info!("Cache clear changed, sending killpill. Expecting to be restarted by supervisor.");
let _ = tx.send();
let _ = tx.send(());
tracing::info!("Waiting 5 seconds to allow others workers to start potential jobs that depend on a potential shared cache volume");
tokio::time::sleep(Duration::from_secs(5)).await;
if let Err(e) = windmill_worker::common::clean_cache().await {
@@ -1538,44 +1533,18 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
Ok(())
}
const RESTART_LIMIT: i32 = 3;
async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker_name: &str) {
let mut zombie_jobs_uuid_restart_limit_reached = vec![];
if *RESTART_ZOMBIE_JOBS {
let restarted = sqlx::query!(
"WITH to_update AS (
SELECT q.id, q.workspace_id, r.ping, COALESCE(zjc.counter, 0) as counter
FROM v2_job_queue q
JOIN v2_job j ON j.id = q.id
JOIN v2_job_runtime r ON r.id = j.id
LEFT JOIN zombie_job_counter zjc ON zjc.job_id = q.id
WHERE ping < now() - ($1 || ' seconds')::interval
"WITH zombie_jobs AS (
UPDATE v2_job_queue q SET running = false, started_at = null
FROM v2_job j, v2_job_runtime r
WHERE j.id = q.id AND j.id = r.id
AND ping < now() - ($1 || ' seconds')::interval
AND running = true
AND kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlescriptflow')
AND same_worker = false
AND (zjc.counter IS NULL OR zjc.counter <= $2)
FOR UPDATE of q SKIP LOCKED
),
zombie_jobs AS (
UPDATE v2_job_queue q
SET running = false, started_at = null
FROM to_update tu
WHERE q.id = tu.id AND (tu.counter IS NULL OR tu.counter < $2)
RETURNING q.id, q.workspace_id, ping, tu.counter
),
update_ping AS (
UPDATE v2_job_runtime r
SET ping = null
FROM zombie_jobs zj
WHERE r.id = zj.id
),
increment_counter AS (
INSERT INTO zombie_job_counter (job_id, counter)
SELECT id, 1 FROM to_update WHERE counter < $2
ON CONFLICT (job_id) DO UPDATE
SET counter = zombie_job_counter.counter + 1
RETURNING q.id, q.workspace_id, ping
),
update_concurrency AS (
UPDATE concurrency_counter cc
@@ -1584,9 +1553,8 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
INNER JOIN concurrency_key ck ON ck.job_id = zj.id
WHERE cc.concurrency_id = ck.key
)
SELECT id AS \"id!\", workspace_id AS \"workspace_id!\", ping, counter + 1 AS counter FROM to_update",
SELECT id, workspace_id, ping FROM zombie_jobs",
*ZOMBIE_JOB_TIMEOUT,
RESTART_LIMIT
)
.fetch_all(db)
.await
@@ -1606,61 +1574,22 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
"no last ping".to_string()
};
let url = format!("{}/run/{}?workspace={}", base_url, r.id, r.workspace_id,);
let restart = r.counter.is_none_or(|x| x < RESTART_LIMIT);
let (critical_error_message, restart_message) = if restart {
(
format!(
"Zombie job {} on {} ({}) detected, restarting it ({}/{} attempts), last ping: {}",
r.id,
r.workspace_id,
url,
r.counter.unwrap_or(0) + 1,
RESTART_LIMIT,
last_ping
),
format!(
"Restarted job after not receiving job's ping for too long the {} ({}/{} attempts)\n\n",
last_ping,
r.counter.unwrap_or(0) + 1,
RESTART_LIMIT
)
)
} else {
(
format!(
"Zombie job {} on {} ({}) detected, but restart limit ({}) reached, job will be processed as an error, last ping: {}",
r.id, r.workspace_id, url, RESTART_LIMIT, last_ping
),
format!(
"job's ping was received last at {}, job will be processed as an error since all {} restart attempts failed",
last_ping, RESTART_LIMIT
)
)
};
let error_message = format!(
"Zombie job {} on {} ({}) detected, restarting it, {}",
r.id, r.workspace_id, url, last_ping
);
let _ = sqlx::query!(
"
let _ = sqlx::query!("
INSERT INTO job_logs (job_id, logs)
VALUES ($1, $2)
VALUES ($1, 'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')
ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\n' || EXCLUDED.logs
WHERE job_logs.job_id = $1",
r.id,
restart_message
r.id
)
.execute(db)
.await;
tracing::error!(critical_error_message);
report_critical_error(
critical_error_message,
db.clone(),
Some(&r.workspace_id),
None,
)
.await;
if !restart {
zombie_jobs_uuid_restart_limit_reached.push(r.id);
}
tracing::error!(error_message);
report_critical_error(error_message, db.clone(), Some(&r.workspace_id), None).await;
}
}
@@ -1735,43 +1664,9 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
.unwrap_or_else(|| vec![])
};
enum ErrorMessage {
RestartLimit,
SameWorker,
RestartDisabled,
}
impl ErrorMessage {
fn to_string(&self) -> String {
match self {
ErrorMessage::RestartLimit => format!("RestartLimit ({})", RESTART_LIMIT),
ErrorMessage::SameWorker => "SameWorker".to_string(),
ErrorMessage::RestartDisabled => "RestartDisabled".to_string(),
}
}
}
let zombie_jobs_restart_limit_reached =
sqlx::query_as::<_, QueuedJob>("SELECT * FROM v2_as_queue WHERE id = ANY($1)")
.bind(&zombie_jobs_uuid_restart_limit_reached[..])
.fetch_all(db)
.await
.ok()
.unwrap_or_else(|| vec![]);
let timeouts = non_restartable_jobs
.into_iter()
.map(|x| (x, ErrorMessage::RestartDisabled))
.chain(
same_worker_timeout_jobs
.into_iter()
.map(|x| (x, ErrorMessage::SameWorker)),
)
.chain(
zombie_jobs_restart_limit_reached
.into_iter()
.map(|x| (x, ErrorMessage::RestartLimit)),
)
.chain(same_worker_timeout_jobs)
.collect::<Vec<_>>();
#[cfg(feature = "prometheus")]
@@ -1779,7 +1674,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
}
for (job, error_kind) in timeouts {
for job in timeouts {
// since the job is unrecoverable, the same worker queue should never be sent anything
let (same_worker_tx_never_used, _same_worker_rx_never_used) =
mpsc::channel::<SameWorkerPayload>(1);
@@ -1814,21 +1709,20 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
};
let last_ping = job.last_ping.clone();
let error_message = format!(
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?})",
last_ping
.map(|x| x.to_string())
.unwrap_or_else(|| "no ping".to_string()),
*ZOMBIE_JOB_TIMEOUT,
error_kind.to_string()
);
let _ = handle_job_error(
db,
&client,
&job,
0,
None,
error::Error::ExecutionErr(error_message),
error::Error::ExecutionErr(format!(
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, same_worker: {})",
last_ping
.map(|x| x.to_string())
.unwrap_or_else(|| "no ping".to_string()),
*ZOMBIE_JOB_TIMEOUT,
job.same_worker
)),
true,
same_worker_tx_never_used,
"",
@@ -1907,27 +1801,16 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
let id = flow.id.clone();
let last_ping = flow.last_ping.clone();
let now = now_from_db(db).await?;
let base_url = BASE_URL.read().await;
let workspace_id = flow.workspace_id.clone();
let reason = format!(
"{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now})",
if flow.is_flow_step.unwrap_or(false) && flow.parent_job.is_some() {
format!("Flow was cancelled because subflow {id} ({base_url}/run/{id}?workspace={workspace_id})")
format!("Flow was cancelled because subflow {id}")
} else {
format!("Flow {id} ({base_url}/run/{id}?workspace={workspace_id}) was cancelled because it")
format!("Flow {id} was cancelled because it")
}
);
report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
cancel_zombie_flow_job(db, flow.id, &flow.workspace_id,
format!(r#"{reason}
This would happen if a worker was interrupted, killed or crashed while doing a state transition at the end of a job which is always an unexpected behavior that should never happen.
Please check your worker logs for more details and feel free to report it to the Windmill team on our Discord or support@windmill.dev (response for non EE customers will be best effort) with as much context as possible, ideally:
- Windmill version
- Worker logs right after the job referenced has finished running
- Is the error consistent when running the same flow
- A minimal flow and its flow.yaml that reproduces the error and that is importable in a fresh workspace
- Your infra setup (helm, docker-compose, configuration of the workers and their number, memory of the database, etc.)
"#)).await?;
cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, reason).await?;
}
}

Some files were not shown because too many files have changed in this diff Show More