Compare commits

...

82 Commits

Author SHA1 Message Date
Ruben Fiszel
7882e355eb tests 2025-02-24 12:25:52 +01:00
Ruben Fiszel
0b6d130efd all 2025-02-24 02:05:32 +01:00
Ruben Fiszel
31a8f24734 all 2025-02-24 02:04:30 +01:00
Ruben Fiszel
34587fea42 all 2025-02-24 02:02:20 +01:00
Ruben Fiszel
0577f999c2 all 2025-02-24 01:45:34 +01:00
Ruben Fiszel
25a49b74f9 all 2025-02-24 01:38:30 +01:00
Ruben Fiszel
1df9457bb1 all 2025-02-24 01:33:33 +01:00
Ruben Fiszel
fd1cfc0df5 bench.sh 2025-02-24 01:20:20 +01:00
Alexander Petric
f1c5b77d7a fix(frontend): make html app component content selectable (#5359) 2025-02-23 23:49:54 +01:00
Ruben Fiszel
089ba7ae00 fix npn check 2025-02-23 23:49:33 +01:00
Ruben Fiszel
66451d4da1 recycle audit logs 2025-02-23 23:38:23 +01:00
Ruben Fiszel
1a54e1ed45 delete audit logs stub older than 14days on CE 2025-02-23 23:33:47 +01:00
Ruben Fiszel
692de3e628 update bun to 1.2.3 2025-02-23 23:11:07 +01:00
Ruben Fiszel
f90841ac5a fix missing lock error logs 2025-02-23 09:18:50 +01:00
Ruben Fiszel
c81a13cdac bun to 1.2.2 while 1.2.3 is not released 2025-02-22 16:31:00 +01:00
Ruben Fiszel
05659816e7 fix: improve cancel/back behavior on editors 2025-02-22 16:28:28 +01:00
Ruben Fiszel
7c0152c353 add runs page links to history section 2025-02-22 15:22:53 +01:00
Ruben Fiszel
b8e6d0da79 feat: make flow lock deployment error visible in UI 2025-02-22 15:02:16 +01:00
Ruben Fiszel
20547493e0 add PRINT_NATIVE_ERRORS env variable 2025-02-22 11:06:42 +01:00
Ruben Fiszel
c6dbd239b4 update deno_core implementation to latest 2.2.1 2025-02-22 11:04:54 +01:00
Ruben Fiszel
b102ff4a46 fix: update deno 2.1.2->2.2.1 2025-02-22 09:25:47 +01:00
Ruben Fiszel
4e477d1f58 fix: update bun t.1.43->1.2.3 2025-02-22 09:23:33 +01:00
Ruben Fiszel
2e4ec7842a chore(main): release 1.467.1 (#5355)
* chore(main): release 1.467.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-22 02:12:27 +01:00
Ruben Fiszel
9ea6776ec1 initialize an uv venv globally but skip it for python executor 2025-02-22 01:53:14 +01:00
Ruben Fiszel
21ca694299 fix python venv 2025-02-22 01:44:45 +01:00
Ruben Fiszel
0aa6a39cad fix: app markdown is selectable in preview mode 2025-02-22 01:15:51 +01:00
Ruben Fiszel
e92a46b088 fix: init_script do not need to use nsjail even in nsjail mode 2025-02-22 01:04:33 +01:00
Ruben Fiszel
85993ccac2 fix: add uv bin path to PATH 2025-02-22 00:50:06 +01:00
Ruben Fiszel
8adedc4964 chore(main): release 1.467.0 (#5344)
* chore(main): release 1.467.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-21 21:29:50 +01:00
Ruben Fiszel
e71882d0a0 add zlib to import parser exception 2025-02-21 21:26:14 +01:00
Ruben Fiszel
ab1c15d92f fix: fix app component header buttons 2025-02-21 21:22:12 +01:00
David Abutbul
d9844fd7f7 feat: enable rust AI gen/fix/edit (#5349)
* init prompts for rust

* generated with yaml2json.py
2025-02-21 21:00:15 +01:00
HugoCasa
a54d8236a8 change workspace id super nit (#5354) 2025-02-21 20:02:13 +01:00
Ruben Fiszel
19a0cde133 job check one last time for completed job before returning 404 2025-02-21 19:57:49 +01:00
HugoCasa
521b6ba92c fix(frontend): improve rename workspace id UX (#5353) 2025-02-21 19:46:58 +01:00
HugoCasa
8a7730efa0 fix(frontend): missing config for Custom AI (#5351) 2025-02-21 18:31:11 +01:00
HugoCasa
7d73decd8d fix(frontend): invalid username for superadmin in some workspaces (#5350) 2025-02-21 18:30:55 +01:00
Ruben Fiszel
7e5469bddf nit 2025-02-21 18:30:37 +01:00
Ruben Fiszel
911fcf2012 improve error message to run executable 2025-02-21 18:09:25 +01:00
Ruben Fiszel
3ac912fa30 fix: handle better forced value propagation in apps 2025-02-21 17:37:25 +01:00
Ruben Fiszel
6d11bb4a00 make cli success page configurable 2025-02-21 17:13:27 +01:00
Ruben Fiszel
6b31a6dee5 fix ansible for nsjail 2025-02-21 17:06:15 +01:00
HugoCasa
19d33bdc7c feat: provision from SSO preferred_username (#5347)
* feat: provision from SSO preferred_username

* update ee ref
2025-02-21 16:20:21 +01:00
Ruben Fiszel
3188bee46e fix: disable toggle is more consistently applied on arg inputs 2025-02-21 14:07:44 +01:00
Ruben Fiszel
03903d06bf nit ansible 2025-02-21 11:22:58 +01:00
Ruben Fiszel
22ff20b6ab nit ansible 2025-02-21 10:12:46 +01:00
Ruben Fiszel
420e65c71c nit ansible galaxy 2025-02-21 10:03:51 +01:00
Ruben Fiszel
90ceb6e4fd uv tool install ansible 2025-02-21 09:50:57 +01:00
Ruben Fiszel
5ad473e12e install ansible with uv pip 2025-02-21 09:44:40 +01:00
Ruben Fiszel
aace016c0c nits install 2025-02-21 09:26:25 +01:00
Ruben Fiszel
676b78b15d fix(image): use debian image instead of python image as base 2025-02-21 09:20:57 +01:00
Ruben Fiszel
312b852c6c do not install pip at all 2025-02-21 08:49:48 +01:00
Ruben Fiszel
f05878271b fix: do not pin python patch version in docker preinstalled python 2025-02-21 08:47:06 +01:00
Ruben Fiszel
c6601da3d8 fix: initialize s3 file input if value already present 2025-02-20 22:05:57 +01:00
Ruben Fiszel
3430f9c439 fix: fix schedule run now args 2025-02-20 19:45:05 +01:00
Ruben Fiszel
c90fe387e8 fix: schedules do not accept 5 units cron syntax on update/create anymore 2025-02-20 17:59:32 +01:00
wendrul
5de1c3ca22 Remove unused import (#5343) 2025-02-20 16:25:24 +01:00
Ruben Fiszel
215dde2813 chore(main): release 1.466.3 (#5341)
* chore(main): release 1.466.3

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-20 14:34:20 +01:00
wendrul
9ba66eacd2 fix: remove db streaming to avoid backpressure on db (#5342) 2025-02-20 14:27:28 +01:00
HugoCasa
03f88349c8 fix(frontend): add warning when integer number if too big for frontend (#5340) 2025-02-20 14:19:27 +01:00
pyranota
5740679627 fix windows build (#5336) 2025-02-20 09:16:43 +01:00
Ruben Fiszel
05150a0118 chore(main): release 1.466.2 (#5335)
* chore(main): release 1.466.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-20 03:13:13 +01:00
Ruben Fiszel
affb0b4c72 fix: add proxy envs (http_proxy) to uv install 2025-02-20 03:10:09 +01:00
Ruben Fiszel
3e03f86e7d chore(main): release 1.466.1 (#5334)
* chore(main): release 1.466.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-20 03:02:39 +01:00
Ruben Fiszel
39c0dd3736 fix(python): PYTHON_PATH overrides python from uv 2025-02-20 02:54:51 +01:00
Ruben Fiszel
d5b3a04b0a fix(cli): improve flow cli dependency error clarity 2025-02-20 02:48:17 +01:00
Ruben Fiszel
2c67e84abe fix(cli): improve dependency job error message (logs in result) 2025-02-20 01:52:18 +01:00
Ruben Fiszel
dcc0d35e97 fix(cli): improve cli dependency error clarity 2025-02-20 01:51:47 +01:00
pyranota
5a3ca3c8f2 fix windows build (#5333) 2025-02-20 00:26:09 +01:00
Ruben Fiszel
00d2d08e75 chore(main): release 1.466.0 (#5326)
* chore(main): release 1.466.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-19 22:25:32 +01:00
Ruben Fiszel
74c0a10c3a fix: improve app db explorer handling of always identity columns 2025-02-19 21:49:25 +01:00
Alexander Petric
ee4a9c6592 teams cleanup cleanup (#5330)
* sqlx

* repo ref

* adding teams channel to criticalerrorchannel in non-ee
2025-02-19 19:01:18 +01:00
pyranota
4ad654fcf0 feat: remove pip fallback option for python and ansible (#5186) 2025-02-19 17:33:11 +01:00
HugoCasa
35d5293fba feat: add support for gemini (#5235)
* feat: add support for gemini

* nit

* fix
2025-02-19 15:03:35 +01:00
Ruben Fiszel
defa66eb06 add otel to rh images 2025-02-19 08:11:06 +01:00
Ruben Fiszel
44a6a62fbe fix(apps): font-size of title text not screen dependent 2025-02-19 01:31:03 +01:00
Ruben Fiszel
e19fc6723e chore(main): release 1.465.0 (#5324)
* chore(main): release 1.465.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-18 23:54:38 +01:00
dieriba
58a67a3ac0 feat: SQS triggers (#5182)
* feat: first commit

* fix: npm check

* fix: openapi file

* feat: update openapi and migration

* feat: basic implementation done

* fix: fix: no used function when no feature

* feat: capture done

* Update capture.rs

* nits: change sqs trigger

* fix: make migration great again

* feat: add message attributes

* feat: nits: fix error messages, remove console.log and add try catch

* update sqs icon and ee feature for sqs_trigger

* update: change sqs name casing and added test connection button

* nits: update Icon and add create from template button

* fix: ci build and error compilation

* update migration type sqs

* update link on create from template button for sqs, add archive in workspace export and update sqlx

* fix: ci

* Update SqsTriggerEditorInner.svelte

* add link to docs, use generic function for resource and fix import error

* chore: update .github ci

* nits: remove empty

* update to match ee repo changement

* Update backend/windmill-api/src/resources.rs

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* add sqs handling for the cli and refacoring sqsEditorInner

* Update cli/sync.ts

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* fix: add break to switch statement for sqs case

* fix: display aws_resource_path when retrieve or create a new trigger

* rework sqs ui, fix postgres optional port

* fix: ci

* update ui for trigger

* update repo ref and specific

* feat: add ready endpoints for workers to enterprise

* update ref

* Update frontend/src/lib/script_helpers.ts

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

---------

Co-authored-by: HugoCasa <hugo@casademont.ch>
Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2025-02-18 23:49:45 +01:00
Ruben Fiszel
0e72991476 fix: fix rendering of app components without component inputs 2025-02-18 23:34:17 +01:00
Ruben Fiszel
8559c4e23e chore(main): release 1.464.0 (#5322)
* chore(main): release 1.464.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com>
2025-02-18 21:57:34 +01:00
Ruben Fiszel
1ef482e8ae feat: add ready endpoints for workers to enterprise 2025-02-18 21:49:35 +01:00
Ruben Fiszel
52ad48a910 back to pipe in bash for efficiency purposes 2025-02-18 20:08:25 +01:00
201 changed files with 7411 additions and 1923 deletions

View File

@@ -7,7 +7,6 @@ services:
# image: mcr.microsoft.com/vscode/devcontainers/rust:bullseye
environment:
- DENO_PATH=/usr/local/cargo/bin/deno
- PYTHON_PATH=/usr/bin/python3
- NSJAIL_PATH=/bin/nsjail
volumes:
- .:/workspace:cached

View File

@@ -40,7 +40,7 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=oven/bun:1.1.31 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.2.3 /usr/local/bin/bun /usr/bin/bun
ARG TARGETPLATFORM

View File

@@ -42,9 +42,6 @@ jobs:
- uses: actions/setup-go@v2
with:
go-version: 1.21.5
- uses: actions/setup-python@v2
with:
python-version: 3.11
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.1.43
@@ -64,7 +61,7 @@ jobs:
deno --version && bun -v && go version && python3 --version &&
SQLX_OFFLINE=true
DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill
DISABLE_EMBEDDING=true RUST_LOG=info PYTHON_PATH=$(which python)
DISABLE_EMBEDDING=true RUST_LOG=info
DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go)
UV_PATH=$(which uv) cargo test --features
enterprise,deno_core,license,python,rust,scoped_cache --all --

View File

@@ -8,317 +8,22 @@ on:
jobs:
benchmark_single:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
--shm-size=2g
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 30
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_single
path: |
*.json
benchmark_dedicated:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: dedicated
DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
options: >-
--pull always --restart unless-stopped --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 20
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
--no-warm-up -c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_dedicated
path: |
*.json
benchmark_4workers:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
windmill_1:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_2:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_3:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 20
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
--workers 4
--factor 3
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_4workers
path: |
*.json
benchmark_8workers:
runs-on: ubicloud-standard-8
services:
postgres:
image: postgres
env:
POSTGRES_DB: windmill
POSTGRES_PASSWORD: changeme
POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB"
options: >-
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
windmill:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
http://localhost:8000/api/version"
ports:
- 8000:8000
windmill_1:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_2:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_3:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_4:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_5:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_6:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
windmill_7:
image: ghcr.io/windmill-labs/windmill-ee:main
env:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- name: benchmark
timeout-minutes: 20
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
--workers 8
--factor 3
- name: Save benchmark results
uses: actions/upload-artifact@v4
with:
name: benchmark_8workers
path: |
*.json
benchmark_graphs:
runs-on: ubicloud
needs:
- benchmark_single
- benchmark_dedicated
- benchmark_4workers
- benchmark_8workers
steps:
- uses: denoland/setup-deno@v2
with:
deno-version: v1.x
- uses: actions/checkout@v4
with:
ref: benchmarks
- name: Download benchmark results
uses: actions/download-artifact@v4
with:
merge-multiple: true
- name: graphs
run: deno run --unstable -A -r
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts
-c
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json
- name: Push changes
run: |
ls -la
pwd
git add .
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git commit -m "Update benchmarks"
git push
docker run -d --network=host -e POSTGRES_PASSWORD=changeme -e POSTGRES_USER=postgres -e POSTGRES_DB=windmill -e POSTGRES_INITDB_ARGS="-c log_duration=on -c log_statement=all -c log_min_duration_statement=0 -c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB -c shared_preload_libraries=auto_explain -c auto_explain.log_min_duration=5 -c auto_explain.log_analyze=on -c auto_explain.log_timing=on -c auto_explain.log_buffers=on -c auto_explain.log_verbose=on \
-c log_statement=all \
-c log_min_duration_statement=0 \
-c shared_buffers=2GB \
-c work_mem=32MB \
-c effective_cache_size=4GB" \
postgres
sleep 5
docker run -d -it --network=host -e DATABASE_URL=postgres://postgres:changeme@localhost/windmill ghcr.io/windmill-labs/windmill:main
sleep 10
deno run --unstable -A -r https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json

View File

@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,http_trigger,zip,oauth2,kafka,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,http_trigger,zip,oauth2,kafka,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}

View File

@@ -45,7 +45,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
- name: Rename binary with corresponding architecture
run: |

View File

@@ -158,7 +158,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
@@ -452,49 +452,6 @@ jobs:
${{ steps.meta-ee-public.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
build_ee_reports_privately:
needs: [build_ee_nsjail]
runs-on: ubicloud
if: github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch')
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
- name: Login to registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,enable=true,priority=100,prefix=,suffix=,format=short
- name: Build and push publicly ee reports
uses: depot/build-push-action@v1
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
file: "./docker/DockerfileReports"
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |
${{ steps.meta-ee-public.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
publish_ecr_s3:
needs: [build_ee_nsjail]

View File

@@ -47,7 +47,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,php,mysql,mssql,bigquery,oracledb,postgres_trigger,websocket,python,smtp,csharp,static_frontend,rust
- name: Rename binary with corresponding architecture
run: |

View File

@@ -1,5 +1,101 @@
# Changelog
## [1.467.1](https://github.com/windmill-labs/windmill/compare/v1.467.0...v1.467.1) (2025-02-22)
### Bug Fixes
* add uv bin path to PATH ([85993cc](https://github.com/windmill-labs/windmill/commit/85993ccac2abc2295e0f1b21544a6674fcf43411))
* app markdown is selectable in preview mode ([0aa6a39](https://github.com/windmill-labs/windmill/commit/0aa6a39cad16bff74adf3326d47ba0ba9851ccf6))
* init_script do not need to use nsjail even in nsjail mode ([e92a46b](https://github.com/windmill-labs/windmill/commit/e92a46b088088148d13a8e625a828657bcf44fe3))
## [1.467.0](https://github.com/windmill-labs/windmill/compare/v1.466.3...v1.467.0) (2025-02-21)
### Features
* enable rust AI gen/fix/edit ([#5349](https://github.com/windmill-labs/windmill/issues/5349)) ([d9844fd](https://github.com/windmill-labs/windmill/commit/d9844fd7f7cf89a0914176944d4af0b485ed3f3c))
* provision from SSO preferred_username ([#5347](https://github.com/windmill-labs/windmill/issues/5347)) ([19d33bd](https://github.com/windmill-labs/windmill/commit/19d33bdc7c4633f0c338c77de1d316f733e4304a))
### Bug Fixes
* disable toggle is more consistently applied on arg inputs ([3188bee](https://github.com/windmill-labs/windmill/commit/3188bee46e3dc46a699096bd3c2668df0cbdb9a1))
* do not pin python patch version in docker preinstalled python ([f058782](https://github.com/windmill-labs/windmill/commit/f05878271becb28f83678c5b0ae498d0192b2458))
* fix app component header buttons ([ab1c15d](https://github.com/windmill-labs/windmill/commit/ab1c15d92f3f86f4bd8d782fa6a806a59f30fdf1))
* fix schedule run now args ([3430f9c](https://github.com/windmill-labs/windmill/commit/3430f9c4390b6c630086394ddfaf1a1b2030c78f))
* **frontend:** improve rename workspace id UX ([#5353](https://github.com/windmill-labs/windmill/issues/5353)) ([521b6ba](https://github.com/windmill-labs/windmill/commit/521b6ba92c86a55b9977463ae05ecd4fca400ce4))
* **frontend:** invalid username for superadmin in some workspaces ([#5350](https://github.com/windmill-labs/windmill/issues/5350)) ([7d73dec](https://github.com/windmill-labs/windmill/commit/7d73decd8dc7039ef84915994074c07dc51280c9))
* **frontend:** missing config for Custom AI ([#5351](https://github.com/windmill-labs/windmill/issues/5351)) ([8a7730e](https://github.com/windmill-labs/windmill/commit/8a7730efa06283e72292d894584b279c908a7604))
* handle better forced value propagation in apps ([3ac912f](https://github.com/windmill-labs/windmill/commit/3ac912fa308fbbf6cf41562cfdbe8eea7c1cc372))
* **image:** use debian image instead of python image as base ([676b78b](https://github.com/windmill-labs/windmill/commit/676b78b15db8e1c749107fa41c4c98ab3a37154e))
* initialize s3 file input if value already present ([c6601da](https://github.com/windmill-labs/windmill/commit/c6601da3d8557af9d32b0202bf50c40b89d481a9))
* schedules do not accept 5 units cron syntax on update/create anymore ([c90fe38](https://github.com/windmill-labs/windmill/commit/c90fe387e882f7767c3b3621e5e230fc8acd80b0))
## [1.466.3](https://github.com/windmill-labs/windmill/compare/v1.466.2...v1.466.3) (2025-02-20)
### Bug Fixes
* **frontend:** add warning when integer number if too big for frontend ([#5340](https://github.com/windmill-labs/windmill/issues/5340)) ([03f8834](https://github.com/windmill-labs/windmill/commit/03f88349c8730bfbb4613105c35482b4f3fadd64))
* remove db streaming to avoid backpressure on db ([#5342](https://github.com/windmill-labs/windmill/issues/5342)) ([9ba66ea](https://github.com/windmill-labs/windmill/commit/9ba66eacd28175607900a7d2294584662b4c26a2))
## [1.466.2](https://github.com/windmill-labs/windmill/compare/v1.466.1...v1.466.2) (2025-02-20)
### Bug Fixes
* add proxy envs (http_proxy) to uv install ([affb0b4](https://github.com/windmill-labs/windmill/commit/affb0b4c720551f7f1c7fa5315e3b39e5580b732))
## [1.466.1](https://github.com/windmill-labs/windmill/compare/v1.466.0...v1.466.1) (2025-02-20)
### Bug Fixes
* **cli:** improve cli dependency error clarity ([dcc0d35](https://github.com/windmill-labs/windmill/commit/dcc0d35e971ab3df6a0122dc881b968e8221f40f))
* **cli:** improve dependency job error message (logs in result) ([2c67e84](https://github.com/windmill-labs/windmill/commit/2c67e84abe98a3c43972cf5555536104119c6527))
* **cli:** improve flow cli dependency error clarity ([d5b3a04](https://github.com/windmill-labs/windmill/commit/d5b3a04b0ab5f003c4c512cc9ba74eb620a3afc1))
* **python:** PYTHON_PATH overrides python from uv ([39c0dd3](https://github.com/windmill-labs/windmill/commit/39c0dd3736da0722c7e18d84183c0e9b06cf2839))
## [1.466.0](https://github.com/windmill-labs/windmill/compare/v1.465.0...v1.466.0) (2025-02-19)
### Features
* add support for gemini ([#5235](https://github.com/windmill-labs/windmill/issues/5235)) ([35d5293](https://github.com/windmill-labs/windmill/commit/35d5293fba47d368e503e9781719e6e9ccc96713))
* remove `pip` fallback option for python and ansible ([#5186](https://github.com/windmill-labs/windmill/issues/5186)) ([4ad654f](https://github.com/windmill-labs/windmill/commit/4ad654fcf0c603aefc5a9b5c41da1ffa24b99d2d))
### Bug Fixes
* **apps:** font-size of title text not screen dependent ([44a6a62](https://github.com/windmill-labs/windmill/commit/44a6a62fbe3a9cae79e2d7ab7efd119f559aa374))
* improve app db explorer handling of always identity columns ([74c0a10](https://github.com/windmill-labs/windmill/commit/74c0a10c3a8a4848341456635f36c0c2061b7943))
## [1.465.0](https://github.com/windmill-labs/windmill/compare/v1.464.0...v1.465.0) (2025-02-18)
### Features
* SQS triggers ([#5182](https://github.com/windmill-labs/windmill/issues/5182)) ([58a67a3](https://github.com/windmill-labs/windmill/commit/58a67a3ac0c57b9504a90a6e454f738cf0810e21))
### Bug Fixes
* fix rendering of app components without component inputs ([0e72991](https://github.com/windmill-labs/windmill/commit/0e72991476ba932a526e1b4cf42bad157be2cfdb))
## [1.464.0](https://github.com/windmill-labs/windmill/compare/v1.463.6...v1.464.0) (2025-02-18)
### Features
* add ready endpoints for workers to enterprise ([1ef482e](https://github.com/windmill-labs/windmill/commit/1ef482e8aee9433c518ce3cbc5bc38174e27c34f))
### Bug Fixes
* **bash:** allow process substitution on nsjail ([d4f61f1](https://github.com/windmill-labs/windmill/commit/d4f61f13fd6a9c2e5707738fba960b7fd926230c))
* **bash:** improve bash last line as result reliability using bash process substitution ([#5321](https://github.com/windmill-labs/windmill/issues/5321)) ([138cedf](https://github.com/windmill-labs/windmill/commit/138cedf1da91290f97c19513daf0c1981488a94a))
## [1.463.6](https://github.com/windmill-labs/windmill/compare/v1.463.5...v1.463.6) (2025-02-18)

View File

@@ -1,6 +1,5 @@
ARG DEBIAN_IMAGE=debian:bookworm-slim
ARG RUST_IMAGE=rust:1.83-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
FROM ${RUST_IMAGE} AS rust_base
@@ -81,7 +80,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
FROM ${PYTHON_IMAGE}
FROM ${DEBIAN_IMAGE}
ARG TARGETPLATFORM
ARG POWERSHELL_VERSION=7.3.5
@@ -102,11 +101,13 @@ ARG WITH_GIT=true
ARG LATEST_STABLE_PY=3.11.10
ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime
ENV UV_PYTHON_PREFERENCE=only-managed
ENV UV_TOOL_BIN_DIR=/usr/local/bin
ENV PATH /usr/local/bin:/root/.local/bin:$PATH
RUN pip install --upgrade pip==24.2
RUN apt-get update \
&& apt-get install -y ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common \
&& apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
@@ -167,12 +168,15 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.6.2/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
# Preinstall python runtimes
RUN uv python install 3.11.10
RUN uv python install 3.11
RUN uv python install $LATEST_STABLE_PY
RUN uv venv
RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \
&& rm -rf /var/lib/apt/lists/*
@@ -182,14 +186,12 @@ RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm
ENV TZ=Etc/UTC
RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.43 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.2.3 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer

View File

@@ -330,7 +330,7 @@ you to have it being synced automatically everyday.
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
| PYTHON_PATH | | The path to the python binary if wanting to not have it managed by uv. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| GOPROXY | | The GOPROXY env variable to use | Worker |

View File

@@ -19,7 +19,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n sqs_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Bool",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1625a84fbcf8c5f77eb0519f60d9418f85d4d3d599f4177403fad5ad99380715"
}

View File

@@ -0,0 +1,50 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'sqs' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "trigger_config!: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
true
]
},
"hash": "1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546"
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'sqs' AND \n server_id IS NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8"
}

View File

@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n sqs_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf"
}

View File

@@ -0,0 +1,106 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM sqs_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "queue_url",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "aws_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "message_attributes",
"type_info": "TextArray"
},
{
"ordinal": 4,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 14,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
true,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false
]
},
"hash": "2b6c13191484b0c664f35e2c811082b00c44fd4a7e98a11b714133674a7b6da7"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow SET path = $1, summary = $2, description = $3,dependency_job = NULL, draft_only = NULL, tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, on_behalf_of_email = $7, value = $8, schema = $9::text::json, edited_by = $10, edited_at = now()\n WHERE path = $11 AND workspace_id = $12",
"query": "UPDATE flow SET path = $1, summary = $2, description = $3,dependency_job = NULL, lock_error_logs = '', draft_only = NULL, tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, on_behalf_of_email = $7, value = $8, schema = $9::text::json, edited_by = $10, edited_at = now()\n WHERE path = $11 AND workspace_id = $12",
"describe": {
"columns": [],
"parameters": {
@@ -21,5 +21,5 @@
},
"nullable": []
},
"hash": "ba8bde5018fdf7b12f85cd3a6557c4accfc78bf160c1277f35d9d8ddcd056963"
"hash": "2b9607ed838c8c62eb0f2856420389f7be648f52edbb875ff52c96219ed3ba84"
}

View File

@@ -0,0 +1,107 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n aws_resource_path,\n message_attributes,\n queue_url,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM \n sqs_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "aws_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "message_attributes",
"type_info": "TextArray"
},
{
"ordinal": 2,
"name": "queue_url",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 13,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 14,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true,
false,
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false
]
},
"hash": "2ef82fad8a6ccdc66228cfbce5393de351653ab9ac171fa0eea447c905440867"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n sqs_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_ansible', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "4a804ee30bfe86c4e2c15a9f6511be5adf0dd22cb942fac64b439fb4e20df447"
}

View File

@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n sqs_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71"
}

View File

@@ -27,7 +27,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}
@@ -60,7 +61,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -19,7 +19,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE workspace_settings\n SET teams_command_script = NULL,\n teams_team_id = NULL,\n teams_team_name = NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "65c339164e7669360d231d70105849e72bdc197c17c0fc51777c1dc9267e2daf"
}

View File

@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53"
}

View File

@@ -30,7 +30,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE global_settings\n SET value = (\n SELECT COALESCE(jsonb_agg(elem), '[]'::jsonb)\n FROM jsonb_array_elements(value) AS elem\n WHERE NOT (elem ? 'teams_channel')\n )\n WHERE name = 'critical_error_channels'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "81b06122c7a12a314d8905ba5c7c14aa7614f2610e79a8c7302eaa63fb74984d"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\"\n ",
"query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ",
"describe": {
"columns": [
{
@@ -27,6 +27,11 @@
"ordinal": 4,
"name": "postgres_used!",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "sqs_used!",
"type_info": "Bool"
}
],
"parameters": {
@@ -39,8 +44,9 @@
null,
null,
null,
null,
null
]
},
"hash": "24178c21aadc1aed90f31e9362c6505a642c8f04b883c278b07e7ef5956ce121"
"hash": "93d0ee34c7b7c56ab9cae28071f8e5fdaff0e601e2760cc5653d197c51b106bb"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock_error_logs FROM flow WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock_error_logs",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO sqs_trigger (\n aws_resource_path,\n queue_url,\n message_attributes,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7,\n $8,\n $9,\n $10\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Varchar",
"Bool",
"Varchar"
]
},
"nullable": []
},
"hash": "9b9bc21023f10a0b4bb45879c7da8e119bffd2982b97fc900358506a0a14bbb8"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n sqs_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n sqs_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "b386d320f9fe1d569a16e6626b723b4376a93342702856da2ac70f6bbfc7933e"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow SET lock_error_logs = $1 WHERE path = $2 AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "ba285edd1c1b1e400e85168ff4f05cf5281fd341096d433c7c0e5712e7726fb0"
}

View File

@@ -22,7 +22,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -19,7 +19,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow SET lock_error_logs = NULL WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "d1876c46c0b1aba168efaebd3a056e999c400998eb699d862d718e7ab4c1f427"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "d8186f0cee285aa50db7626409aec7e0504b068ffa8bb185d9384ce1422fd3d1"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE \n FROM \n sqs_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "dea056c89313f4facd62cbbc9fa33ba30fa85efcc83fafe4dd7b4e535b96a8d8"
}

View File

@@ -27,7 +27,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -35,7 +35,8 @@
"kafka",
"email",
"nats",
"postgres"
"postgres",
"sqs"
]
}
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow (workspace_id, path, summary, description, dependency_job, draft_only, tag, dedicated_worker, visible_to_runner_only, on_behalf_of_email, value, schema, edited_by, edited_at) \n VALUES ($1, $2, $3, $4, NULL, $5, $6, $7, $8, $9, $10, $11::text::json, $12, now())",
"query": "INSERT INTO flow (workspace_id, path, summary, description, dependency_job, lock_error_logs, draft_only, tag, dedicated_worker, visible_to_runner_only, on_behalf_of_email, value, schema, edited_by, edited_at) \n VALUES ($1, $2, $3, $4, NULL, '', $5, $6, $7, $8, $9, $10, $11::text::json, $12, now())",
"describe": {
"columns": [],
"parameters": {
@@ -21,5 +21,5 @@
},
"nullable": []
},
"hash": "4205d237c123d8d1c9ff2d61118027a80ccc8cd75e9703cb1d014b45f57c2be6"
"hash": "e4f1ee1568ce3c186b569421c6c8a3039f73d04fc53c67c70e67371f06416ef3"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE global_settings SET value = $1 WHERE name = 'teams'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "e565f3b2e51059f563d18a8a9442bcae9640cee7b936820cb46c011222a77ff0"
}

View File

@@ -0,0 +1,104 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n queue_url,\n aws_resource_path,\n message_attributes,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled\n FROM\n sqs_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "queue_url",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "aws_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "message_attributes",
"type_info": "TextArray"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 13,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 14,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
true,
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
true,
false
]
},
"hash": "e6adaebcade2e25be800e8b888b23c94caae2421f3cae7c06d6346bd6de1d94a"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "ed318070b26861fda2d591a4356fdbeb6c7fdc965be43bddb010fd8299af1286"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n sqs_trigger \n SET \n aws_resource_path = $1,\n queue_url = $2,\n message_attributes = $3, \n is_flow = $4, \n edited_by = $5, \n email = $6,\n script_path = $7,\n path = $8,\n edited_at = now(), \n error = NULL,\n server_id = NULL\n WHERE \n workspace_id = $9 AND \n path = $10\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
"Bool",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "f68d23841e0e31cdf8633aaf0f32777e04e4965682e5b37afbe84194b756d5f5"
}

View File

@@ -11,5 +11,5 @@
"remote.autoForwardPorts": true,
"conventionalCommits.scopes": [
"restructring triggers, decoding trigger message on work"
],
]
}

3416
backend/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.463.6"
version = "1.467.1"
authors.workspace = true
edition.workspace = true
@@ -30,7 +30,7 @@ members = [
]
[workspace.package]
version = "1.463.6"
version = "1.467.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -74,6 +74,7 @@ mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
websocket = ["windmill-api/websocket"]
postgres_trigger = ["windmill-api/postgres_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger"]
python = ["windmill-worker/python"]
smtp = ["windmill-api/smtp", "windmill-common/smtp"]
csharp = ["windmill-worker/csharp"]
@@ -159,12 +160,11 @@ windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-api-client = { path = "./windmill-api-client" }
v8 = "=130.0.7" # Exact version
memchr = "2.7.4"
axum = { version = "^0.7", features = ["multipart"] }
headers = "^0"
hyper = { version = "^1", features = ["full"] }
tokio = { version = "^1.42.0", features = ["full", "tracing"] }
tokio = { version = "^1.42.0", features = ["full", "tracing", "time"] }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors"] }
tower-cookies = "^0.10"
@@ -211,22 +211,28 @@ itertools = "^0"
regex = "^1"
semver = "^1"
deno_fetch = "0.203.0"
deno_tls = "0.166.0"
deno_console = "0.179.0"
deno_url = "0.179.0"
deno_webidl = "0.179.0"
deno_web = "0.210.0"
deno_net = "0.171.0"
deno_core = "0.321.0"
deno_ast = { version = "=0.43.3", features = ["transpiling"] }
deno_permissions = "0.39.0"
v8 = "=134.4.0" # Exact version
deno_fetch = "0.216.0"
deno_tls = "0.179.0"
deno_console = "0.192.0"
deno_url = "0.192.0"
deno_webidl = "0.192.0"
deno_web = "0.223.0"
deno_io = "0.102.0"
deno_net = "0.184.0"
deno_core = "0.338.0"
deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_permissions = "0.51.0"
deno_runtime = { version = "0.200.0", features = ["transpile"] }
deno_telemetry = "0.14.0"
deno_error = "=0.5.5"
swc_common = "=0.37.5"
swc_ecma_parser = "=0.149.1"
swc_ecma_ast = "=0.118.2"
swc_ecma_visit = "=0.104.8"
async-recursion = "^1"
base64 = "^0"
@@ -263,9 +269,9 @@ once_cell = "1.17.1"
gosyn = "0.2.6"
bytes = "1.4.0"
gethostname = "0.4.3"
wasm-bindgen = "=0.2.92"
serde-wasm-bindgen = "0.6.5"
wasm-bindgen-test = "0.3.42"
wasm-bindgen = "^0"
serde-wasm-bindgen = "^0"
wasm-bindgen-test = "^0"
convert_case = "0.6.0"
getrandom = "0.2"
tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]}
@@ -306,6 +312,7 @@ datafusion = "39.0.0"
object_store = { version = "0.10.0", features = ["aws", "azure"] }
openidconnect = { version = "4.0.0-rc.1" }
aws-config = "^1"
aws-sdk-sqs = "1.57.0"
aws-sdk-sts = "^1"
crc = "^3"

View File

@@ -1 +1 @@
5d25cf2cd15c1953794045fd7debea14a33c7519
2dca712cde013e7dda96dc3162e47a148973a0dc

View File

@@ -0,0 +1,2 @@
-- Add down migration script here
DROP TABLE IF EXISTS sqs_trigger;

View File

@@ -0,0 +1,69 @@
-- Add up migration script here
CREATE TABLE sqs_trigger(
path VARCHAR(255) NOT NULL,
queue_url VARCHAR(255) NOT NULL,
aws_resource_path VARCHAR(255) NOT NULL,
message_attributes TEXT[],
script_path VARCHAR(255) NOT NULL,
is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
edited_by VARCHAR(50) NOT NULL,
email VARCHAR(255) NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
extra_perms JSONB NULL,
error TEXT NULL,
server_id VARCHAR(50) NULL,
last_server_ping TIMESTAMPTZ NULL,
enabled BOOLEAN NOT NULL,
CONSTRAINT PK_sqs_trigger PRIMARY KEY (path,workspace_id),
CONSTRAINT fk_sqs_trigger_workspace FOREIGN KEY (workspace_id)
REFERENCES workspace(id) ON DELETE CASCADE
);
GRANT ALL ON sqs_trigger TO windmill_user;
GRANT ALL ON sqs_trigger TO windmill_admin;
ALTER TABLE sqs_trigger ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON sqs_trigger FOR ALL TO windmill_admin USING (true);
CREATE POLICY see_folder_extra_perms_user_select ON sqs_trigger FOR SELECT TO windmill_user
USING (SPLIT_PART(sqs_trigger.path, '/', 1) = 'f' AND SPLIT_PART(sqs_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_insert ON sqs_trigger FOR INSERT TO windmill_user
WITH CHECK (SPLIT_PART(sqs_trigger.path, '/', 1) = 'f' AND SPLIT_PART(sqs_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON sqs_trigger FOR UPDATE TO windmill_user
USING (SPLIT_PART(sqs_trigger.path, '/', 1) = 'f' AND SPLIT_PART(sqs_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_delete ON sqs_trigger FOR DELETE TO windmill_user
USING (SPLIT_PART(sqs_trigger.path, '/', 1) = 'f' AND SPLIT_PART(sqs_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_own ON sqs_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(sqs_trigger.path, '/', 1) = 'u' AND SPLIT_PART(sqs_trigger.path, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON sqs_trigger FOR ALL TO windmill_user
USING (SPLIT_PART(sqs_trigger.path, '/', 1) = 'g' AND SPLIT_PART(sqs_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
CREATE POLICY see_extra_perms_user_select ON sqs_trigger FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
CREATE POLICY see_extra_perms_user_insert ON sqs_trigger FOR INSERT TO windmill_user
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_update ON sqs_trigger FOR UPDATE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_user_delete ON sqs_trigger FOR DELETE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups_select ON sqs_trigger FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
CREATE POLICY see_extra_perms_groups_insert ON sqs_trigger FOR INSERT TO windmill_user
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_update ON sqs_trigger FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE POLICY see_extra_perms_groups_delete ON sqs_trigger FOR DELETE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));

View File

@@ -0,0 +1 @@
-- Add down migration script here

View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'sqs';

View File

@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE flow DROP COLUMN lock_error_logs;

View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TABLE flow ADD COLUMN lock_error_logs TEXT;

View File

@@ -318,7 +318,7 @@ async fn parse_python_imports_inner(
}
}
const STDIMPORTS: [&str; 302] = [
const STDIMPORTS: [&str; 303] = [
"--future--",
"-abc",
"-aix-support",
@@ -620,5 +620,6 @@ const STDIMPORTS: [&str; 302] = [
"zipapp",
"zipfile",
"zipimport",
"zlib",
"",
];

View File

@@ -59,18 +59,15 @@ use tikv_jemallocator::Jemalloc;
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;
#[cfg(feature = "enterprise")]
use windmill_common::METRICS_ADDR;
#[cfg(feature = "parquet")]
use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR,
LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR,
PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR,
TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR,
TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
@@ -901,14 +898,25 @@ Windmill Community Edition {GIT_VERSION}
};
let metrics_f = async {
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
#[cfg(not(feature = "enterprise"))]
tracing::error!("Metrics are only available in the EE, ignoring...");
let enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
#[cfg(feature = "enterprise")]
windmill_common::serve_metrics(*METRICS_ADDR, _killpill_phase2_rx, num_workers > 0)
.await;
#[cfg(not(all(feature = "enterprise", feature = "prometheus")))]
if enabled {
tracing::error!("Metrics are only available in the EE, ignoring...");
}
#[cfg(all(feature = "enterprise", feature = "prometheus"))]
if let Err(e) = windmill_common::serve_metrics(
*windmill_common::METRICS_ADDR,
_killpill_phase2_rx,
num_workers > 0,
enabled,
)
.await
{
tracing::error!("Error serving metrics: {e:#}");
}
Ok(()) as anyhow::Result<()>
};
@@ -1033,10 +1041,8 @@ pub async fn run_workers(
let mut handles = Vec::with_capacity(num_workers as usize);
for x in [
LOCK_CACHE_DIR,
TMP_LOGS_DIR,
UV_CACHE_DIR,
TAR_PIP_CACHE_DIR,
DENO_CACHE_DIR,
DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM,
@@ -1049,7 +1055,6 @@ pub async fn run_workers(
TAR_PY311_CACHE_DIR,
TAR_PY312_CACHE_DIR,
TAR_PY313_CACHE_DIR,
PIP_CACHE_DIR,
BUN_BUNDLE_CACHE_DIR,
GO_CACHE_DIR,
GO_BIN_CACHE_DIR,

View File

@@ -731,6 +731,22 @@ pub async fn delete_expired_items(db: &DB) -> () {
Err(e) => tracing::error!("Error deleting log file: {:?}", e),
}
#[cfg(not(feature = "enterprise"))]
let audit_retention_secs = 1 * 60 * 60 * 24 * 14;
#[cfg(feature = "enterprise")]
let audit_retention_secs = 1 * 60 * 60 * 24 * 365;
if let Err(e) = sqlx::query_scalar!(
"DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval",
audit_retention_secs,
)
.fetch_all(db)
.await
{
tracing::error!("Error deleting audit log on CE: {:?}", e);
}
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
match db.begin().await {

View File

@@ -1015,15 +1015,10 @@ fn spawn_test_worker(
tokio::sync::broadcast::Sender<()>,
tokio::task::JoinHandle<()>,
) {
for x in [
windmill_worker::LOCK_CACHE_DIR,
windmill_worker::GO_BIN_CACHE_DIR,
] {
std::fs::DirBuilder::new()
.recursive(true)
.create(x)
.expect("could not create initial worker dir");
}
std::fs::DirBuilder::new()
.recursive(true)
.create(windmill_worker::GO_BIN_CACHE_DIR)
.expect("could not create initial worker dir");
let (tx, rx) = tokio::sync::broadcast::channel(1);
let db = db.to_owned();
@@ -2905,7 +2900,7 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
.await
.unwrap()
.into_inner()
.subtype_0
.open_flow
.value
.modules;
modules.into_iter()

View File

@@ -29,6 +29,7 @@ oauth2 = ["dep:async-oauth2"]
http_trigger = ["dep:matchit"]
static_frontend = ["dep:rust-embed"]
postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"]
sqs_trigger = ["dep:aws-sdk-sqs", "dep:thiserror", "dep:aws-config"]
[dependencies]
windmill-queue.workspace = true
@@ -118,4 +119,6 @@ pg_escape = { workspace = true, optional = true }
byteorder = { workspace = true, optional = true }
thiserror = { workspace = true, optional = true }
rust_decimal = { workspace = true, optional = true }
rust-postgres-native-tls = { workspace = true, optional = true}
rust-postgres-native-tls = { workspace = true, optional = true}
aws-sdk-sqs = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true}

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.463.6
version: 1.467.1
title: Windmill API
contact:
@@ -2611,12 +2611,15 @@ paths:
type: boolean
postgres_used:
type: boolean
sqs_used:
type: boolean
required:
- http_routes_used
- websocket_used
- kafka_used
- nats_used
- postgres_used
- sqs_used
/w/{workspace}/users/list:
get:
summary: list users
@@ -5363,6 +5366,27 @@ paths:
schema:
$ref: "#/components/schemas/Flow"
/w/{workspace}/flows/deployment_status/p/{path}:
get:
summary: get flow deployment status
operationId: getFlowDeploymentStatus
tags:
- flow
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: flow status
content:
application/json:
schema:
type: object
properties:
lock_error_logs:
type: string
/w/{workspace}/flows/get_triggers_count/{path}:
get:
summary: get triggers count of flow
@@ -8564,6 +8588,196 @@ paths:
schema:
type: string
/w/{workspace}/sqs_triggers/create:
post:
summary: create sqs trigger
operationId: createSqsTrigger
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: new sqs trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/NewSqsTrigger"
responses:
"201":
description: sqs trigger created
content:
text/plain:
schema:
type: string
/w/{workspace}/sqs_triggers/update/{path}:
post:
summary: update sqs trigger
operationId: updateSqsTrigger
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/EditSqsTrigger"
responses:
"200":
description: sqs trigger updated
content:
text/plain:
schema:
type: string
/w/{workspace}/sqs_triggers/delete/{path}:
delete:
summary: delete sqs trigger
operationId: deleteSqsTrigger
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: sqs trigger deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/sqs_triggers/get/{path}:
get:
summary: get sqs trigger
operationId: getSqsTrigger
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: sqs trigger deleted
content:
application/json:
schema:
$ref: "#/components/schemas/SqsTrigger"
/w/{workspace}/sqs_triggers/list:
get:
summary: list sqs triggers
operationId: listSqsTriggers
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
required: true
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
- name: path
description: filter by path
in: query
schema:
type: string
- name: is_flow
in: query
schema:
type: boolean
- name: path_start
in: query
schema:
type: string
responses:
"200":
description: sqs trigger list
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/SqsTrigger"
/w/{workspace}/sqs_triggers/exists/{path}:
get:
summary: does sqs trigger exists
operationId: existsSqsTrigger
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: sqs trigger exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/sqs_triggers/setenabled/{path}:
post:
summary: set enabled sqs trigger
operationId: setSqsTriggerEnabled
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated sqs trigger enable
required: true
content:
application/json:
schema:
type: object
properties:
enabled:
type: boolean
required:
- enabled
responses:
"200":
description: sqs trigger enabled set
content:
text/plain:
schema:
type: string
/w/{workspace}/sqs_triggers/test:
post:
summary: test sqs connection
operationId: testSqsConnection
tags:
- sqs_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: test sqs connection
required: true
content:
application/json:
schema:
type: object
properties:
connection:
type: object
required:
- connection
responses:
"200":
description: successfuly connected to sqs
content:
text/plain:
schema:
type: string
/w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}:
get:
summary: check if postgres configuration is set to logical
@@ -9851,6 +10065,7 @@ paths:
kafka_trigger,
nats_trigger,
postgres_trigger,
sqs_trigger
]
responses:
"200":
@@ -9892,6 +10107,7 @@ paths:
kafka_trigger,
nats_trigger,
postgres_trigger,
sqs_trigger
]
requestBody:
description: acl to add
@@ -9944,6 +10160,7 @@ paths:
kafka_trigger,
nats_trigger,
postgres_trigger,
sqs_trigger
]
requestBody:
description: acl to add
@@ -11783,7 +12000,7 @@ components:
AIProvider:
type: string
enum: [openai, anthropic, mistral, deepseek, groq, openrouter, customai]
enum: [openai, anthropic, mistral, deepseek, googleai, groq, openrouter, customai]
AIResource:
type: object
@@ -13166,6 +13383,10 @@ components:
TriggerExtraProperty:
type: object
properties:
path:
type: string
script_path:
type: string
email:
type: string
extra_perms:
@@ -13179,22 +13400,23 @@ components:
edited_at:
type: string
format: date-time
is_flow:
type: boolean
required:
- path
- script_path
- email
- extra_perms
- workspace_id
- edited_by
- edited_at
- is_flow
HttpTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
path:
type: string
script_path:
type: string
route_path:
type: string
static_asset_config:
@@ -13208,8 +13430,6 @@ components:
type: string
required:
- s3
is_flow:
type: boolean
http_method:
type: string
enum:
@@ -13226,15 +13446,7 @@ components:
type: boolean
required:
- path
- edited_by
- edited_at
- script_path
- route_path
- extra_perms
- is_flow
- email
- workspace_id
- is_async
- requires_auth
- http_method
@@ -13357,20 +13569,16 @@ components:
type: number
nats_count:
type: number
sqs_count:
type: number
WebsocketTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
path:
type: string
script_path:
type: string
url:
type: string
is_flow:
type: boolean
server_id:
type: string
last_server_ping:
@@ -13401,15 +13609,7 @@ components:
type: boolean
required:
- path
- edited_by
- edited_at
- script_path
- url
- extra_perms
- is_flow
- email
- workspace_id
- enabled
- filters
- can_return_message
@@ -13519,6 +13719,88 @@ components:
required:
- runnable_result
SqsTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
queue_url:
type: string
aws_resource_path:
type: string
message_attributes:
type: array
items:
type: string
server_id:
type: string
last_server_ping:
type: string
format: date-time
error:
type: string
enabled:
type: boolean
required:
- queue_url
- aws_resource_path
- enabled
NewSqsTrigger:
type: object
properties:
queue_url:
type: string
aws_resource_path:
type: string
message_attributes:
type: array
items:
type: string
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
required:
- queue_url
- aws_resource_path
- path
- script_path
- is_flow
EditSqsTrigger:
type: object
properties:
queue_url:
type: string
aws_resource_path:
type: string
message_attributes:
type: array
items:
type: string
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
required:
- queue_url
- aws_resource_path
- path
- script_path
- is_flow
- enabled
Slot:
type: object
properties:
@@ -13600,12 +13882,6 @@ components:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
path:
type: string
script_path:
type: string
is_flow:
type: boolean
enabled:
type: boolean
postgres_resource_path:
@@ -13622,9 +13898,6 @@ components:
type: string
format: date-time
required:
- path
- script_path
- is_flow
- enabled
- postgres_resource_path
- replication_slot_name
@@ -13685,17 +13958,10 @@ components:
- replication_slot_name
KafkaTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
path:
type: string
edited_by:
type: string
edited_at:
type: string
format: date-time
script_path:
type: string
kafka_resource_path:
type: string
group_id:
@@ -13704,16 +13970,6 @@ components:
type: array
items:
type: string
is_flow:
type: boolean
extra_perms:
type: object
additionalProperties:
type: boolean
email:
type: string
workspace_id:
type: string
server_id:
type: string
last_server_ping:
@@ -13725,17 +13981,9 @@ components:
type: boolean
required:
- path
- edited_by
- edited_at
- script_path
- kafka_resource_path
- group_id
- topics
- extra_perms
- is_flow
- email
- workspace_id
- enabled
NewKafkaTrigger:
@@ -13793,17 +14041,10 @@ components:
- is_flow
NatsTrigger:
allOf:
- $ref: "#/components/schemas/TriggerExtraProperty"
type: object
properties:
path:
type: string
edited_by:
type: string
edited_at:
type: string
format: date-time
script_path:
type: string
nats_resource_path:
type: string
use_jetstream:
@@ -13816,16 +14057,6 @@ components:
type: array
items:
type: string
is_flow:
type: boolean
extra_perms:
type: object
additionalProperties:
type: boolean
email:
type: string
workspace_id:
type: string
server_id:
type: string
last_server_ping:
@@ -13837,17 +14068,9 @@ components:
type: boolean
required:
- path
- edited_by
- edited_at
- script_path
- nats_resource_path
- use_jetstream
- subjects
- extra_perms
- is_flow
- email
- workspace_id
- enabled
NewNatsTrigger:
@@ -14134,6 +14357,10 @@ components:
allOf:
- $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow"
- $ref: "#/components/schemas/FlowMetadata"
- type: object
properties:
lock_error_logs:
type: string
ExtraPerms:
type: object
@@ -14867,7 +15094,7 @@ components:
CaptureTriggerKind:
type: string
enum: [webhook, http, websocket, kafka, email, nats, postgres]
enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs]
Capture:
type: object

View File

@@ -380,6 +380,7 @@ pub enum AIProvider {
Anthropic,
Mistral,
DeepSeek,
GoogleAI,
Groq,
OpenRouter,
CustomAI,
@@ -389,6 +390,9 @@ impl AIProvider {
pub fn get_openai_compatible_base_url(&self) -> Result<Option<String>> {
match self {
AIProvider::DeepSeek => Ok(Some("https://api.deepseek.com/v1".to_string())),
AIProvider::GoogleAI => Ok(Some(
"https://generativelanguage.googleapis.com/v1beta/openai".to_string(),
)),
AIProvider::Groq => Ok(Some("https://api.groq.com/openai/v1".to_string())),
AIProvider::OpenRouter => Ok(Some("https://openrouter.ai/api/v1".to_string())),
AIProvider::CustomAI => Ok(None),
@@ -409,6 +413,7 @@ impl TryFrom<&str> for AIProvider {
"groq" => Ok(AIProvider::Groq),
"openrouter" => Ok(AIProvider::OpenRouter),
"deepseek" => Ok(AIProvider::DeepSeek),
"googleai" => Ok(AIProvider::GoogleAI),
"customai" => Ok(AIProvider::CustomAI),
_ => Err(Error::BadRequest(format!("Invalid AI provider: {}", s))),
}
@@ -417,7 +422,7 @@ impl TryFrom<&str> for AIProvider {
#[derive(Deserialize, Debug)]
pub struct AIResource {
pub path: String,
pub path: Option<String>,
pub provider: AIProvider,
}
@@ -472,28 +477,34 @@ async fn proxy(
.await?;
if ai_resource.is_none() {
return Err(Error::internal_err("AI resource not configured".to_string()));
return Err(Error::internal_err(
"AI resource not configured".to_string(),
));
}
let ai_resource = serde_json::from_value::<AIResource>(ai_resource.unwrap())
.map_err(|e| Error::BadRequest(e.to_string()))?;
let path = ai_resource.path.unwrap_or("".to_string());
if path.is_empty() {
return Err(Error::BadRequest("Resource path is empty".to_string()));
}
let resource = sqlx::query_scalar!(
"SELECT value
FROM resource
WHERE path = $1 AND workspace_id = $2",
&ai_resource.path,
&path,
&w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| {
Error::NotFound(format!(
"Could not find the {:?} resource at path {}, update the resource path in the workspace settings", ai_resource.provider, ai_resource.path
"Could not find the {:?} resource at path {}, update the resource path in the workspace settings", ai_resource.provider, path
))
})?;
(resource, ai_resource.path, ai_resource.provider)
(resource, path, ai_resource.provider)
};
if resource.is_none() {

View File

@@ -98,6 +98,7 @@ pub enum TriggerKind {
Kafka,
Email,
Nats,
Sqs,
Postgres,
}
@@ -110,6 +111,7 @@ impl fmt::Display for TriggerKind {
TriggerKind::Kafka => "kafka",
TriggerKind::Email => "email",
TriggerKind::Nats => "nats",
TriggerKind::Sqs => "sqs",
TriggerKind::Postgres => "postgres",
};
write!(f, "{}", s)
@@ -132,6 +134,14 @@ pub struct KafkaTriggerConfig {
pub group_id: String,
}
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
#[derive(Debug, Serialize, Deserialize)]
pub struct SqsTriggerConfig {
pub queue_url: String,
pub aws_resource_path: String,
pub message_attributes: Option<Vec<String>>
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
#[derive(Serialize, Deserialize)]
pub struct NatsTriggerConfig {
@@ -171,6 +181,8 @@ enum TriggerConfig {
Postgres(PostgresTriggerConfig),
#[cfg(feature = "websocket")]
Websocket(WebsocketTriggerConfig),
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
Sqs(SqsTriggerConfig),
#[cfg(all(feature = "enterprise", feature = "kafka"))]
Kafka(KafkaTriggerConfig),
#[cfg(all(feature = "enterprise", feature = "nats"))]

View File

@@ -59,6 +59,7 @@ pub fn workspaced_service() -> Router {
.route("/get_triggers_count/*path", get(get_triggers_count))
.route("/list_tokens/*path", get(list_tokens))
.route("/get/*path", get(get_flow_by_path))
.route("/deployment_status/p/*path", get(get_deployment_status))
.route("/get/draft/*path", get(get_flow_by_path_w_draft))
.route("/exists/*path", get(exists_flow_by_path))
.route("/list_paths", get(list_paths))
@@ -356,8 +357,8 @@ async fn create_flow(
sqlx::query!(
"INSERT INTO flow (workspace_id, path, summary, description, \
dependency_job, draft_only, tag, dedicated_worker, visible_to_runner_only, on_behalf_of_email, value, schema, edited_by, edited_at)
VALUES ($1, $2, $3, $4, NULL, $5, $6, $7, $8, $9, $10, $11::text::json, $12, now())",
dependency_job, lock_error_logs, draft_only, tag, dedicated_worker, visible_to_runner_only, on_behalf_of_email, value, schema, edited_by, edited_at)
VALUES ($1, $2, $3, $4, NULL, '', $5, $6, $7, $8, $9, $10, $11::text::json, $12, now())",
w_id,
nf.path,
nf.summary,
@@ -683,7 +684,7 @@ async fn update_flow(
sqlx::query!(
"UPDATE flow SET path = $1, summary = $2, description = $3,\
dependency_job = NULL, draft_only = NULL, tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, on_behalf_of_email = $7, \
dependency_job = NULL, lock_error_logs = '', draft_only = NULL, tag = $4, dedicated_worker = $5, visible_to_runner_only = $6, on_behalf_of_email = $7, \
value = $8, schema = $9::text::json, edited_by = $10, edited_at = now()
WHERE path = $11 AND workspace_id = $12",
if is_new_path { flow_path } else { &nf.path }, // if new path, do not rename directly (to avoid flow_version foreign key constraint)
@@ -950,6 +951,31 @@ async fn list_tokens(
list_tokens_internal(&db, &w_id, &path, true).await
}
#[derive(FromRow, Serialize)]
struct DeploymentStatus {
lock_error_logs: Option<String>,
}
async fn get_deployment_status(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<DeploymentStatus> {
let path = path.to_path();
let mut tx = db.begin().await?;
let status_o: Option<DeploymentStatus> = sqlx::query_as!(
DeploymentStatus,
"SELECT lock_error_logs FROM flow WHERE path = $1 AND workspace_id = $2",
path,
w_id,
)
.fetch_optional(&mut *tx)
.await?;
let status = not_found_if_none(status_o, "DeploymentStatus", path)?;
tx.commit().await?;
Ok(Json(status))
}
async fn get_flow_by_path(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -961,7 +987,7 @@ async fn get_flow_by_path(
let flow_o = if query.with_starred_info.unwrap_or(false) {
sqlx::query_as::<_, FlowWithStarred>(
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, favorite.path IS NOT NULL as starred
"SELECT flow.workspace_id, flow.path, flow.lock_error_logs, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, favorite.path IS NOT NULL as starred
FROM flow
LEFT JOIN favorite
ON favorite.favorite_kind = 'flow'
@@ -978,7 +1004,7 @@ async fn get_flow_by_path(
.await?
} else {
sqlx::query_as::<_, FlowWithStarred>(
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, NULL as starred
"SELECT flow.workspace_id, flow.path, flow.lock_error_logs, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, NULL as starred
FROM flow
LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 AND flow.workspace_id = $2"

View File

@@ -915,7 +915,16 @@ impl<'a> GetQuery<'a> {
.fetch_queued(db, job_id, workspace_id)
.await?
.map(Job::QueuedJob);
not_found_if_none(job_maybe, "Job", job_id.to_string())
// potential race condition here, if the job was in queue and completed right after the fetch completed, so we need to check one last time
if let Some(job) = job_maybe {
return Ok(job);
} else {
let cjob2 = self
.fetch_completed(db, job_id, workspace_id)
.await?
.map(Job::CompletedJob);
not_found_if_none(cjob2, "Job", job_id.to_string())
}
}
}
}

View File

@@ -39,4 +39,4 @@ pub struct KafkaTrigger {
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
pub enabled: bool,
}
}

View File

@@ -105,6 +105,8 @@ mod settings;
mod slack_approvals;
#[cfg(feature = "smtp")]
mod smtp_server_ee;
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
mod sqs_triggers_ee;
mod static_assets;
mod stripe_ee;
mod teams_ee;
@@ -319,6 +321,48 @@ pub async fn run_server(
}
};
let sqs_triggers_service = {
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
sqs_triggers_ee::workspaced_service()
}
#[cfg(not(all(feature = "enterprise", feature = "sqs_trigger")))]
{
Router::new()
}
};
let websocket_triggers_service = {
#[cfg(feature = "websocket")]
{
websocket_triggers::workspaced_service()
}
#[cfg(not(feature = "websocket"))]
Router::new()
};
let http_triggers_service = {
#[cfg(feature = "http_trigger")]
{
http_triggers::workspaced_service()
}
#[cfg(not(feature = "http_trigger"))]
Router::new()
};
let postgres_triggers_service = {
#[cfg(feature = "postgres_trigger")]
{
postgres_triggers::workspaced_service()
}
#[cfg(not(feature = "postgres_trigger"))]
Router::new()
};
if !*CLOUD_HOSTED {
#[cfg(feature = "websocket")]
{
@@ -337,11 +381,18 @@ pub async fn run_server(
let nats_killpill_rx = rx.resubscribe();
nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx);
}
#[cfg(feature = "postgres_trigger")]
{
let db_killpill_rx = rx.resubscribe();
postgres_triggers::start_database(db.clone(), db_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
let sqs_killpill_rx = rx.resubscribe();
sqs_triggers_ee::start_sqs(db.clone(), sqs_killpill_rx);
}
}
// build our application with a route
@@ -392,35 +443,12 @@ pub async fn run_server(
.nest("/variables", variables::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/oidc", oidc_ee::workspaced_service())
.nest("/http_triggers", {
#[cfg(feature = "http_trigger")]
{
http_triggers::workspaced_service()
}
#[cfg(not(feature = "http_trigger"))]
Router::new()
})
.nest("/websocket_triggers", {
#[cfg(feature = "websocket")]
{
websocket_triggers::workspaced_service()
}
#[cfg(not(feature = "websocket"))]
Router::new()
})
.nest("/http_triggers", http_triggers_service)
.nest("/websocket_triggers", websocket_triggers_service)
.nest("/kafka_triggers", kafka_triggers_service)
.nest("/nats_triggers", nats_triggers_service)
.nest("/postgres_triggers", {
#[cfg(feature = "postgres_trigger")]
{
postgres_triggers::workspaced_service()
}
#[cfg(not(feature = "postgres_trigger"))]
Router::new()
}),
.nest("/sqs_triggers", sqs_triggers_service)
.nest("/postgres_triggers", postgres_triggers_service),
)
.nest("/workspaces", workspaces::global_service())
.nest(

View File

@@ -40,4 +40,4 @@ pub struct NatsTrigger {
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
pub enabled: bool,
}
}

View File

@@ -36,11 +36,11 @@ use super::{
use lazy_static::lazy_static;
#[derive(FromRow, Serialize, Deserialize, Debug)]
pub struct Database {
pub struct Postgres {
pub user: String,
pub password: String,
pub host: String,
pub port: u16,
pub port: Option<u16>,
pub dbname: String,
#[serde(default)]
pub sslmode: String,
@@ -113,7 +113,6 @@ pub struct TestPostgres {
pub postgres_resource_path: String,
}
pub async fn test_postgres_connection(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -706,7 +705,7 @@ pub async fn get_publication_info(
let publication_data =
get_publication_scope_and_transaction(&mut connection, &publication_name).await;
let (all_table, transaction_to_track) = match publication_data {
let (all_table, transaction_to_track) = match publication_data {
Ok(pub_data) => pub_data,
Err(Error::SqlErr { error: sqlx::Error::RowNotFound, .. }) => {
return Err(Error::NotFound(
@@ -1457,4 +1456,4 @@ pub async fn is_database_in_logical_level(
};
Ok(Json(is_logical))
}
}

View File

@@ -1,7 +1,7 @@
use crate::{
db::{ApiAuthed, DB},
jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery},
resources::get_resource_value_interpolated_internal,
resources::try_get_resource_from_db_as,
users::fetch_api_authed,
};
use chrono::Utc;
@@ -20,18 +20,17 @@ use axum::{
routing::{delete, get, post},
Router,
};
pub use handler::PostgresTrigger;
use handler::{
alter_publication, create_postgres_trigger, create_publication, create_slot,
create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name,
exists_postgres_trigger, get_postgres_trigger, get_publication_info, get_template_script,
is_database_in_logical_level, list_database_publication, list_postgres_triggers,
list_slot_name, set_enabled, test_postgres_connection, update_postgres_trigger, Database,
list_slot_name, set_enabled, test_postgres_connection, update_postgres_trigger, Postgres,
Relations,
};
pub use handler::PostgresTrigger;
use windmill_common::{db::UserDB, error::Error, utils::StripPath};
use windmill_queue::PushArgsOwned;
mod bool;
mod converter;
mod handler;
@@ -55,13 +54,15 @@ pub async fn get_database_connection(
postgres_resource_path: &str,
w_id: &str,
) -> std::result::Result<PgConnection, windmill_common::error::Error> {
let database = get_database_resource(authed, user_db, db, postgres_resource_path, w_id).await?;
let database =
try_get_resource_from_db_as::<Postgres>(authed, user_db, db, postgres_resource_path, w_id)
.await?;
Ok(get_raw_postgres_connection(&database).await?)
}
pub async fn get_raw_postgres_connection(
db: &Database,
db: &Postgres,
) -> std::result::Result<PgConnection, Error> {
let options = {
let sslmode = if !db.sslmode.is_empty() {
@@ -69,12 +70,19 @@ pub async fn get_raw_postgres_connection(
} else {
PgSslMode::Prefer
};
let options = PgConnectOptions::new()
.host(&db.host)
.database(&db.dbname)
.port(db.port)
.ssl_mode(sslmode)
.username(&db.user);
let options = {
let inner_options = PgConnectOptions::new()
.host(&db.host)
.database(&db.dbname)
.ssl_mode(sslmode)
.username(&db.user);
if let Some(port) = db.port {
inner_options.port(port)
} else {
inner_options
}
};
let options = if !db.root_certificate_pem.is_empty() {
options.ssl_root_cert_from_pem(db.root_certificate_pem.as_bytes().to_vec())
@@ -202,39 +210,6 @@ pub fn generate_random_string() -> String {
format!("{}_{}", timestamp, random_part)
}
pub async fn get_database_resource(
authed: ApiAuthed,
user_db: Option<UserDB>,
db: &DB,
database_resource_path: &str,
w_id: &str,
) -> Result<Database, Error> {
let resource = get_resource_value_interpolated_internal(
&authed,
user_db,
&db,
&w_id,
&database_resource_path,
None,
"",
)
.await
.map_err(|_| Error::NotFound("Database resource do not exist".to_string()))?;
let resource = match resource {
Some(resource) => serde_json::from_value::<Database>(resource)?,
None => {
return {
Err(Error::NotFound(
"Database resource do not exist".to_string(),
))
}
}
};
Ok(resource)
}
fn publication_service() -> Router {
Router::new()
.route("/get/:publication_name/*path", get(get_publication_info))

View File

@@ -4,7 +4,6 @@ use crate::{
capture::{insert_capture_payload, PostgresTriggerConfig, TriggerKind},
db::{ApiAuthed, DB},
postgres_triggers::{
get_database_resource,
relation::RelationConverter,
replication_message::{
LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update},
@@ -12,7 +11,7 @@ use crate::{
},
run_job,
},
users::fetch_api_authed,
users::fetch_api_authed, resources::try_get_resource_from_db_as,
};
use bytes::{BufMut, Bytes, BytesMut};
use chrono::TimeZone;
@@ -33,7 +32,7 @@ use windmill_queue::PushArgsOwned;
use super::{
drop_logical_replication_slot_query, drop_publication_query, get_database_connection,
handler::{Database, PostgresTrigger},
handler::{Postgres, PostgresTrigger},
replication_message::PrimaryKeepAliveBody,
ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS,
};
@@ -80,7 +79,7 @@ enum Error {
pub struct PostgresSimpleClient(Client);
impl PostgresSimpleClient {
async fn new(database: &Database) -> Result<Self, Error> {
async fn new(database: &Postgres) -> Result<Self, Error> {
let ssl_mode = match database.sslmode.as_ref() {
"disable" => SslMode::Disable,
"" | "prefer" | "allow" => SslMode::Prefer,
@@ -98,11 +97,14 @@ impl PostgresSimpleClient {
config
.dbname(&database.dbname)
.host(&database.host)
.port(database.port)
.user(&database.user)
.ssl_mode(ssl_mode)
.replication_mode(rust_postgres::config::ReplicationMode::Logical);
if let Some(port) = database.port {
config.port(port);
};
if !database.password.is_empty() {
config.password(&database.password);
}
@@ -451,7 +453,7 @@ impl PostgresConfig {
PostgresConfig::Capture(capture) => capture.fetch_authed(db).await?,
};
let database = get_database_resource(
let database = try_get_resource_from_db_as::<Postgres>(
authed,
Some(UserDB::new(db.clone())),
&db,

View File

@@ -515,7 +515,9 @@ pub async fn transform_json_value<'c>(
Value::String(y) if y.starts_with("$res:") => {
let path = y.strip_prefix("$res:").unwrap();
if path.split("/").count() < 2 {
return Err(Error::internal_err(format!("Invalid resource path: {path}")));
return Err(Error::internal_err(format!(
"Invalid resource path: {path}"
)));
}
let mut tx: Transaction<'_, Postgres> =
authed_transaction_or_default(authed, user_db.clone(), db).await?;
@@ -1205,3 +1207,44 @@ async fn update_resource_type(
Ok(format!("resource_type {} updated", name))
}
#[cfg(any(
feature = "postgres_trigger",
all(feature = "sqs_trigger", feature = "enterprise")
))]
pub async fn try_get_resource_from_db_as<T>(
authed: ApiAuthed,
user_db: Option<UserDB>,
db: &DB,
resource_path: &str,
w_id: &str,
) -> Result<T>
where
T: serde::de::DeserializeOwned,
{
let resource = get_resource_value_interpolated_internal(
&authed,
user_db,
&db,
&w_id,
&resource_path,
None,
"",
)
.await?;
let resource = match resource {
Some(resource) => serde_json::from_value::<T>(resource)
.map_err(|e| Error::SerdeJson { error: e, location: "resources.rs".to_string() })?,
None => {
return {
Err(Error::NotFound(format!(
"resource at path :{} do not exist",
&resource_path
)))
}
}
};
Ok(resource)
}

View File

@@ -154,7 +154,7 @@ async fn create_schedule(
let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?;
// Check schedule for error
ScheduleType::from_str(&ns.schedule, ns.cron_version.as_deref())?;
ScheduleType::from_str(&ns.schedule, ns.cron_version.as_deref(), true)?;
check_path_conflict(&mut tx, &w_id, &ns.path).await?;
check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
@@ -249,7 +249,7 @@ async fn edit_schedule(
let mut tx = user_db.begin(&authed).await?;
// Check schedule for error
ScheduleType::from_str(&es.schedule, es.cron_version.as_deref())?;
ScheduleType::from_str(&es.schedule, es.cron_version.as_deref(), true)?;
clear_schedule(&mut tx, path, &w_id).await?;
let schedule = sqlx::query_as::<_, Schedule>(
@@ -468,7 +468,8 @@ pub struct PreviewPayload {
pub async fn preview_schedule(
Json(payload): Json<PreviewPayload>,
) -> JsonResult<Vec<DateTime<Utc>>> {
let schedule = ScheduleType::from_str(&payload.schedule, payload.cron_version.as_deref())?;
let schedule =
ScheduleType::from_str(&payload.schedule, payload.cron_version.as_deref(), true)?;
let tz =
chrono_tz::Tz::from_str(&payload.timezone).map_err(|e| Error::BadRequest(e.to_string()))?;

View File

@@ -0,0 +1,31 @@
use crate::db::DB;
use axum::Router;
use serde::{Deserialize, Serialize};
pub fn workspaced_service() -> Router {
Router::new()
}
pub fn start_sqs(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () {
// implementation is not open source
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct SqsTrigger {
pub queue_url: String,
pub aws_resource_path: String,
pub message_attributes: Option<Vec<String>>,
pub path: String,
pub script_path: String,
pub is_flow: bool,
pub workspace_id: String,
pub edited_by: String,
pub email: String,
pub edited_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: Option<serde_json::Value>,
pub error: Option<String>,
pub server_id: Option<String>,
pub last_server_ping: Option<chrono::DateTime<chrono::Utc>>,
pub enabled: bool,
}

View File

@@ -2044,7 +2044,7 @@ pub struct LoginUserInfo {
pub email: Option<String>,
pub name: Option<String>,
pub company: Option<String>,
pub preferred_username: Option<String>,
pub displayName: Option<String>,
}

View File

@@ -31,9 +31,9 @@ use windmill_common::{
};
use lazy_static::lazy_static;
use windmill_common::variables::{decrypt, encrypt};
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
use windmill_common::variables::{decrypt, encrypt};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
lazy_static! {

View File

@@ -732,7 +732,8 @@ async fn edit_copilot_config(
.await?;
if let Some(cached) = AI_KEY_CACHE.get(&w_id) {
if cached.path != parsed_ai_resource.path {
if parsed_ai_resource.path.is_none() || parsed_ai_resource.path.unwrap() != cached.path
{
AI_KEY_CACHE.remove(&w_id);
}
}
@@ -1362,6 +1363,7 @@ struct UsedTriggers {
pub kafka_used: bool,
pub nats_used: bool,
pub postgres_used: bool,
pub sqs_used: bool,
}
async fn get_used_triggers(
@@ -1380,7 +1382,8 @@ async fn get_used_triggers(
EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS "http_routes_used!",
EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as "kafka_used!",
EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!",
EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!"
EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!",
EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!"
"#,
w_id
)

View File

@@ -594,6 +594,28 @@ pub(crate) async fn tarball_workspace(
}
}
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
let sqs_triggers = sqlx::query_as!(
crate::sqs_triggers_ee::SqsTrigger,
"SELECT * FROM sqs_trigger
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
.await?;
for trigger in sqs_triggers {
let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap();
archive
.write_to_archive(
&trigger_str,
&format!("{}.sqs_trigger.json", trigger.path),
)
.await?;
}
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
{
let nats_triggers = sqlx::query_as!(

View File

@@ -29,6 +29,15 @@ pub async fn get_license_plan() -> LicensePlan {
pub enum CriticalErrorChannel {
Email { email: String },
Slack { slack_channel: String },
Teams { teams_channel: TeamsChannel },
}
#[derive(Deserialize)]
pub struct TeamsChannel {
pub team_id: String,
pub team_name: String,
pub channel_id: String,
pub channel_name: String,
}
pub enum CriticalAlertKind {

View File

@@ -60,6 +60,8 @@ pub struct FlowWithStarred {
pub flow: Flow,
#[serde(skip_serializing_if = "Option::is_none")]
pub starred: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub lock_error_logs: Option<String>,
}
fn is_none_or_false(b: &Option<bool>) -> bool {

View File

@@ -155,8 +155,6 @@ pub async fn shutdown_signal(
}
use tokio::sync::RwLock;
#[cfg(feature = "prometheus")]
use tokio::task::JoinHandle;
use utils::rd_string;
#[cfg(feature = "prometheus")]
@@ -164,23 +162,31 @@ pub async fn serve_metrics(
addr: SocketAddr,
mut rx: tokio::sync::broadcast::Receiver<()>,
ready_worker_endpoint: bool,
) -> JoinHandle<()> {
use std::sync::atomic::Ordering;
metrics_endpoint: bool,
) -> anyhow::Result<()> {
if !metrics_endpoint && !ready_worker_endpoint {
return Ok(());
}
use axum::{
routing::{get, post},
Router,
};
use hyper::StatusCode;
let router = Router::new()
.route("/metrics", get(metrics))
.route("/reset", post(reset));
let router = Router::new();
let router = if metrics_endpoint {
router
.route("/metrics", get(metrics))
.route("/reset", post(reset))
} else {
router
};
let router = if ready_worker_endpoint {
router.route(
"/ready",
get(|| async {
if IS_READY.load(Ordering::Relaxed) {
if IS_READY.load(std::sync::atomic::Ordering::Relaxed) {
(StatusCode::OK, "ready")
} else {
(StatusCode::INTERNAL_SERVER_ERROR, "not ready")
@@ -193,8 +199,12 @@ pub async fn serve_metrics(
tokio::spawn(async move {
tracing::info!("Serving metrics at: {addr}");
let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
if let Err(e) = axum::serve(listener, router.into_make_service())
let listener = tokio::net::TcpListener::bind(addr).await;
if let Err(e) = listener {
tracing::error!("Error binding to metrics address: {}", e);
return;
}
if let Err(e) = axum::serve(listener.unwrap(), router.into_make_service())
.with_graceful_shutdown(async move {
rx.recv().await.ok();
tracing::info!("Graceful shutdown of metrics");
@@ -204,6 +214,8 @@ pub async fn serve_metrics(
tracing::error!("Error serving metrics: {}", e);
}
})
.await?;
Ok(())
}
#[cfg(feature = "prometheus")]

View File

@@ -424,7 +424,11 @@ impl ScheduleType {
}
}
pub fn from_str(schedule_str: &str, version: Option<&str>) -> Result<ScheduleType> {
pub fn from_str(
schedule_str: &str,
version: Option<&str>,
seconds_required: bool,
) -> Result<ScheduleType> {
tracing::debug!(
"Attempting to parse schedule string: {}, with version: {:?}",
schedule_str,
@@ -448,7 +452,13 @@ impl ScheduleType {
Some("v2") | Some(_) => {
// Use Croner for v2
let schedule_type_result = panic::catch_unwind(AssertUnwindSafe(|| {
Cron::new(schedule_str).with_seconds_optional().parse()
let mut croner = Cron::new(schedule_str);
if seconds_required {
croner.with_seconds_required();
} else {
croner.with_seconds_optional();
};
croner.parse()
}))
.map_err(|_| {
tracing::error!(

View File

@@ -352,3 +352,4 @@ pub async fn get_reserved_variables(
is_custom: true,
})).collect()
}

View File

@@ -362,9 +362,6 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
#[annotations("#")]
pub struct PythonAnnotations {
pub no_cache: bool,
pub no_uv: bool,
pub no_uv_install: bool,
pub no_uv_compile: bool,
pub no_postinstall: bool,
pub py310: bool,
pub py311: bool,

View File

@@ -38,7 +38,8 @@ pub async fn push_scheduled_job<'c>(
));
}
let sched = ScheduleType::from_str(&schedule.schedule, schedule.cron_version.as_deref())?;
let sched =
ScheduleType::from_str(&schedule.schedule, schedule.cron_version.as_deref(), false)?;
let tz = chrono_tz::Tz::from_str(&schedule.timezone)
.map_err(|e| error::Error::BadRequest(e.to_string()))?;

View File

@@ -20,7 +20,7 @@ flow_testing = []
cloud = []
sqlx = []
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions"]
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error"]
otel = ["windmill-common/otel", "dep:opentelemetry"]
dind = ["dep:bollard"]
php = ["dep:windmill-parser-php"]
@@ -70,6 +70,7 @@ dyn-iter.workspace = true
once_cell.workspace = true
tokio-postgres.workspace = true
bit-vec.workspace = true
deno_telemetry = { workspace = true, optional = true }
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
deno_web = { workspace = true, optional = true }
@@ -80,6 +81,8 @@ deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
deno_io = { workspace = true, optional = true }
deno_error = { workspace = true, optional = true }
postgres-native-tls.workspace = true
native-tls.workspace = true
@@ -118,3 +121,6 @@ deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
deno_io = { workspace = true, optional = true }
deno_runtime = { workspace = true, optional = true }
deno_telemetry = { workspace = true, optional = true }

View File

@@ -32,9 +32,10 @@ impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_read<'a>(
&mut self,
_resolved: bool,
_p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
unreachable!("snapshotting")
}
}
@@ -95,7 +96,7 @@ fn main() {
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
// deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_webidl::deno_webidl::init_ops_and_esm(),
deno_url::deno_url::init_ops_and_esm(),
deno_console::deno_console::init_ops_and_esm(),
@@ -117,7 +118,9 @@ fn main() {
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
startup_snapshot: None,
extension_transpiler: None,
extension_transpiler: Some(std::rc::Rc::new(|specifier, source| {
deno_runtime::transpile::maybe_transpile_source(specifier, source)
})),
extensions: exts,
with_runtime_cb: None,
skip_op_registration: false,

View File

@@ -1,86 +0,0 @@
name: "python download pip"
mode: ONCE
hostname: "python"
log_level: ERROR
time_limit: 900
rlimit_as: 2048
rlimit_cpu: 1000
rlimit_fsize: 1024
rlimit_nofile: 64
envar: "HOME=/user"
envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
keep_caps: true
keep_env: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
}
mount {
src: "{WORKER_DIR}/download_deps.py.pip.sh"
dst: "/download_deps.sh"
is_bind: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
exec_bin {
path: "/bin/sh"
arg: "/download_deps.sh"
}

View File

@@ -1,24 +0,0 @@
#/bin/sh
INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" )
EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" )
TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST")
if [ ! -z "$INDEX_URL" ]
then
echo "\$INDEX_URL is set to $INDEX_URL"
fi
if [ ! -z "$EXTRA_INDEX_URL" ]
then
echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL"
fi
if [ ! -z "$TRUSTED_HOST" ]
then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
echo $CMD
eval $CMD

View File

@@ -16,6 +16,7 @@ clone_newuser: {CLONE_NEWUSER}
keep_caps: false
keep_env: true
mount_proc: true
mount {
src: "/bin"
@@ -37,6 +38,12 @@ mount {
mandatory: false
}
mount {
src: "/root/.local/share/uv/tools/ansible"
dst: "/root/.local/share/uv/tools/ansible"
is_bind: true
}
mount {
src: "/usr"
dst: "/usr"
@@ -126,6 +133,13 @@ mount {
is_bind: true
}
mount {
src: "{PY_INSTALL_DIR}"
dst: "{PY_INSTALL_DIR}"
is_bind: true
}
{SHARED_MOUNT}
{SHARED_DEPENDENCIES}

View File

@@ -2,11 +2,7 @@
use std::{collections::HashMap, os::unix::fs::PermissionsExt, path::PathBuf, process::Stdio};
#[cfg(windows)]
use std::{
collections::HashMap,
path::{Path, PathBuf},
process::Stdio,
};
use std::{collections::HashMap, path::PathBuf, process::Stdio};
use anyhow::anyhow;
use itertools::Itertools;
@@ -30,7 +26,7 @@ use crate::{
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion},
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, TZ_ENV,
PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
};
lazy_static::lazy_static! {
@@ -86,7 +82,6 @@ async fn handle_ansible_python_deps(
&mut Some(occupancy_metrics),
PyVersion::Py311,
false,
false,
)
.await
.map_err(|e| {
@@ -113,7 +108,6 @@ async fn handle_ansible_python_deps(
worker_dir,
&mut Some(occupancy_metrics),
crate::python_executor::PyVersion::Py311,
false,
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -339,6 +333,7 @@ mount {{
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT
.replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR)
.replace("{JOB_DIR}", job_dir)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)

View File

@@ -96,6 +96,8 @@ cleanup() {{
# Ignore SIGTERM and SIGINT
trap '' SIGTERM SIGINT
rm -f bp 2>/dev/null
# Kill the process group of the script (negative PID value)
pkill -P $$ 2>/dev/null || true
exit
@@ -105,17 +107,26 @@ cleanup() {{
# Trap SIGTERM (or other signals) and call cleanup function
trap cleanup SIGTERM SIGINT
# Create a named pipe
mkfifo bp
# Start background processes
cat bp | tail -1 >> ./result2.out &
tail_pid=$!
# Run main.sh in the same process group
{bash} ./main.sh "$@" 2>&1 | tee >(tail -1 >> ./result2.out) &
{bash} ./main.sh "$@" 2>&1 | tee bp &
pid=$!
# Wait for main.sh to finish and capture its exit status
wait $pid
exit_status=$?
# Ensure tail has finished before cleanup
wait $tail_pid 2>/dev/null || true
# Clean up the named pipe and background processes
rm -f bp
pkill -P $$ || true
# Exit with the captured status
@@ -150,7 +161,13 @@ exit $exit_status
let _ = write_file(job_dir, "result.out", "")?;
let _ = write_file(job_dir, "result2.out", "")?;
let child = if !*DISABLE_NSJAIL {
let nsjail = !*DISABLE_NSJAIL
&& job
.script_path
.as_ref()
.map(|x| !x.starts_with("init_script_"))
.unwrap_or(true);
let child = if nsjail {
let _ = write_file(
job_dir,
"run.config.proto",
@@ -202,7 +219,7 @@ exit $exit_status
mem_peak,
canceled_by,
child,
!*DISABLE_NSJAIL,
nsjail,
worker_name,
&job.workspace_id,
"bash run",
@@ -618,7 +635,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
write_file(
job_dir,
"wrapper.sh",
&format!("set -o pipefail\nset -e\n{} -F ./main.ps1 \"$@\" 2>&1 | tee >(tail -1 >> ./result2.out) &\nwait $!", POWERSHELL_PATH.as_str()),
&format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()),
)?;
#[cfg(windows)]

View File

@@ -936,18 +936,18 @@ pub async fn save_in_cache(
fn tentatively_improve_error(err: Error, executable: &str) -> Error {
#[cfg(unix)]
let err_msg = "No such file or directory (os error 2)";
let err_msgs = vec!["os error 2", "os error 3", "No such file or directory"];
#[cfg(windows)]
let err_msg = "program not found";
let err_msgs = vec!["program not found", "os error 2", "os error 3"];
if err.to_string().contains(&err_msg) {
if err_msgs.iter().any(|msg| err.to_string().contains(msg)) {
return Error::internal_err(format!(
"Executable {executable} not found on worker. PATH: {}",
*PATH_ENV
));
}
return err;
return Error::ExecutionErr(format!("Error executing {executable}: {err:#}"));
}
pub async fn clean_cache() -> error::Result<()> {

View File

@@ -22,11 +22,10 @@ pub async fn build_tar_and_push(
folder: String,
// python_311
python_xyz: String,
no_uv: bool,
) -> error::Result<()> {
use object_store::path::Path;
use crate::{TAR_PIP_CACHE_DIR, TAR_PYBASE_CACHE_DIR};
use crate::TAR_PYBASE_CACHE_DIR;
tracing::info!("Started building and pushing piptar {folder}");
let start = Instant::now();
@@ -34,11 +33,7 @@ pub async fn build_tar_and_push(
// e.g. tiny==1.0.0
let folder_name = folder.split("/").last().unwrap();
let prefix = if no_uv {
TAR_PIP_CACHE_DIR
} else {
&format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz)
};
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz);
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
let tar_file = std::fs::File::create(&tar_path)?;
@@ -59,10 +54,7 @@ pub async fn build_tar_and_push(
// })?;
if let Err(e) = s3_client
.put(
&Path::from(format!(
"/tar/{TARGET}/{}/{folder_name}.tar",
if no_uv { "pip" } else { &python_xyz }
)),
&Path::from(format!("/tar/{TARGET}/{python_xyz}/{folder_name}.tar")),
std::fs::read(&tar_path)?.into(),
)
.await
@@ -92,7 +84,6 @@ pub async fn pull_from_tar(
folder: String,
// python_311
python_xyz: String,
no_uv: bool,
) -> error::Result<()> {
use windmill_common::s3_helpers::attempt_fetch_bytes;
@@ -102,10 +93,7 @@ pub async fn pull_from_tar(
let start = Instant::now();
let tar_path = format!(
"tar/{TARGET}/{}/{folder_name}.tar",
if no_uv { "pip".to_owned() } else { python_xyz }
);
let tar_path = format!("tar/{TARGET}/{python_xyz}/{folder_name}.tar");
let bytes = attempt_fetch_bytes(client, &tar_path).await?;
extract_tar(bytes, &folder).await.map_err(|e| {

View File

@@ -108,9 +108,10 @@ impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_read<'a>(
&mut self,
_resolved: bool,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
) -> Result<Cow<'a, std::path::Path>, deno_io::fs::FsError> {
Ok(Cow::Borrowed(p))
}
}
@@ -554,12 +555,17 @@ function get_from_env(name) {{
async fn op_variable(
op_state: Rc<RefCell<OpState>>,
#[string] path: String,
) -> Result<String, anyhow::Error> {
) -> Result<String, deno_error::JsErrorBox> {
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
if let Some(client) = client {
Ok(client.get_variable_value(&path).await?)
Ok(client
.get_variable_value(&path)
.await
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?)
} else {
anyhow::bail!("No client found in op state");
Err(deno_error::JsErrorBox::generic(
"No client found in op state",
))
}
}
@@ -569,16 +575,18 @@ async fn op_variable(
async fn op_get_result(
op_state: Rc<RefCell<OpState>>,
#[string] id: String,
) -> Result<String, anyhow::Error> {
) -> Result<String, deno_error::JsErrorBox> {
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
if let Some(client) = client {
let result = client
client
.get_completed_job_result::<Box<RawValue>>(&id, None)
.await?
.clone();
Ok(result.get().to_string())
.await
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))
.map(|x| x.get().to_string())
} else {
anyhow::bail!("No client found in op state");
Err(deno_error::JsErrorBox::generic(
"No client found in op state",
))
}
}
@@ -589,7 +597,7 @@ async fn op_get_id(
op_state: Rc<RefCell<OpState>>,
#[string] flow_job_id: String,
#[string] node_id: String,
) -> Result<Option<String>, anyhow::Error> {
) -> Result<Option<String>, deno_error::JsErrorBox> {
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
if let Some(client) = client {
let result = client
@@ -602,7 +610,9 @@ async fn op_get_id(
Ok(None)
}
} else {
anyhow::bail!("No client found in op state");
Err(deno_error::JsErrorBox::generic(
"No client found in op state",
))
}
}
@@ -612,15 +622,18 @@ async fn op_get_id(
async fn op_resource(
op_state: Rc<RefCell<OpState>>,
#[string] path: String,
) -> Result<Option<String>, anyhow::Error> {
) -> Result<Option<String>, deno_error::JsErrorBox> {
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
if let Some(client) = client {
client
.get_resource_value_interpolated::<Option<Box<RawValue>>>(&path, None)
.await
.map(|x| x.map(|x| x.get().to_string()))
.map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))
} else {
anyhow::bail!("No client found in op state");
Err(deno_error::JsErrorBox::generic(
"No client found in op state",
))
}
}
@@ -822,6 +835,7 @@ pub async fn eval_fetch_timeout(
};
let exts: Vec<Extension> = vec![
deno_telemetry::deno_telemetry::init_ops(),
deno_webidl::deno_webidl::init_ops(),
deno_url::deno_url::init_ops(),
deno_console::deno_console::init_ops(),
@@ -953,6 +967,9 @@ fn write_error_expr(expr: &str, uuid: &Uuid) {
}
};
if std::env::var("PRINT_NATIVE_ERRORS").is_ok() {
tracing::info!("native error for job {uuid}: {expr}");
}
if dir_entries >= 100 {
tracing::info!("Too many error files in {ERROR_DIR}, skipping write");
return;

View File

@@ -39,8 +39,10 @@ use std::env::var;
use windmill_queue::{append_logs, CanceledBy};
lazy_static::lazy_static! {
static ref PYTHON_PATH: String =
var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
tracing::warn!("PYTHON_PATH is set to {} and thus python will not be managed by uv and stay static regardless of annotation and instance settings. NOT RECOMMENDED", v);
v
});
static ref UV_PATH: String =
var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string());
@@ -48,31 +50,18 @@ lazy_static::lazy_static! {
static ref PY_CONCURRENT_DOWNLOADS: usize =
var("PY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20);
static ref FLOCK_PATH: String =
var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string());
static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap();
static ref TRUSTED_HOST: Option<String> = var("PY_TRUSTED_HOST").ok().or(var("PIP_TRUSTED_HOST").ok());
static ref INDEX_CERT: Option<String> = var("PY_INDEX_CERT").ok().or(var("PIP_INDEX_CERT").ok());
static ref NATIVE_CERT: bool = var("PY_NATIVE_CERT").ok().or(var("UV_NATIVE_TLS").ok()).map(|flag| flag == "true").unwrap_or(false);
pub static ref USE_SYSTEM_PYTHON: bool = var("USE_SYSTEM_PYTHON")
.ok().map(|flag| flag == "true").unwrap_or(false);
pub static ref USE_PIP_COMPILE: bool = var("USE_PIP_COMPILE")
.ok().map(|flag| flag == "true").unwrap_or(false);
pub static ref USE_PIP_INSTALL: bool = var("USE_PIP_INSTALL")
.ok().map(|flag| flag == "true").unwrap_or(false);
static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
static ref EPHEMERAL_TOKEN_CMD: Option<String> = var("EPHEMERAL_TOKEN_CMD").ok();
}
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str =
include_str!("../nsjail/download.py.pip.config.proto");
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
@@ -89,8 +78,8 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION,
LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL,
PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
UV_CACHE_DIR,
};
// To change latest stable version:
@@ -311,6 +300,7 @@ impl PyVersion {
.env_clear()
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.envs(PROXY_ENVS.clone())
.args(["python", "install", v, "--python-preference=only-managed"])
// TODO: Do we need these?
.envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)])
@@ -376,6 +366,7 @@ impl PyVersion {
"python",
"find",
self.to_string_with_dot(),
"--system",
"--python-preference=only-managed",
])
.envs([
@@ -451,8 +442,6 @@ pub async fn uv_pip_compile(
py_version: PyVersion,
// Debug-only flag
no_cache: bool,
// Fallback to pip-compile. Will be removed in future
mut no_uv: bool,
) -> error::Result<String> {
let mut logs = String::new();
logs.push_str(&format!("\nresolving dependencies..."));
@@ -495,19 +484,8 @@ pub async fn uv_pip_compile(
#[cfg(feature = "enterprise")]
let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?;
let mut req_hash = format!("py-{}", calculate_hash(&requirements));
let req_hash = format!("py-{}", calculate_hash(&requirements));
if no_uv || *USE_PIP_COMPILE {
logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)"));
// Set no_uv if not setted
no_uv = true;
// Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable.
// Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db).
// It will recalculate lockfile even if inputs have not been changed.
req_hash.push_str("-no_uv");
// Will be in format:
// py-000..000-no_uv
}
if !no_cache {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
@@ -530,76 +508,7 @@ pub async fn uv_pip_compile(
write_file(job_dir, file, &requirements)?;
// Fallback pip-compile. Will be removed in future
if no_uv {
tracing::debug!("Fallback to pip-compile");
let mut args = vec![
"-q",
"--no-header",
file,
"--resolver=backtracking",
"--strip-extras",
];
let mut pip_args = vec![];
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
.read()
.await
.clone()
.map(handle_ephemeral_token);
if let Some(url) = pip_extra_index_url.as_ref() {
url.split(",").for_each(|url| {
args.extend(["--extra-index-url", url]);
pip_args.push(format!("--extra-index-url {}", url));
});
args.push("--no-emit-index-url");
}
let pip_index_url = PIP_INDEX_URL
.read()
.await
.clone()
.map(handle_ephemeral_token);
if let Some(url) = pip_index_url.as_ref() {
args.extend(["--index-url", url, "--no-emit-index-url"]);
pip_args.push(format!("--index-url {}", url));
}
if let Some(host) = TRUSTED_HOST.as_ref() {
args.extend(["--trusted-host", host]);
}
if let Some(cert_path) = INDEX_CERT.as_ref() {
args.extend(["--cert", cert_path]);
}
let pip_args_str = pip_args.join(" ");
if pip_args.len() > 0 {
args.extend(["--pip-args", &pip_args_str]);
}
tracing::debug!("pip-compile args: {:?}", args);
let mut child_cmd = Command::new("pip-compile");
child_cmd
.current_dir(job_dir)
.args(args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child_process = start_child_process(child_cmd, "pip-compile").await?;
append_logs(&job_id, &w_id, logs, db).await;
handle_child(
job_id,
db,
mem_peak,
canceled_by,
child_process,
false,
worker_name,
&w_id,
"pip-compile",
None,
false,
occupancy_metrics,
)
.await
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
} else {
{
// Make sure we have python runtime installed
py_version
.get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics)
@@ -868,6 +777,30 @@ fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Resul
Ok(())
}
async fn get_python_path(
py_version: PyVersion,
worker_name: &str,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
db: &sqlx::Pool<sqlx::Postgres>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> windmill_common::error::Result<String> {
let python_path = if let Some(python_path) = PYTHON_PATH.clone() {
python_path
} else if let Some(python_path) = py_version
.get_python(&job_id, mem_peak, db, worker_name, w_id, occupancy_metrics)
.await?
{
python_path
} else {
return Err(Error::ExecutionErr(format!(
"uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path"
)));
};
Ok(python_path)
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_python_job(
requirements_o: Option<&String>,
@@ -904,25 +837,18 @@ pub async fn handle_python_job(
)
.await?;
let PythonAnnotations { no_uv, no_postinstall, .. } = PythonAnnotations::parse(inner_content);
let PythonAnnotations { no_postinstall, .. } = PythonAnnotations::parse(inner_content);
tracing::debug!("Finished handling python dependencies");
let python_path = if no_uv {
PYTHON_PATH.clone()
} else if let Some(python_path) = py_version
.get_python(
&job.id,
mem_peak,
db,
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
)
.await?
{
python_path
} else {
PYTHON_PATH.clone()
};
let python_path = get_python_path(
py_version,
worker_name,
&job.id,
&job.workspace_id,
mem_peak,
db,
&mut Some(occupancy_metrics),
)
.await?;
if !no_postinstall {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await {
@@ -931,15 +857,7 @@ pub async fn handle_python_job(
tracing::debug!("Finished deps postinstall stage");
}
if no_uv {
append_logs(
&job.id,
&job.workspace_id,
format!("\n\n--- SYSTEM PYTHON (Fallback) CODE EXECUTION ---\n",),
db,
)
.await;
} else {
{
append_logs(
&job.id,
&job.workspace_id,
@@ -1525,7 +1443,6 @@ async fn handle_python_deps(
occupancy_metrics,
annotated_pyv.unwrap_or(instance_pyv),
annotations.no_cache,
annotations.no_uv || annotations.no_uv_compile,
)
.await
.map_err(|e| {
@@ -1587,7 +1504,6 @@ async fn handle_python_deps(
worker_dir,
occupancy_metrics,
final_version,
annotations.no_uv || annotations.no_uv_install,
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -1611,7 +1527,6 @@ async fn spawn_uv_install(
(pip_extra_index_url, pip_index_url): (Option<String>, Option<String>),
// If none, it is system python
py_path: Option<String>,
no_uv_install: bool,
worker_dir: &str,
) -> Result<tokio::process::Child, Error> {
if !*DISABLE_NSJAIL {
@@ -1653,15 +1568,12 @@ async fn spawn_uv_install(
let _ = write_file(
job_dir,
&nsjail_proto,
&(if no_uv_install {
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK
} else {
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
})
.replace("{WORKER_DIR}", worker_dir)
.replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR)
.replace("{TARGET_DIR}", &venv_p)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
.replace("{WORKER_DIR}", worker_dir)
.replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR)
.replace("{TARGET_DIR}", &venv_p)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.as_str(),
)?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
@@ -1675,72 +1587,47 @@ async fn spawn_uv_install(
.stderr(Stdio::piped());
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await
} else {
let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string();
#[cfg(unix)]
let req = if no_uv_install {
format!("'{}'", req)
} else {
req.to_owned()
};
let req = req.to_owned();
#[cfg(windows)]
let req = format!("{}", req);
let mut command_args = if no_uv_install {
vec![
PYTHON_PATH.as_str(),
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p,
]
} else {
vec![
UV_PATH.as_str(),
"pip",
"install",
&req,
"--no-deps",
"--no-color",
// Prevent uv from discovering configuration files.
"--no-config",
"--link-mode=copy",
"--system",
// Prefer main index over extra
// https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
// TODO: Use env variable that can be toggled from UI
"--index-strategy",
"unsafe-best-match",
"--target",
venv_p,
"--no-cache",
// If we invoke uv pip install, then we want to overwrite existing data
"--reinstall",
]
};
let mut command_args = vec![
UV_PATH.as_str(),
"pip",
"install",
&req,
"--no-deps",
"--no-color",
// Prevent uv from discovering configuration files.
"--no-config",
"--link-mode=copy",
"--system",
// Prefer main index over extra
// https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
// TODO: Use env variable that can be toggled from UI
"--index-strategy",
"unsafe-best-match",
"--target",
venv_p,
"--no-cache",
// If we invoke uv pip install, then we want to overwrite existing data
"--reinstall",
];
if !no_uv_install {
if let Some(py_path) = py_path.as_ref() {
command_args.extend([
"-p",
py_path.as_str(),
"--python-preference",
"only-managed", //
]);
} else {
command_args.extend([
"--python-preference",
"only-system", //
]);
}
if let Some(py_path) = py_path.as_ref() {
command_args.extend([
"-p",
py_path.as_str(),
"--python-preference",
"only-managed", //
]);
} else {
command_args.extend([
"--python-preference",
"only-system", //
]);
}
if let Some(url) = pip_extra_index_url.as_ref() {
@@ -1774,42 +1661,19 @@ async fn spawn_uv_install(
#[cfg(unix)]
{
if no_uv_install {
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(flock_cmd, FLOCK_PATH.as_str()).await
} else {
let mut cmd = Command::new(command_args[0]);
cmd.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args(&command_args[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(cmd, UV_PATH.as_str()).await
}
let mut cmd = Command::new(command_args[0]);
cmd.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args(&command_args[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(cmd, UV_PATH.as_str()).await
}
#[cfg(windows)]
{
let installer_path = if no_uv_install { command_args[0] } else { "uv" };
let mut cmd: Command = Command::new(&installer_path);
let mut cmd: Command = Command::new("uv");
cmd.env_clear()
.envs(envs)
.envs(PROXY_ENVS.clone())
@@ -1822,7 +1686,7 @@ async fn spawn_uv_install(
.args(&command_args[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(cmd, installer_path).await
start_child_process(cmd, "uv").await
}
}
}
@@ -1853,8 +1717,6 @@ pub async fn handle_python_reqs(
worker_dir: &str,
_occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
py_version: PyVersion,
// TODO: Remove (Deprecated)
mut no_uv_install: bool,
) -> error::Result<Vec<String>> {
let worker_dir = worker_dir.to_string();
@@ -1906,19 +1768,10 @@ pub async fn handle_python_reqs(
.await;
// Drop lock, so next print success can fire
}
no_uv_install |= *USE_PIP_INSTALL;
if no_uv_install {
append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await;
tracing::warn!("Fallback to pip");
}
// Parallelism level (N)
let parallel_limit = if no_uv_install {
1
} else {
// Semaphore will panic if value less then 1
PY_CONCURRENT_DOWNLOADS.clamp(1, 30)
};
let parallel_limit = // Semaphore will panic if value less then 1
PY_CONCURRENT_DOWNLOADS.clamp(1, 30);
tracing::info!(
workspace_id = %w_id,
@@ -1953,11 +1806,7 @@ pub async fn handle_python_reqs(
if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() {
continue;
}
let py_prefix = if no_uv_install {
PIP_CACHE_DIR
} else {
&py_version.to_cache_dir()
};
let py_prefix = &py_version.to_cache_dir();
let venv_p = format!(
"{py_prefix}/{}",
@@ -2097,13 +1946,7 @@ pub async fn handle_python_reqs(
let mut req_tl = 0;
if total_to_install > 0 {
let mut logs = String::new();
// Do we use UV?
if no_uv_install {
logs.push_str("\n\n--- PIP INSTALL ---\n");
} else {
logs.push_str("\n\n--- UV PIP INSTALL ---\n");
}
logs.push_str("\n\n--- UV PIP INSTALL ---\n");
logs.push_str("\nTo be installed: \n\n");
for (req, _) in &req_with_penv {
if req.len() > req_tl {
@@ -2135,13 +1978,9 @@ pub async fn handle_python_reqs(
let is_not_pro = !matches!(get_license_plan().await, LicensePlan::Pro);
let total_time = std::time::Instant::now();
let py_path = if no_uv_install {
None
} else {
py_version
.get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics)
.await?
};
let py_path = py_version
.get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics)
.await?;
let has_work = req_with_penv.len() > 0;
for ((i, (req, venv_p)), mut kill_rx) in
@@ -2196,7 +2035,7 @@ pub async fn handle_python_reqs(
tokio::select! {
// Cancel was called on the job
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install) => {
pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level()) => {
if let Err(e) = pull {
tracing::info!(
workspace_id = %w_id,
@@ -2242,7 +2081,6 @@ pub async fn handle_python_reqs(
&job_dir,
pip_indexes,
py_path,
no_uv_install,
&worker_dir
).await {
Ok(r) => r,
@@ -2348,7 +2186,7 @@ pub async fn handle_python_reqs(
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
if s3_push {
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install));
tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level()));
}
}
@@ -2580,8 +2418,20 @@ for line in sys.stdin:
base_internal_url.to_string(),
);
proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string());
let py_version = PyVersion::from_instance_version().await;
let python_path = get_python_path(
py_version,
worker_name,
&Uuid::nil(),
w_id,
&mut mem_peak,
db,
&mut None,
)
.await?;
handle_dedicated_process(
&*PYTHON_PATH,
&python_path,
job_dir,
context_envs,
envs,

View File

@@ -324,7 +324,7 @@ pub async fn process_result(
}
}
err @ _ => to_raw_value(&SerializedError {
message: format!("error during execution of the script:\n{err:#}",),
message: format!("execution error:\n{err:#}",),
name: "ExecutionErr".to_string(),
step_id: job.flow_step_id.clone(),
exit_code: None,

Some files were not shown because too many files have changed in this diff Show More