Compare commits

..

164 Commits

Author SHA1 Message Date
Alexander Petric
f9d194894d updating hub script: adding error logging to git sync 2024-12-12 18:05:34 +01:00
Alexander Petric
503930ff56 Merge branch 'main' into alp/slack_interactive_approvals 2024-12-12 17:06:50 +01:00
Alexander Petric
3529625774 date time picker and default values 2024-12-12 17:00:46 +01:00
pyranota
7398c1c09c Display memory peak while installing wheels with uv (#4889)
* fix no_uv not affecting deploy

Before this fix no_uv, no_uv_compile and no_uv_install were not affecting Dependency jobs

These jobs are only affected if used USE_PIP_COMPILE or USE_PIP_INSTALL env variables

To make it more consistant, no_uv should also affect dep jobs.

Also make ansible use uv by default

* Make it build

* Add no_uv_usage stats

* Provide build-env for Samael through shell.nix

* Run update_sqlx.sh

* Update ee-repo-ref

* Display memory peak while installing wheels with uv

* Invert DISABLE_NSJAIL

* Make it safer

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-12-12 11:26:02 +01:00
Ruben Fiszel
0dab23f6bf improve subobjects handling in schemaForm 2024-12-12 02:19:20 +01:00
Alexander Petric
045307ab41 polish messages 2024-12-12 01:25:49 +01:00
HugoCasa
a6db5cddf8 add php and mysql feature flags (#4905) 2024-12-12 01:16:25 +01:00
HugoCasa
563a492008 feat: accept direct file upload for webhook/http (s3) (#4903)
* feat: accept direct file upload for webhook/http (s3)

* nit

* nit
2024-12-12 00:33:19 +01:00
Guilhem
2f667d4065 add manual container size breakpoint (#4902)
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-12-12 00:32:27 +01:00
pyranota
3618e26326 Add no_uv_usage stats (#4883)
* fix no_uv not affecting deploy

Before this fix no_uv, no_uv_compile and no_uv_install were not affecting Dependency jobs

These jobs are only affected if used USE_PIP_COMPILE or USE_PIP_INSTALL env variables

To make it more consistant, no_uv should also affect dep jobs.

Also make ansible use uv by default

* Make it build

* Add no_uv_usage stats

* Provide build-env for Samael through shell.nix

* Run update_sqlx.sh

* Update ee-repo-ref

* Update sqlx cache

* Update ee-repo-ref

* Update ee repo ref

* Update sqlx cache

* Update ee-repo-ref.txt
2024-12-12 00:32:03 +01:00
HugoCasa
4ce4fba18f fix(frontend): form and content update when script is emptied (#4887)
* fix(frontend): form and content update when script is emptied

* fix no schema on init
2024-12-12 00:31:45 +01:00
Alexander Petric
e437764f5e adding python client 2024-12-12 00:11:43 +01:00
Ruben Fiszel
ee5c95002e frontend check fix 2024-12-11 22:44:09 +01:00
Ruben Fiszel
a75c576a65 clear env for uv pip-compile 2024-12-11 17:06:05 +01:00
Lucas Abel
1b922f3ba9 fix s3_storage decl not being the same as in ee (#4901) 2024-12-11 14:39:45 +01:00
Lucas Abel
ca1ff291d7 fix run rendering of flownode job (#4900) 2024-12-11 14:33:42 +01:00
Lucas Abel
af5cca1b00 feat(cache): re-work job results cache (#4898) 2024-12-11 12:01:48 +01:00
Alexander Petric
6b3881bd00 Merge branch 'main' into alp/slack_interactive_approvals 2024-12-11 10:35:01 +01:00
Alexander Petric
410fb25309 move form creation logic to backend 2024-12-11 10:34:33 +01:00
Lucas Abel
343cd02ad5 nix: update flake (#4895) 2024-12-11 08:57:14 +01:00
Lucas Abel
e77ae1c25b fmt: re-format Rust leftovers (#4897) 2024-12-11 08:57:00 +01:00
Ruben Fiszel
cbdb6ab8e2 chore(main): release 1.437.1 (#4891)
* chore(main): release 1.437.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-10 19:45:16 +01:00
Ruben Fiszel
bcd2cfe674 fix(frontend): fix newly deployed apps with frontend scripts 2024-12-10 19:41:11 +01:00
Ruben Fiszel
7ab42de977 chore(main): release 1.437.0 (#4877)
* chore(main): release 1.437.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-10 17:47:48 +01:00
Ruben Fiszel
5695767c2e handle empty requirement line in python 2024-12-10 16:40:12 +01:00
Ruben Fiszel
c173e46a72 fix(python): do not follow symlinks when copying recursively in overlapping sites-package 2024-12-10 15:52:47 +01:00
Ruben Fiszel
1639bb2b35 improve debug logs 2024-12-10 12:35:56 +01:00
HugoCasa
9d180ba61c add xero oauth (#4867)
* add xero oauth

* revert

* Revert "revert"

This reverts commit 46a9ed3456.

* remove xero

* add back xero
2024-12-10 00:43:39 +01:00
Ruben Fiszel
96714b8b9b update sqlx 2024-12-10 00:42:37 +01:00
Ruben Fiszel
7fbf3cb7f5 improve nested field editor 2024-12-10 00:35:14 +01:00
wendrul
4dfbc32109 Don't truncate hostnames when graph is not showing (#4862) 2024-12-10 00:02:16 +01:00
Ruben Fiszel
23875033dd feat: add ctx.name to apps #4885 2024-12-09 23:58:30 +01:00
HugoCasa
7014389d67 fix(parsers): more robust pwsh param parsing (#4884) 2024-12-09 20:55:43 +01:00
Ruben Fiszel
69195b71f3 add home in uv pip compile env 2024-12-09 20:53:43 +01:00
Ruben Fiszel
5366c8a882 chore(main): release 1.436.0 (#4858)
* chore(main): release 1.436.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-09 19:11:21 +01:00
HugoCasa
1046c0b055 fix: disabling app custom url (#4868) 2024-12-09 19:09:18 +01:00
Ruben Fiszel
b0f4caf376 backend tests 2024-12-09 18:41:13 +01:00
Ruben Fiszel
2c5d07f3d9 feat: docker runtime 2024-12-09 18:02:18 +01:00
Ruben Fiszel
81f35e7d21 update tokio and deno_core (#4871)
* all

* update
2024-12-08 16:10:46 +01:00
Ruben Fiszel
465ec09f96 improve otel 2024-12-08 11:06:35 +01:00
Ruben Fiszel
fccc889e14 add hostname to logs 2024-12-08 09:49:51 +01:00
Ruben Fiszel
d551bf424f nits 2024-12-08 01:44:06 +01:00
Ruben Fiszel
5bd0d556dd add tls to tonic 2024-12-08 00:34:46 +01:00
Ruben Fiszel
74d794a983 add tls to tonic 2024-12-07 17:49:13 +01:00
Ruben Fiszel
fb510f33fc fix tracing 2024-12-07 17:17:25 +01:00
Ruben Fiszel
6d047449e2 feat: add otlp support (#4869)
* all

* all

* add otel

* add otel

* update docker-image

* update

* update

* update

* update

* update

* update

* update

* update

* update

* update

* update
2024-12-07 16:29:20 +01:00
HugoCasa
8e9cb9c4d3 ee compute units telemetry (#4865)
* ee compute units telemetry

* update ee ref
2024-12-06 16:52:03 +01:00
Lucas Abel
691ef64688 feat(frontend): render new job kinds (#4864) 2024-12-06 15:11:57 +01:00
Lucas Abel
2bc4934c4f feat: add db storage for app inline scripts (#4837) 2024-12-06 13:37:45 +01:00
Lucas Abel
3c4408e3db feat(cache): refurbish fs backed cache (#4863)
* feat(cache): refurbish fs backed cache

* add `cached` exemple
2024-12-06 12:26:36 +01:00
Lucas Abel
f175158b9f fix: handle flow & workspace renames for flow_node (#4861) 2024-12-06 09:40:32 +01:00
Ruben Fiszel
45d4fc2de7 bump deno to 2.1.2 and bun to 1.1.38 2024-12-06 01:24:36 +01:00
Ruben Fiszel
e5efefc7e8 improve retries max value handling 2024-12-06 01:03:46 +01:00
Ruben Fiszel
9ecc94a268 optimize for loop rendering with more than 500 iterations 2024-12-05 23:52:34 +01:00
Ruben Fiszel
be624b1a9a load all branchall even if branches > 20 2024-12-05 22:45:45 +01:00
Alexander Petric
804fd3468c Merge branch 'main' into alp/slack_interactive_approvals 2024-12-05 14:56:30 -05:00
Henri Courdent
b8fd0e3849 Update changelog nov (#4859)
* Update changelog nov

* Updated link tantivy

* No duplicate
2024-12-05 19:19:58 +01:00
Ruben Fiszel
97901182b6 nit cache and skip on input node 2024-12-05 17:54:45 +01:00
Lucas Abel
667167a022 fix: fix flow_node uniqueness (#4850)
`jsonb` comparison wasn't working as expected, and duplicated entries
were inserted within `flow_node`. To resolve this add a second hash
column, `hash_v2` with a unique default for uniqueness, and use this new
column to ensure unique entries. The previous hash column is left for
backward compatibility. Duplicated entries already insterted will remain
as is without breaking, and only new ones will preserve uniqueness.
2024-12-05 17:42:33 +01:00
Ruben Fiszel
555851706a chore update vite to 5.4.11 + adapters 2024-12-05 17:17:11 +01:00
Ruben Fiszel
efd9fbe1ca update svelte-kit to 2.9.0 2024-12-05 17:03:49 +01:00
Alexander Petric
c4163aabcd update version update script to match new pwsh module formatting (#4855) 2024-12-05 17:00:20 +01:00
Ruben Fiszel
205c1a69f2 Revert "chore(deps-dev): bump @sveltejs/kit from 2.5.0 to 2.9.0 in /frontend …" (#4854)
This reverts commit cf7278e245.
2024-12-05 16:42:04 +01:00
dependabot[bot]
cf7278e245 chore(deps-dev): bump @sveltejs/kit from 2.5.0 to 2.9.0 in /frontend (#4832)
Bumps [@sveltejs/kit](https://github.com/sveltejs/kit/tree/HEAD/packages/kit) from 2.5.0 to 2.9.0.
- [Release notes](https://github.com/sveltejs/kit/releases)
- [Changelog](https://github.com/sveltejs/kit/blob/main/packages/kit/CHANGELOG.md)
- [Commits](https://github.com/sveltejs/kit/commits/@sveltejs/kit@2.9.0/packages/kit)

---
updated-dependencies:
- dependency-name: "@sveltejs/kit"
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-12-05 16:40:35 +01:00
Ruben Fiszel
185848ceb1 chore(main): release 1.435.2 (#4853)
* chore(main): release 1.435.2

* Apply automatic changes

* Update WindmillClient.psd1

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-05 16:28:00 +01:00
Alexander Petric
c67fd8dbb9 removing unneccessary requests to backend (#4852) 2024-12-05 16:19:59 +01:00
wendrul
a99e63f543 fix: job search toast on error (#4851) 2024-12-05 16:10:59 +01:00
Lucas Abel
96d4af0254 nit: format worker_lockfiles.rs (#4849) 2024-12-05 15:17:41 +01:00
Ruben Fiszel
b19faa45cd fix compile 2024-12-05 15:17:24 +01:00
Ruben Fiszel
85df359c0a improve handling of unexpected toast errors 2024-12-05 15:01:47 +01:00
Ruben Fiszel
2c934cc3b8 improve windows worker 2024-12-05 14:45:23 +01:00
Ruben Fiszel
7284f72427 chore(main): release 1.435.1 (#4846)
* chore(main): release 1.435.1

* Apply automatic changes

* Update WindmillClient.psd1

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-05 13:37:44 +01:00
Henri Courdent
93319f45bb Fix dead links (#4847) 2024-12-05 13:36:23 +01:00
Henri Courdent
1693f330b5 Added space after Use simplified builder button (#4848) 2024-12-05 13:36:12 +01:00
Ruben Fiszel
548cfcfbde fix: improve critical alerts filters 2024-12-05 13:00:17 +01:00
Ruben Fiszel
a7bdeb5fca chore(main): release 1.435.0 (#4840)
* chore(main): release 1.435.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-05 12:36:05 +01:00
Lucas Abel
c443c2b3a8 nit: format cache.rs (#4845) 2024-12-05 11:10:30 +01:00
Guilhem
e888c84a36 polish critical alert modal (#4802)
* polish critical alert modal

* Use Table component

* use table component pagination

* Add acknowledge to the table

* Polishing table

* fix nit

* Filter alerts

* Add notification count on modal

* Adjust table height

* Change mute description

* Fix layout

* minor fix

* Fix small screen issue

* fix toast on refresh

* Revert "fix toast on refresh"

This reverts commit ae3593e1af.

* filtering to backend, superadmin also acknowledges workspace (unless CLOUD_HOSTED), simplifications

* sqlx prep

* improve reactivity

* improvements

---------

Co-authored-by: Alexander Petric <alpetric@users.noreply.github.com>
Co-authored-by: Alexander Petric <petric.al@gmail.com>
Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
2024-12-05 11:07:11 +01:00
Ruben Fiszel
5404ec9b48 fix: pass USERPROFILE on windows 2024-12-05 10:12:44 +01:00
Alexander Petric
fdce1fd73f set/get state from powershell client (#4838)
* set/get state from powershell client

* format

* removing unneccessary arg
2024-12-05 00:06:15 +01:00
Alexander Petric
0b4b5404bd fix ci error unused superadmin import (#4844) 2024-12-04 23:29:36 +01:00
wendrul
ff8e98feb8 Remove unused env vars from docker compose + css fixes (#4841) 2024-12-04 18:33:41 +01:00
Ruben Fiszel
db62a2c53c simplify warn_after_seconds 2024-12-04 18:28:28 +01:00
Ruben Fiszel
bb9dd7d368 simplify warn_after_seconds 2024-12-04 18:24:31 +01:00
Ruben Fiszel
b534ef6074 improve transparency for slow queries 2024-12-04 17:30:58 +01:00
HugoCasa
1ec6c6f765 feat: app custom paths (#4828)
* feat: app custom paths

* nit

* make ee only + fix sqlx

* fix: custom http routes auth

* nits

* fix auth + nits

* apps_ee

* move custom path to ee

* fix app jwt

* update ee ref
2024-12-04 16:50:17 +01:00
Ruben Fiszel
4efa9c2b0a chore(main): release 1.434.2 (#4836)
* chore(main): release 1.434.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-12-04 14:11:44 +01:00
Henri Courdent
bf4d903083 Trigger panels texts + doclinks (#4796)
* Trigger panels texts + doclinks

* create description component

* Description module workspace settings

* Delete package-lock.json

---------

Co-authored-by: Guilhem <guilhemlemouel@gmail.com>
2024-12-04 14:07:29 +01:00
Ruben Fiszel
7642b1121e nit editors 2024-12-04 07:11:31 +01:00
Alexander Petric
e27a97dfbb feat: interactive slack approvals 2024-12-03 18:04:01 -05:00
Ruben Fiszel
71fc31c471 improve draft only delete security 2024-12-03 23:13:35 +01:00
HugoCasa
02611e42e1 fix: custom http routes auth (#4835) 2024-12-02 22:16:58 +01:00
Ruben Fiszel
7a6041d11e improve container group creation logic 2024-12-02 15:10:07 +01:00
Ruben Fiszel
75aea948ba improve public app login fallback 2024-12-02 09:10:03 +01:00
Ruben Fiszel
73a783d1b9 preprocess script for pkg export 2024-11-30 10:45:35 +01:00
Ruben Fiszel
131056a9b3 chore(main): release 1.434.1 (#4825)
* chore(main): release 1.434.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-29 22:07:25 +01:00
Ruben Fiszel
5d3ac63daf improve flow status viewer 2024-11-29 22:05:55 +01:00
Ruben Fiszel
35a4a5390d fix: improve flow status viewer performance 2024-11-29 21:30:31 +01:00
Ruben Fiszel
392023dabf chore(main): release 1.434.0 (#4823)
* chore(main): release 1.434.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-29 19:31:35 +01:00
Ruben Fiszel
dfa090f9bd update ee ref 2024-11-29 19:23:47 +01:00
HugoCasa
8b20530584 apply jwt ext tag scopes to single job endpoints (#4824)
* apply jwt ext tag scopes to single job endpoints

* fix sqlx
2024-11-29 19:23:20 +01:00
wendrul
088c666a7f feat: indexer extra settings + parallel downloads + many improvements (#4822)
* Nits on the service logs page

* Show all the hosts returned by query + sumOtherDocCount warning

* Remove from index endpoint

* Fix tests

* Prepare sqlx
2024-11-29 19:14:05 +01:00
Ruben Fiszel
aa1c3702d7 fix empty branchall subflow 2024-11-29 19:02:47 +01:00
Ruben Fiszel
a2411bcea5 fix: improve flow status viwer iteration picker 2024-11-29 18:33:34 +01:00
Ruben Fiszel
658f71ffe9 schedule nits 2024-11-29 17:01:20 +01:00
Ruben Fiszel
75e9132817 app nits (#4820)
* all

* all
2024-11-29 16:44:00 +01:00
Ruben Fiszel
45d434c875 prettify cache msg 2024-11-29 13:01:54 +01:00
Ruben Fiszel
b1603690f3 nit pip install 2024-11-29 12:37:52 +01:00
Ruben Fiszel
d36e2a2647 add total time 2024-11-29 12:35:10 +01:00
Ruben Fiszel
9db86306a9 small uv s3 pull fix 2024-11-29 12:31:50 +01:00
Ruben Fiszel
30a98c4abe improve oauth popup flows using localstorage as fallback 2024-11-29 12:24:12 +01:00
Ruben Fiszel
f6eeef0dc6 chore(main): release 1.433.0 (#4819)
* chore(main): release 1.433.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-29 10:21:39 +01:00
Ruben Fiszel
0a95b6f9d2 fix: fix windows build 2024-11-29 09:53:05 +01:00
Lucas Abel
3fbb2bfc8a feat(cache): implement flow node caching (#4808)
* feat(cache): implement flow node caching

* feat(cache): implement script caching

* feat(cache): improve cache
2024-11-29 09:35:00 +01:00
Ruben Fiszel
f7908682d3 chore(main): release 1.432.0 (#4818)
* chore(main): release 1.432.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-29 09:12:46 +01:00
Ruben Fiszel
8954b0b68c skip s3 python cache on windows and pro 2024-11-29 08:35:08 +01:00
pyranota
96371bc89d feat: Parallelize uv install (#4774)
* Implement MVP of Parallel uv installation

* Implement PY_CONCURRENT_DOWNLOADS

* Remove Flock for uv installs

* Make S3 pull/push parallel

* Refactor and allow to Cancel installation

* Dont print S3 in output if disabled

* Implement better error handling

* Polishing

* More polishing

* Implement error-handler for kill_tx_2.send()

* Fix and Format prev merge

* Presubscribe to all kill_tx's

We do it now before first event could fire
Meaning no events can be lost anymore

* Early print errors and safer error handling

* Return Err if installation failed

* Final changes

* Return error instead of just printing it

* Safer the way to acquire permit

* Fix compilation error

* Remove double error logs
2024-11-29 08:21:51 +01:00
Ruben Fiszel
8ebb1fe363 window openeer nits 2024-11-29 07:48:37 +01:00
Ruben Fiszel
05416054bb nit exportable whitelabel 2024-11-28 19:56:19 +01:00
Ruben Fiszel
12dbbab401 chore(main): release 1.431.1 (#4816)
* chore(main): release 1.431.1

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-28 17:36:52 +01:00
Lucas Abel
390ee3113b fix: invalid null comparison while inserting flow node (#4815) 2024-11-28 17:29:16 +01:00
Ruben Fiszel
8a814d4699 chore(main): release 1.431.0 (#4788)
* chore(main): release 1.431.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-28 17:22:24 +01:00
Ruben Fiszel
5656102b13 fix: fix OAuth being stuck if following a public app login 2024-11-28 16:40:18 +01:00
Ruben Fiszel
8f56456701 fix: global timeout not set correctly on load 2024-11-28 16:02:52 +01:00
Lucas Abel
77937a494a fix: invalid jsonb comparison while inserting flow node (#4814) 2024-11-28 14:43:54 +01:00
wendrul
a9c31b35f4 fix: add missing export tab on mobile view of flow detail (#4812) 2024-11-28 13:25:44 +01:00
pyranota
a853c8d6a5 Remove flock (#4809) 2024-11-28 12:42:25 +01:00
Ruben Fiszel
bf6c4131ca save to workspace handle trigger scripts 2024-11-28 10:30:23 +01:00
Ruben Fiszel
ac8d359dfb fix apply migration to current_user 2024-11-27 19:07:31 +01:00
wendrul
09fe535bdc fix: service logs: aggregation query + killpill on index pull + retention period for logs on s3 (#4795)
* Search across all hosts in index. killpill on index pull

* Add retention period for service logs

* Remove console log

* monitor log_file when older than biggest retention period (job / log)

* Retention period deletes service and job logs

* Add setting to monitor logs out of s3

* Prepare sqlx

* Update ee repo ref

* Prepare sqlx

* Unused variables
2024-11-27 18:57:23 +01:00
Ruben Fiszel
c998102cf3 simplify logic to check for license key on workers 2024-11-27 18:51:06 +01:00
Alexander Petric
2668e37574 adding sqlx migration to grant windmill permissions to db user (#4806) 2024-11-27 18:27:06 +01:00
Ruben Fiszel
c25c638bdf add more links to operators menu 2024-11-27 17:53:45 +01:00
Lucas Abel
2911775d0a feat(backend): store flow inner modules into flow_node table (#4778) 2024-11-27 16:08:31 +01:00
Lucas Abel
ff92e2f897 fix simple step flow rendering for while loop (#4804) 2024-11-27 16:07:16 +01:00
pyranota
34609a7d8c Fix tar prefix (#4781)
* Fix tar prefix

* Use TAR_PIP(PYTHON_311)_CACHE_DIR instead

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2024-11-27 15:54:47 +01:00
Henri Courdent
1344517bc7 Gi syntax docker compose (#4803) 2024-11-27 14:29:41 +01:00
Lucas Abel
d86ea30221 enable is_simple optimization for FlowScript (#4799)
* backend: remove some allocations in `worker_flow.rs`

* feat(backend): enable `is_simple` optimization for `FlowScript`

* fix: fix node retrieval for `flowscript` inner step
2024-11-27 10:06:19 +01:00
Lucas Abel
150ca33f0d fix: disable flow_version_lite in dedicated workers (#4800) 2024-11-27 10:02:44 +01:00
Ruben Fiszel
24480260bd fix: improve default formatters for deno 2024-11-27 09:01:06 +01:00
pyranota
e876ae4f42 fix(python): Merge to site-packages packages with same name (#4793)
* (python) Initially implement merge

Iterate overall all python paths and if same folder has same name multiple times,
then merge the content and put to <job_dir>/site-packages

Solves problem with imports for some dependencies.

Default layout (/windmill/cache/):

dep==x.y.z
└── X
   └── A
dep-ext==x.y.z
└── X
   └── B

In this case python would be confused with finding B module.

This function will convert it to (/<job_id>):

site-packages
└── X
   ├── A
   └── B

This way python has no problems with finding correct module

* Better error-handling

* Optimize and fix bug

* Update backend/windmill-worker/src/python_executor.rs

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* Update backend/windmill-worker/src/python_executor.rs

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>

* Polish

* Add path to PathBuf

* Add & before PathBuf

---------

Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com>
2024-11-26 18:24:26 +01:00
Ruben Fiszel
66235c0406 fix darkMode theme on apps 2024-11-26 17:44:28 +01:00
Ruben Fiszel
e92750c687 first row fix 2024-11-26 17:07:27 +01:00
Ruben Fiszel
5d0eba273a feat: expose settable col id for app aggrid tables 2024-11-26 16:38:15 +01:00
Ruben Fiszel
f69f743b30 feat: allow forcing dark/light theme on apps 2024-11-26 13:38:31 +01:00
Ruben Fiszel
ead6323324 nit schema form resource type change 2024-11-26 12:48:28 +01:00
Ruben Fiszel
ef721f9a5f fix: improve flow status viwer iteration picker 2024-11-26 12:38:34 +01:00
Ruben Fiszel
a426e7f3ab app nits 2024-11-26 10:00:55 +01:00
Ruben Fiszel
fe4b2b73d4 app nits 2024-11-26 09:39:03 +01:00
Ruben Fiszel
7fb26f58e8 improve app editor context panel performance for large apps 2024-11-26 09:28:11 +01:00
Ruben Fiszel
2427d64aa8 nit 2024-11-25 23:32:14 +01:00
Lucas Abel
e4784e89da feat(backend): implement flow scripts (#4748)
* feat(backend): implement flow scripts

* conditionally resolve reduced flow value

* conditionally use flow scripts based on the min worker version

* grant all to `flow_{version_lite,node}`

* do not silently discard errors

* env: add flow script feature flag

* flow script on cloud automatically

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
2024-11-25 23:20:51 +01:00
Ruben Fiszel
a182b91750 nit UX 2024-11-25 23:14:34 +01:00
Ruben Fiszel
dd5081592f email quotas only apply to top-level job 2024-11-25 16:20:55 +01:00
Ruben Fiszel
e81e239f43 fix: fix getting logs locally for ee without instance settings 2024-11-25 12:02:35 +01:00
Ruben Fiszel
cd7385f38d improve error message for log file not found on ee 2024-11-25 11:07:48 +01:00
Ruben Fiszel
1baf9d008b chore(main): release 1.430.2 (#4782)
* chore(main): release 1.430.2

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2024-11-25 09:42:04 +01:00
Ruben Fiszel
d13143a409 rm devfrontend 2024-11-25 09:05:55 +01:00
Lucas Abel
29faefe856 fix: rename job workspace as well (#4785) 2024-11-25 08:09:30 +01:00
Lucas Abel
39b902c118 backend: keep tracking minimal worker version (#4786)
Be sure to also track the min version of workers that have been started
after the min version variable has been set. e.g. All workers are up to
date and `MIN_VERSION_IS_AT_LEAST_1_427` is set to true, a worker is
then powered on with an anterior version, leading to a false positive
value. This commit fix that.
2024-11-25 08:08:16 +01:00
Ruben Fiszel
6768e5bbbd fix: retry on inserting completed job (#4784)
* fix: retry on pushing next scheduled job of schedule

* all

* improve error handling

* update sqlx
2024-11-25 01:33:44 +01:00
Ruben Fiszel
5ed0ae697b improve backoff for schedules 2024-11-24 18:48:01 +01:00
Ruben Fiszel
89456f502f remove rsmq 2024-11-24 11:04:54 +01:00
Ruben Fiszel
278f593358 fix: retry on pushing next scheduled job of schedule 2024-11-24 08:52:51 +01:00
347 changed files with 14763 additions and 8082 deletions

View File

@@ -16,7 +16,7 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
sed -i '' -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile

View File

@@ -17,7 +17,7 @@ sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/pac
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
sed -i -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile

View File

@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}

View File

@@ -62,7 +62,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |

View File

@@ -45,7 +45,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql
- name: Rename binary with corresponding architecture
run: |

View File

@@ -67,7 +67,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,deno_core
features=embedding,parquet,openidconnect,deno_core,php,mysql
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
${{ steps.meta-public.outputs.tags }}

View File

@@ -76,7 +76,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core
features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -138,7 +138,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +200,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql
PYTHON_IMAGE=python:3.12.2-slim-bookworm
tags: |
${{ steps.meta-ee-public-py312.outputs.tags }}

View File

@@ -47,7 +47,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql
- name: Rename binary with corresponding architecture
run: |

View File

@@ -1,5 +1,153 @@
# Changelog
## [1.437.1](https://github.com/windmill-labs/windmill/compare/v1.437.0...v1.437.1) (2024-12-10)
### Bug Fixes
* **frontend:** fix newly deployed apps with frontend scripts ([bcd2cfe](https://github.com/windmill-labs/windmill/commit/bcd2cfe674038042f08f22d05286596b3442caf9))
## [1.437.0](https://github.com/windmill-labs/windmill/compare/v1.436.0...v1.437.0) (2024-12-10)
### Features
* add ctx.name to apps [#4885](https://github.com/windmill-labs/windmill/issues/4885) ([2387503](https://github.com/windmill-labs/windmill/commit/23875033dd19d6d585f2433fa7b18a703175e329))
### Bug Fixes
* **parsers:** more robust pwsh param parsing ([#4884](https://github.com/windmill-labs/windmill/issues/4884)) ([7014389](https://github.com/windmill-labs/windmill/commit/7014389d677fba563bec02e08815452b195ecd0a))
* **python:** do not follow symlinks when copying recursively in overlapping sites-package ([c173e46](https://github.com/windmill-labs/windmill/commit/c173e46a724b50d27416bbc169bf2a57fe68808c))
## [1.436.0](https://github.com/windmill-labs/windmill/compare/v1.435.2...v1.436.0) (2024-12-09)
### Features
* add db storage for app inline scripts ([#4837](https://github.com/windmill-labs/windmill/issues/4837)) ([2bc4934](https://github.com/windmill-labs/windmill/commit/2bc4934c4f38ca2631eb99140d89a0b3d3b3a2cf))
* add otlp support ([#4869](https://github.com/windmill-labs/windmill/issues/4869)) ([6d04744](https://github.com/windmill-labs/windmill/commit/6d047449e200d51f060d4107ff5ad0e10af99a66))
* **cache:** refurbish fs backed cache ([#4863](https://github.com/windmill-labs/windmill/issues/4863)) ([3c4408e](https://github.com/windmill-labs/windmill/commit/3c4408e3dbf8a37c0b862805b1991af3d39ac054))
* docker runtime ([2c5d07f](https://github.com/windmill-labs/windmill/commit/2c5d07f3d9069b21b877ea2d085f0a851b1975c1))
* **frontend:** render new job kinds ([#4864](https://github.com/windmill-labs/windmill/issues/4864)) ([691ef64](https://github.com/windmill-labs/windmill/commit/691ef6468823c99ab129c53ae699de1a91f77c97))
### Bug Fixes
* fix `flow_node` uniqueness ([#4850](https://github.com/windmill-labs/windmill/issues/4850)) ([667167a](https://github.com/windmill-labs/windmill/commit/667167a022ed31cb67a6c189cc5bd54b09473f4f))
* handle flow & workspace renames for `flow_node` ([#4861](https://github.com/windmill-labs/windmill/issues/4861)) ([f175158](https://github.com/windmill-labs/windmill/commit/f175158b9ffcc0d9c35938e61dfe3ce56d3b64c2))
## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05)
### Bug Fixes
* job search toast on error ([#4851](https://github.com/windmill-labs/windmill/issues/4851)) ([a99e63f](https://github.com/windmill-labs/windmill/commit/a99e63f5435725c42e38b46933ff3caa345002b5))
## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05)
### Bug Fixes
* improve critical alerts filters ([548cfcf](https://github.com/windmill-labs/windmill/commit/548cfcfbde23ac7ae129e10f6e92d57516e61f20))
## [1.435.0](https://github.com/windmill-labs/windmill/compare/v1.434.2...v1.435.0) (2024-12-05)
### Features
* app custom paths ([#4828](https://github.com/windmill-labs/windmill/issues/4828)) ([1ec6c6f](https://github.com/windmill-labs/windmill/commit/1ec6c6f765904361e641d89495890bc87e8544aa))
### Bug Fixes
* pass USERPROFILE on windows ([5404ec9](https://github.com/windmill-labs/windmill/commit/5404ec9b48e8d7a0cb27b2319d54f04c94ec6fd0))
## [1.434.2](https://github.com/windmill-labs/windmill/compare/v1.434.1...v1.434.2) (2024-12-04)
### Bug Fixes
* custom http routes auth ([#4835](https://github.com/windmill-labs/windmill/issues/4835)) ([02611e4](https://github.com/windmill-labs/windmill/commit/02611e42e1b5c81d9bc57e3b46795e4c8cad67f7))
## [1.434.1](https://github.com/windmill-labs/windmill/compare/v1.434.0...v1.434.1) (2024-11-29)
### Bug Fixes
* improve flow status viewer performance ([35a4a53](https://github.com/windmill-labs/windmill/commit/35a4a5390da0d3c2acad0f8af9aae9e4fdc149f5))
## [1.434.0](https://github.com/windmill-labs/windmill/compare/v1.433.0...v1.434.0) (2024-11-29)
### Features
* indexer extra settings + parallel downloads + many improvements ([#4822](https://github.com/windmill-labs/windmill/issues/4822)) ([088c666](https://github.com/windmill-labs/windmill/commit/088c666a7f9c0f074cdd312e39b79418045dc7db))
### Bug Fixes
* improve flow status viwer iteration picker ([a2411bc](https://github.com/windmill-labs/windmill/commit/a2411bcea5abc5388543ef903ec52de04e3a2082))
## [1.433.0](https://github.com/windmill-labs/windmill/compare/v1.432.0...v1.433.0) (2024-11-29)
### Features
* **cache:** implement flow node caching ([#4808](https://github.com/windmill-labs/windmill/issues/4808)) ([3fbb2bf](https://github.com/windmill-labs/windmill/commit/3fbb2bfc8a191af9a1657824f2d9bff7e57e4b86))
### Bug Fixes
* fix windows build ([0a95b6f](https://github.com/windmill-labs/windmill/commit/0a95b6f9d241c59869c78919352e1bbacf7c1aa7))
## [1.432.0](https://github.com/windmill-labs/windmill/compare/v1.431.1...v1.432.0) (2024-11-29)
### Features
* Parallelize `uv install` ([#4774](https://github.com/windmill-labs/windmill/issues/4774)) ([96371bc](https://github.com/windmill-labs/windmill/commit/96371bc89dfaabdf1e7a9ecd34d13db819985b3f))
## [1.431.1](https://github.com/windmill-labs/windmill/compare/v1.431.0...v1.431.1) (2024-11-28)
### Bug Fixes
* invalid `null` comparison while inserting flow node ([#4815](https://github.com/windmill-labs/windmill/issues/4815)) ([390ee31](https://github.com/windmill-labs/windmill/commit/390ee3113bd203cad89ff7d7cb0cf2da154c40c4))
## [1.431.0](https://github.com/windmill-labs/windmill/compare/v1.430.2...v1.431.0) (2024-11-28)
### Features
* allow forcing dark/light theme on apps ([f69f743](https://github.com/windmill-labs/windmill/commit/f69f743b30f3fcfaf83979f2fe55c2e6e1cf9a03))
* **backend:** implement flow scripts ([#4748](https://github.com/windmill-labs/windmill/issues/4748)) ([e4784e8](https://github.com/windmill-labs/windmill/commit/e4784e89dac9fbfe492fa2d25fcaf373afc3b340))
* **backend:** store flow inner modules into `flow_node` table ([#4778](https://github.com/windmill-labs/windmill/issues/4778)) ([2911775](https://github.com/windmill-labs/windmill/commit/2911775d0a138220cc353aa40e416201130bbd1e))
* expose settable col id for app aggrid tables ([5d0eba2](https://github.com/windmill-labs/windmill/commit/5d0eba273ab7a5264b4ecb7270fb9f13423ccec7))
### Bug Fixes
* add missing export tab on mobile view of flow detail ([#4812](https://github.com/windmill-labs/windmill/issues/4812)) ([a9c31b3](https://github.com/windmill-labs/windmill/commit/a9c31b35f4d50ee399c1dc965e181504cc3bf8aa))
* disable `flow_version_lite` in dedicated workers ([#4800](https://github.com/windmill-labs/windmill/issues/4800)) ([150ca33](https://github.com/windmill-labs/windmill/commit/150ca33f0d022aec55a7c8c0f4516b635ea824d6))
* fix getting logs locally for ee without instance settings ([e81e239](https://github.com/windmill-labs/windmill/commit/e81e239f43fb4302940f0a9d910fafd560084da3))
* fix OAuth being stuck if following a public app login ([5656102](https://github.com/windmill-labs/windmill/commit/5656102b13d63bae5c4c78876d839be73f823450))
* global timeout not set correctly on load ([8f56456](https://github.com/windmill-labs/windmill/commit/8f564567018fa7ff5bfd74aeeb95699fefcf5a63))
* improve default formatters for deno ([2448026](https://github.com/windmill-labs/windmill/commit/24480260bd638fb819ccfb4ace68be844ae3776f))
* improve flow status viwer iteration picker ([ef721f9](https://github.com/windmill-labs/windmill/commit/ef721f9a5f03a06378941d382ce02fa069213ae6))
* invalid `jsonb` comparison while inserting flow node ([#4814](https://github.com/windmill-labs/windmill/issues/4814)) ([77937a4](https://github.com/windmill-labs/windmill/commit/77937a494ada78891cd5aa01e11c2d813fe8d45a))
* **python:** Merge to site-packages packages with same name ([#4793](https://github.com/windmill-labs/windmill/issues/4793)) ([e876ae4](https://github.com/windmill-labs/windmill/commit/e876ae4f42ed0655fa5e1f02e29a203785cfb4cd))
* service logs: aggregation query + killpill on index pull + retention period for logs on s3 ([#4795](https://github.com/windmill-labs/windmill/issues/4795)) ([09fe535](https://github.com/windmill-labs/windmill/commit/09fe535bdca548e0445ffb07d86a1c3cf656d5e5))
## [1.430.2](https://github.com/windmill-labs/windmill/compare/v1.430.1...v1.430.2) (2024-11-25)
### Bug Fixes
* rename `job` workspace as well ([#4785](https://github.com/windmill-labs/windmill/issues/4785)) ([29faefe](https://github.com/windmill-labs/windmill/commit/29faefe856041b29c58dbeef862320a13abcf8a8))
* retry on inserting completed job ([#4784](https://github.com/windmill-labs/windmill/issues/4784)) ([6768e5b](https://github.com/windmill-labs/windmill/commit/6768e5bbbd207fd21da2d63e43138a3a0488c0c4))
* retry on pushing next scheduled job of schedule ([278f593](https://github.com/windmill-labs/windmill/commit/278f5933587102f244047d4d6f54defe6d6066eb))
## [1.430.1](https://github.com/windmill-labs/windmill/compare/v1.430.0...v1.430.1) (2024-11-23)

View File

@@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer

View File

@@ -334,7 +334,8 @@ you to have it being synced automatically everyday.
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| GOPROXY | | The GOPROXY env variable to use | Worker |
| NETRC | | The netrc content to use a private go registry | Worker |
| NETRC | | The netrc content to use a private go registry | Worker | | Worker |
| PY_CONCURRENT_DOWNLOADS | 20 | Sets the maximum number of in-flight concurrent python downloads that windmill will perform at any given time. | Worker |
| PATH | None | The path environment variable, usually inherited | Worker |
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value as \"value: sqlx::types::Json<Box<RawValue>>\" FROM flow_version WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "00a16dbe1729d530f136279471fa3d89ca7f1a88a48fcf0b1ac07144924c1707"
}

View File

@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET last_ping = now() WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "062859f1d0e5cfba3115f4241115753b86a4ad239708851c998ff5620ebca5b8"
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts\n WHERE COALESCE(acknowledged, false) = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Bool"
]
},
"nullable": [
null
]
},
"hash": "0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe"
}

View File

@@ -42,7 +42,10 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Bpchar",
"Text",
"Text",
"Bpchar"
]
},
"nullable": [
false
]
},
"hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE global_settings SET value = $1 WHERE name = 'indexer_settings'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465"
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!: String",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20"
}

View File

@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'python' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_py'\n\n UNION ALL\n \n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'ansible' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_ansible'\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_deploy",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "deploys_count",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "language",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null
]
},
"hash": "240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0"
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "log_file_index",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": [
true
]
},
"hash": "288e99211bbd45a337fc9b79c43c5139ee535e23c8b3362c52eef49998349f15"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO flow_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "2de4ac337d25cc483829818393fa3254556c0307bb6bb596613b1ce2ee7ec4cb"
}

View File

@@ -32,7 +32,10 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE job SET workspace_id = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "43fcdf5243e17bfbdcd21f09feee6e104b40f4b937914f56f01c299cddfc17e9"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_logs WHERE job_id = ANY($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa"
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT \n MAX (created_at) as last_deploy, \n COUNT (*) as deploys_count \n FROM metrics \n WHERE id = 'no_uv_usage_py'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_deploy",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "deploys_count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "497d93db931922b09f96cf73239513d7141f3d37f85ada46597079991b3bff30"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.value \n FROM flow \n LEFT JOIN flow_version \n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "4a764cdcb847b71183425a7a0e863708ef7fe2c88b28d0667988a47b9e995c0e"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id FROM queue WHERE id = ANY($1) AND schedule_path IS NULL",
"query": "SELECT id FROM queue WHERE id = ANY($1) AND schedule_path IS NULL AND ($2::text[] IS NULL OR tag = ANY($2))",
"describe": {
"columns": [
{
@@ -11,12 +11,13 @@
],
"parameters": {
"Left": [
"UuidArray"
"UuidArray",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "273d275be89516b135d7846d179c84ba0d684a620e9e5c0c87f82bdc9cd57dbe"
"hash": "55541316c690e4f2e1b7a41071ef0a297a2e65c8e25a17d5c43715481e7633a0"
}

View File

@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bpchar",
"Text",
"Text",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "policy: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite \n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value!: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "625c400bd81369a8142b8c7fa67b2be02f0f945e587c90d7e0f23706bf6e9a31"
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"Text"
]
},
"nullable": []
},
"hash": "65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a"
}

View File

@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM queue \n LEFT JOIN job_logs ON job_logs.job_id = queue.id \n WHERE queue.id = $1 AND queue.workspace_id = $2",
"query": "SELECT created_by, CONCAT(coalesce(queue.logs, ''), coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index\n FROM queue \n LEFT JOIN job_logs ON job_logs.job_id = queue.id \n WHERE queue.id = $1 AND queue.workspace_id = $2 AND ($3::text[] IS NULL OR queue.tag = ANY($3))",
"describe": {
"columns": [
{
@@ -27,7 +27,8 @@
"parameters": {
"Left": [
"Uuid",
"Text"
"Text",
"TextArray"
]
},
"nullable": [
@@ -37,5 +38,5 @@
true
]
},
"hash": "b2c30137b3b64f8c3f6aea062e8687e119599641beee0a0d6c89c5d90e82d0db"
"hash": "6682bf34caf7efa95b60c747b45dcdd41de7f8e197b163f7865810391471db5b"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"describe": {
"columns": [
{
@@ -32,6 +32,11 @@
"ordinal": 5,
"name": "started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "worker_group",
"type_info": "Varchar"
}
],
"parameters": {
@@ -43,8 +48,9 @@
true,
true,
false,
false,
false
]
},
"hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7"
"hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM usr where username = $1 AND workspace_id = $2",
"query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n ",
"describe": {
"columns": [
{
@@ -42,6 +42,16 @@
"ordinal": 7,
"name": "role",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
@@ -58,8 +68,10 @@
false,
false,
false,
true,
false,
true
]
},
"hash": "3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a"
"hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only)\n VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
"query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only, custom_path)\n VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id",
"describe": {
"columns": [
{
@@ -15,12 +15,13 @@
"Varchar",
"Varchar",
"Jsonb",
"Bool"
"Bool",
"Text"
]
},
"nullable": [
false
]
},
"hash": "75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2"
"hash": "6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5"
}

View File

@@ -26,7 +26,10 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_node SET workspace_id = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "79992fd5650a99a4391e7c9e590d9ee7969f98975406fa13118d2b7e808958cf"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2 THEN\n CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n ELSE true\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a"
}

View File

@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3, edited_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n ORDER BY log_ts ASC LIMIT $1",
"query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > NOW() - make_interval(secs => $1)\n ORDER BY log_ts ASC LIMIT $2",
"describe": {
"columns": [
{
@@ -46,6 +46,7 @@
],
"parameters": {
"Left": [
"Float8",
"Int8"
]
},
@@ -60,5 +61,5 @@
true
]
},
"hash": "dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6"
"hash": "8d207cc9ed101ff116b617d25a94633c1531170ded1fdf09114718b941f5e1db"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n LEFT JOIN flow_version_lite \n ON flow_version_lite.id = flow_version.id\n WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value!: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "94a38b83d59f8c010bd96852bf57163476791448201bdd22a9e463a4116b9582"
}

View File

@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "file_path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "hostname",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033"
}

View File

@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f"
}

View File

@@ -26,7 +26,10 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}

View File

@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', ''::text::jsonb)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a1f8667bfd5b000dd7ba384e78f2bd7fabb6b8055f4559f77e20eef7c2b1c902"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.value AS \"value: sqlx::types::Json<Box<RawValue>>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "ae543dfa106fa6ad4e9bf45cda1110d4702a80f455c63af6fcbd7cf45bbc4f7a"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by, CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM completed_job \n LEFT JOIN job_logs ON job_logs.job_id = completed_job.id \n WHERE completed_job.id = $1 AND completed_job.workspace_id = $2",
"query": "SELECT created_by, CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n FROM completed_job \n LEFT JOIN job_logs ON job_logs.job_id = completed_job.id \n WHERE completed_job.id = $1 AND completed_job.workspace_id = $2 AND ($3::text[] IS NULL OR completed_job.tag = ANY($3))",
"describe": {
"columns": [
{
@@ -27,7 +27,8 @@
"parameters": {
"Left": [
"Uuid",
"Text"
"Text",
"TextArray"
]
},
"nullable": [
@@ -37,5 +38,5 @@
true
]
},
"hash": "b4c4b1aa4bcf7e5f49ce3bd93fff3b7c49e812271e4888d33eb73dd638486488"
"hash": "afd0d1b0511f32ba1981bc8d09b729639a6c0b468aa3f9108071d160d2dee250"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts\n WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.value AS \"value!: sqlx::types::Json<Box<sqlx::types::JsonRawValue>>\" \n FROM flow \n LEFT JOIN flow_version \n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value!: sqlx::types::Json<Box<sqlx::types::JsonRawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "bbce3e1eae78c48409d4204cd6cb3b9db088f6e51bea5e74a494c4e9f4c3b78e"
}

View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock AS \"lock: String\", code AS \"code: String\", flow::text AS \"flow: Box<str>\" FROM flow_node WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock: String",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "code: String",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "flow: Box<str>",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
true,
null
]
},
"hash": "c57ed2d91de46d7de88e20b94b7afbafb622528864f2b23c8b7278bd506d967f"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $3 THEN\n CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n ELSE true\n END\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8"
}

View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_ansible', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052"
}

View File

@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.value as \"value!: sqlx::types::Json<Box<RawValue>>\" FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 AND flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value!: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "dc44783a555d41c9df4cd303af686e43bbf89add915bfb09d94e25e8517d65ff"
}

View File

@@ -0,0 +1,72 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock AS \"lock: String\", content AS \"code!: String\",\n language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", codebase AS \"codebase: String\" FROM script WHERE hash = $1 AND workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock: String",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "code!: String",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "language: Option<ScriptLang>",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible"
]
}
}
}
},
{
"ordinal": 3,
"name": "envs: Vec<String>",
"type_info": "VarcharArray"
},
{
"ordinal": 4,
"name": "codebase: String",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
true,
false,
false,
true,
true
]
},
"hash": "df52a71d59eb84a2b08133d25f0a8bba7f2b56625fdb7a7c0e10b51377eeb1d4"
}

View File

@@ -1,10 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value FROM app_version WHERE id = $1",
"query": "SELECT app_id, value FROM app_version WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "app_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Json"
}
@@ -15,8 +20,9 @@
]
},
"nullable": [
false,
false
]
},
"hash": "29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a"
"hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b"
}

View File

@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT \n MAX (created_at) as last_deploy, \n COUNT (*) as deploys_count \n FROM metrics \n WHERE id = 'no_uv_usage_ansible'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_deploy",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "deploys_count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "eb125df0f64c0caa07f50dc82e8ae9c6cd2872c7afe96d41bed731c5041ab671"
}

View File

@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE flow_node SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "ed12d54e9cdc18672bbb34d2500adc18b72323e06a1f32e7a1a7589649888927"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority\n )\n VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000, $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms",
"query": "INSERT INTO completed_job AS cj\n ( workspace_id\n , id\n , parent_job\n , created_by\n , created_at\n , started_at\n , duration_ms\n , success\n , script_hash\n , script_path\n , args\n , result\n , raw_code\n , raw_lock\n , canceled\n , canceled_by\n , canceled_reason\n , job_kind\n , schedule_path\n , permissioned_as\n , flow_status\n , raw_flow\n , is_flow_step\n , is_skipped\n , language\n , email\n , visible_to_owner\n , mem_peak\n , tag\n , priority\n )\n VALUES ($1, $2, $3, $4, $5, COALESCE($6, now()), COALESCE($30::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)\n ON CONFLICT (id) DO UPDATE SET success = $7, result = $11 RETURNING duration_ms",
"describe": {
"columns": [
{
@@ -46,7 +46,10 @@
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
@@ -87,12 +90,13 @@
"Bool",
"Int4",
"Varchar",
"Int2"
"Int2",
"Int8"
]
},
"nullable": [
false
]
},
"hash": "d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1"
"hash": "f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548"
}

View File

@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df"
}

View File

@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "code",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
false
]
},
"hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c"
}

1447
backend/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.430.1"
version = "1.437.1"
authors.workspace = true
edition.workspace = true
@@ -29,7 +29,7 @@ members = [
]
[workspace.package]
version = "1.430.1"
version = "1.437.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -63,6 +63,10 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
kafka = ["windmill-api/kafka"]
otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
php = ["windmill-worker/php"]
mysql = ["windmill-worker/mysql"]
[dependencies]
anyhow.workspace = true
@@ -83,7 +87,6 @@ chrono.workspace = true
git-version.workspace = true
base64.workspace = true
sha2.workspace = true
rsmq_async.workspace = true
url.workspace = true
lazy_static.workspace = true
once_cell.workspace = true
@@ -96,7 +99,6 @@ deno_core = { workspace = true, optional = true }
object_store = { workspace = true, optional = true }
quote.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
tikv-jemalloc-sys = { optional = true, workspace = true }
@@ -138,7 +140,7 @@ windmill-api-client = { path = "./windmill-api-client" }
axum = { version = "^0.7", features = ["multipart"] }
headers = "^0"
hyper = { version = "^1", features = ["full"] }
tokio = { version = "^1", features = ["full", "tracing"] }
tokio = { version = "^1.42.0", features = ["full", "tracing"] }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors"] }
tower-cookies = "^0.10"
@@ -183,15 +185,16 @@ itertools = "^0"
regex = "^1"
semver = "^1"
deno_fetch = "0.195.0"
deno_tls = "0.158.0"
deno_console = "0.171.0"
deno_url = "0.171.0"
deno_webidl = "0.171.0"
deno_web = "0.202.0"
deno_net = "0.163.0"
deno_core = "0.311.0"
deno_ast = { version = "=0.42.2", features = ["transpiling"] }
deno_fetch = "0.203.0"
deno_tls = "0.166.0"
deno_console = "0.179.0"
deno_url = "0.179.0"
deno_webidl = "0.179.0"
deno_web = "0.210.0"
deno_net = "0.171.0"
deno_core = "0.321.0"
deno_ast = { version = "=0.43.3", features = ["transpiling"] }
deno_permissions = "0.39.0"
swc_common = "=0.37.5"
swc_ecma_parser = "=0.149.1"
@@ -231,7 +234,6 @@ async-stripe = { version = "0.39.1", features = [
] }
async_zip = { version = "0.0.11", features = ["full"] }
once_cell = "1.17.1"
rsmq_async = { version = "5.1.5" }
gosyn = "0.2.6"
bytes = "1.4.0"
gethostname = "0.4.3"
@@ -281,14 +283,28 @@ tar = "^0"
http = "^1"
async-stream = "^0"
opentelemetry = "0.27.0"
tracing-opentelemetry = "0.28.0"
opentelemetry_sdk = { version = "*", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
opentelemetry-appender-tracing = "0.27.0"
opentelemetry-semantic-conventions = { version = "*", features = ["semconv_experimental"] }
bollard = "0.18.1"
tonic = { version = "^0", features = ["tls-native-roots"] }
tikv-jemallocator = { version = "0.5" }
tikv-jemalloc-sys = { version = "^0.5" }
tikv-jemalloc-ctl = { version = "^0.5" }
triomphe = "^0"
pin-project-lite = "^0"
tantivy = "0.22.0"
backon = "1.3.0"
# Macro-related
proc-macro2 = "1.0"
pulldown-cmark = "0.9"
@@ -297,4 +313,4 @@ syn = { version = "2.0.74", features = ["full"] }
quote = "1.0.36"
regex-lite = "0.1.6"
yaml-rust = "0.4.5"
tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }

View File

@@ -1 +1 @@
4a0e64ab860eb88ff0d21b8705e647c9b76c46ee
271c210ddbbdcad0f4d6c007e650eceda5ddfa64

View File

@@ -0,0 +1,4 @@
-- Add down migration script here
DROP TABLE IF EXISTS flow_version_lite;
DROP TABLE IF EXISTS flow_node;
DROP INDEX IF EXISTS flow_node_hash;

View File

@@ -0,0 +1,29 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'flowscript';
-- Same as `flow_version` but with a "lite" value (e.g. `RawScript` replaced by `FlowScript`).
CREATE TABLE flow_version_lite (
id BIGSERIAL PRIMARY KEY,
value JSONB,
FOREIGN KEY (id) REFERENCES flow_version (id) ON DELETE CASCADE
);
GRANT ALL ON flow_version_lite TO windmill_user;
GRANT ALL ON flow_version_lite TO windmill_admin;
-- Either a script or a flow value.
CREATE TABLE flow_node (
id BIGSERIAL PRIMARY KEY,
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
hash BIGINT NOT NULL,
path VARCHAR(255) NOT NULL, -- flow path.
lock TEXT,
code TEXT,
flow JSONB,
FOREIGN KEY (path, workspace_id) REFERENCES flow (path, workspace_id) ON DELETE CASCADE
);
GRANT ALL ON flow_node TO windmill_user;
GRANT ALL ON flow_node TO windmill_admin;
CREATE INDEX flow_node_hash ON flow_node (hash);

View File

@@ -0,0 +1 @@
-- Add down migration script here

View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'flownode';

View File

@@ -0,0 +1,12 @@
-- for some managed dbs (e.g Azure Postgres) the role is not
-- automatically applied to the user when created
DO
$do$
BEGIN
GRANT windmill_user to CURRENT_USER;
GRANT windmill_admin to CURRENT_USER;
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'error granting windmill_user and windmill_admin to current_user: %', SQLERRM;
END
$do$;

View File

@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE IF EXISTS app_version_lite;
DROP TABLE IF EXISTS app_script;

View File

@@ -0,0 +1,26 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript';
-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`).
CREATE TABLE app_version_lite (
id BIGSERIAL PRIMARY KEY,
value JSONB,
FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE
);
GRANT ALL ON app_version_lite TO windmill_user;
GRANT ALL ON app_version_lite TO windmill_admin;
-- App `inlineScript`.
CREATE TABLE app_script (
id BIGSERIAL PRIMARY KEY,
app BIGSERIAL NOT NULL,
hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`.
lock TEXT,
code TEXT NOT NULL,
code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy.
FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE
);
GRANT ALL ON app_script TO windmill_user;
GRANT ALL ON app_script TO windmill_admin;

View File

@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE app DROP COLUMN custom_path;

View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TABLE app ADD COLUMN custom_path TEXT CHECK (custom_path ~ '^[\w-]+(\/[\w-]+)*$');

View File

@@ -0,0 +1 @@
-- Add down migration script here

View File

@@ -0,0 +1,3 @@
-- Add up migration script here
GRANT ALL ON concurrency_key TO windmill_admin;
GRANT ALL ON concurrency_key TO windmill_user;

View File

@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE flow_node DROP COLUMN hash_v2;

View File

@@ -0,0 +1,4 @@
-- Add up migration script here
CREATE SEQUENCE IF NOT EXISTS flow_node_hash_seq;
ALTER TABLE flow_node ALTER COLUMN hash DROP NOT NULL;
ALTER TABLE flow_node ADD COLUMN hash_v2 CHAR(64) NOT NULL UNIQUE DEFAULT to_hex(nextval('flow_node_hash_seq'));

View File

@@ -0,0 +1,3 @@
-- Add down migration script here
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2;
ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2);

View File

@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2);
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key;

View File

@@ -0,0 +1 @@
-- Add down migration script here

View File

@@ -0,0 +1,2 @@
-- Add up migration script here
delete from app_version_lite;

View File

@@ -172,5 +172,13 @@
"user-library-read"
]
},
"xero": {
"auth_url": "https://login.xero.com/identity/connect/authorize",
"token_url": "https://identity.xero.com/connect/token",
"scopes": [
"offline_access",
"accounting.transactions"
]
},
"snowflake_oauth": {}
}

View File

@@ -9,7 +9,7 @@ use regex_lite::Regex;
use serde_json::json;
use std::collections::HashMap;
use std::{collections::HashMap, str::FromStr};
use windmill_parser::{Arg, MainArgSignature, Typ};
pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
@@ -84,36 +84,94 @@ fn parse_bash_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
Ok(Some(args))
}
enum ParserState {
Normal,
InSingleQuote,
InDoubleQuote,
}
fn split_pwsh_args(code: &str) -> Vec<&str> {
let mut chars = code.char_indices().peekable();
let mut state = ParserState::Normal;
let mut splits = vec![];
let mut last_idx = 0;
while let Some((idx, char)) = chars.next() {
match (&state, char) {
(ParserState::Normal, '\'') => {
state = ParserState::InSingleQuote;
}
(ParserState::Normal, '"') => {
state = ParserState::InDoubleQuote;
}
(ParserState::InSingleQuote, '\'') => {
state = ParserState::Normal;
}
(ParserState::InDoubleQuote, '"') => {
state = ParserState::Normal;
}
(ParserState::Normal, ',') => {
splits.push(&code[last_idx..idx]);
last_idx = idx + 1; // skip the comma
}
_ => {}
}
}
if last_idx < code.len() {
splits.push(&code[last_idx..]);
}
splits
}
fn parse_powershell_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
let param_wrapper = RE_POWERSHELL_PARAM.captures(code);
let mut args = vec![];
if let Some(param_wrapper) = param_wrapper {
let param_wrapper = param_wrapper.get(1).unwrap().as_str();
for cap in RE_POWERSHELL_ARGS.captures_iter(param_wrapper) {
let typ = cap
.get(1)
.map(|x| x.as_str().to_string())
.unwrap_or("string".to_string());
let name = cap.get(2).unwrap().as_str().to_string();
let default = cap
.get(3)
.or(cap.get(4))
.map(|x| json!(x.as_str().to_string()));
let params = split_pwsh_args(param_wrapper);
for param in params {
if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) {
let typ = cap.get(1).map(|x| x.as_str().to_string());
let name = cap.get(2).unwrap().as_str().to_string();
args.push(Arg {
name: name,
typ: match typ.as_str() {
"string" => Typ::Str(None),
"int" | "long" => Typ::Int,
"decimal" | "double" | "single" => Typ::Float,
"datetime" | "DateTime" => Typ::Datetime,
_ => Typ::Str(None),
},
default: default.clone(),
otyp: None,
has_default: default.is_some(),
oidx: None,
});
let mut parsed_typ = if let Some(typ) = typ {
match typ.as_str() {
"string" => Some(Typ::Str(None)),
"int" | "long" => Some(Typ::Int),
"decimal" | "double" | "single" => Some(Typ::Float),
"datetime" | "DateTime" => Some(Typ::Datetime),
_ => None,
}
} else {
None
};
let default = if let Some(x) = cap.get(3) {
Some(json!(x.as_str().to_string()))
} else if let Some(x) = cap.get(4) {
if parsed_typ.is_none() {
if x.as_str().parse::<i64>().is_ok() {
parsed_typ = Some(Typ::Int);
} else if x.as_str().parse::<f64>().is_ok() {
parsed_typ = Some(Typ::Float);
}
}
serde_json::Number::from_str(x.as_str())
.ok()
.map(serde_json::Value::Number)
} else {
None
};
args.push(Arg {
name: name,
typ: parsed_typ.unwrap_or(Typ::Str(None)),
default: default.clone(),
otyp: None,
has_default: default.is_some(),
oidx: None,
});
}
}
}
Ok(Some(args))
@@ -192,4 +250,77 @@ non_required="${5:-}"
Ok(())
}
#[test]
fn test_parse_powershell_sig() -> anyhow::Result<()> {
let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO)"#;
assert_eq!(
parse_powershell_sig(code)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "Msg".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "Msg2".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "Dflt".to_string(),
typ: Typ::Str(None),
default: Some(json!("default value, with comma")),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb".to_string(),
typ: Typ::Int,
default: Some(json!(3)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb2".to_string(),
typ: Typ::Float,
default: Some(json!(5.0)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb3".to_string(),
typ: Typ::Int,
default: Some(json!(5)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Wahoo".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
}
],
no_main_func: None,
has_preprocessor: None
}
);
Ok(())
}
}

View File

@@ -8,7 +8,9 @@
use anyhow::Context;
use monitor::{
reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store
load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
};
use rand::Rng;
use sqlx::{postgres::PgListener, Pool, Postgres};
@@ -29,7 +31,15 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE
use windmill_common::{
global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
scripts::ScriptLang,
stats_ee::schedule_stats,
@@ -58,7 +68,8 @@ use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, TMP_LOGS_DIR,
UV_CACHE_DIR,
};
use crate::monitor::{
@@ -210,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> {
let hostname = hostname();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname);
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
println!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
println!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
println!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
eprintln!("mode not recognized, defaulting to standalone: {x}");
} else {
println!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
tracing::info!("jemalloc enabled");
println!("jemalloc enabled");
#[cfg(feature = "flamegraph")]
let _guard = windmill_common::tracing_init::setup_flamegraph();
@@ -225,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> {
"cache" => {
#[cfg(feature = "embedding")]
{
tracing::info!("Caching embedding model...");
println!("Caching embedding model...");
windmill_api::embeddings::ModelInstance::load_model_files().await?;
tracing::info!("Cached embedding model");
println!("Cached embedding model");
}
#[cfg(not(feature = "embedding"))]
{
tracing::warn!("Embeddings are not enabled, ignoring...");
println!("Embeddings are not enabled, ignoring...");
}
cache_hub_scripts(std::env::args().nth(2)).await?;
@@ -245,60 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> {
_ => {}
}
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
tracing::info!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
Mode::Worker
} else if &x == "agent" {
tracing::info!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
tracing::info!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
tracing::error!("mode not recognized, defaulting to standalone: {x}");
} else {
tracing::info!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[allow(unused_mut)]
let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
0
@@ -310,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> {
};
if num_workers > 1 {
tracing::warn!(
println!(
"We STRONGLY recommend using at most 1 worker per container, use at your own risks"
);
}
@@ -332,31 +344,26 @@ async fn windmill_main() -> anyhow::Result<()> {
IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
};
let rsmq_config = std::env::var("REDIS_URL").ok().map(|x| {
let url = x.parse::<url::Url>().unwrap();
let mut config = rsmq_async::RsmqOptions { ..Default::default() };
config.host = url.host_str().expect("redis host required").to_owned();
config.password = url.password().map(|s| s.to_owned());
config.db = url
.path_segments()
.and_then(|mut segments| segments.next())
.and_then(|segment| segment.parse().ok())
.unwrap_or(0);
config.ns = url
.query_pairs()
.find(|s| s.0 == "rsmq_namespace")
.map(|s| s.1)
.unwrap_or(std::borrow::Cow::Borrowed("rsmq"))
.into_owned();
config.port = url.port().unwrap_or(6379).to_string();
config
});
tracing::info!("Connecting to database...");
println!("Connecting to database...");
let db = windmill_common::connect_db(server_mode, indexer_mode).await?;
load_otel(&db).await;
tracing::info!("Database connected");
let environment = load_base_url(&db)
.await
.unwrap_or_else(|_| "local".to_string())
.trim_start_matches("https://")
.trim_start_matches("http://")
.split(".")
.next()
.unwrap_or_else(|| "local")
.to_string();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);
let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
tracing::info!(
@@ -367,13 +374,6 @@ async fn windmill_main() -> anyhow::Result<()> {
.unwrap_or_else(|| "UNKNOWN".to_string())
);
let rsmq = if let Some(config) = rsmq_config {
tracing::info!("Redis config set: {:?}", config);
Some(rsmq_async::MultiplexedRsmq::new(config).await.unwrap())
} else {
None
};
let is_agent = mode == Mode::Agent;
if !is_agent {
@@ -434,7 +434,7 @@ Windmill Community Edition {GIT_VERSION}
}
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key && !server_mode {
panic!("Invalid license key, workers require a valid license key");
tracing::error!("Invalid license key, workers require a valid license key");
}
if server_mode {
// only force renewal if invalid but not empty (= expired)
@@ -449,13 +449,6 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!("Failed to reload license key: {err:#}");
}
}
if num_workers > 0 {
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key {
tracing::warn!("License key invalid, setting num_workers to 0");
num_workers = 0;
}
}
}
}
@@ -488,7 +481,6 @@ Windmill Community Edition {GIT_VERSION}
monitor_db(
&db,
&base_internal_url,
rsmq.clone(),
server_mode,
worker_mode,
true,
@@ -507,7 +499,6 @@ Windmill Community Edition {GIT_VERSION}
let addr = SocketAddr::from((server_bind_address, port));
let rsmq2 = rsmq.clone();
let (base_internal_tx, base_internal_rx) = tokio::sync::oneshot::channel::<String>();
DirBuilder::new()
@@ -524,8 +515,21 @@ Windmill Community Edition {GIT_VERSION}
#[cfg(feature = "tantivy")]
let (index_reader, index_writer) = if should_index_jobs {
let (r, w) = windmill_indexer::completed_runs_ee::init_index(&db).await?;
(Some(r), Some(w))
let mut indexer_rx = killpill_rx.resubscribe();
let (mut reader, mut writer) = (None, None);
tokio::select! {
_ = indexer_rx.recv() => {
tracing::info!("Received killpill, aborting index initialization");
},
res = windmill_indexer::completed_runs_ee::init_index(&db) => {
let res = res?;
reader = Some(res.0);
writer = Some(res.1);
}
}
(reader, writer)
} else {
(None, None)
};
@@ -549,8 +553,21 @@ Windmill Community Edition {GIT_VERSION}
#[cfg(all(feature = "tantivy", feature = "parquet"))]
let (log_index_reader, log_index_writer) = if should_index_jobs {
let (r, w) = windmill_indexer::service_logs_ee::init_index(&db).await?;
(Some(r), Some(w))
let mut indexer_rx = killpill_rx.resubscribe();
let (mut reader, mut writer) = (None, None);
tokio::select! {
_ = indexer_rx.recv() => {
tracing::info!("Received killpill, aborting index initialization");
},
res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => {
let res = res?;
reader = Some(res.0);
writer = Some(res.1);
}
}
(reader, writer)
} else {
(None, None)
};
@@ -588,7 +605,6 @@ Windmill Community Edition {GIT_VERSION}
if !is_agent {
windmill_api::run_server(
db.clone(),
rsmq2,
index_reader,
log_index_reader,
addr,
@@ -620,7 +636,6 @@ Windmill Community Edition {GIT_VERSION}
killpill_tx.clone(),
num_workers,
base_internal_url.clone(),
rsmq.clone(),
mode.clone() == Mode::Agent,
hostname.clone(),
)
@@ -636,13 +651,12 @@ Windmill Community Edition {GIT_VERSION}
killpill_phase2_tx.send(())?;
tracing::info!("Phase 2 of shutdown completed");
}
Ok(()) as anyhow::Result<()>
Ok(())
};
let monitor_f = async {
let db = db.clone();
let tx = killpill_tx.clone();
let rsmq = rsmq.clone();
let base_internal_url = base_internal_url.to_string();
let h = tokio::spawn(async move {
@@ -659,7 +673,6 @@ Windmill Community Edition {GIT_VERSION}
monitor_db(
&db,
&base_internal_url,
rsmq.clone(),
server_mode,
worker_mode,
false,
@@ -708,14 +721,6 @@ Windmill Community Edition {GIT_VERSION}
if let Err(e) = reload_license_key(&db).await {
tracing::error!("Failed to reload license key: {e:#}");
}
#[cfg(feature = "enterprise")]
if worker_mode {
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key {
tracing::error!("Invalid license key, exiting...");
tx.send(()).expect("send");
}
}
},
DEFAULT_TAGS_PER_WORKSPACE_SETTING => {
if let Err(e) = load_tag_per_workspace_enabled(&db).await {
@@ -739,6 +744,9 @@ Windmill Community Edition {GIT_VERSION}
RETENTION_PERIOD_SECS_SETTING => {
reload_retention_period_setting(&db).await
},
MONITOR_LOGS_ON_OBJECT_STORE_SETTING => {
reload_delete_logs_periodically_setting(&db).await
},
JOB_DEFAULT_TIMEOUT_SECS_SETTING => {
reload_job_default_timeout_setting(&db).await
},
@@ -781,6 +789,15 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
OTEL_SETTING => {
tracing::info!("OTEL setting changed, restarting");
// we wait a bit randomly to avoid having all servers and workers shutdown at same time
let rd_delay = rand::thread_rng().gen_range(0..4);
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
if let Err(e) = tx.send(()) {
tracing::error!(error = %e, "Could not send killpill");
}
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
@@ -950,13 +967,12 @@ fn display_config(envs: &[&str]) {
)
}
pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
pub async fn run_workers(
db: Pool<Postgres>,
mut rx: tokio::sync::broadcast::Receiver<()>,
tx: tokio::sync::broadcast::Sender<()>,
num_workers: i32,
base_internal_url: String,
rsmq: Option<R>,
agent_mode: bool,
hostname: String,
) -> anyhow::Result<()> {
@@ -995,6 +1011,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
TMP_LOGS_DIR,
UV_CACHE_DIR,
TAR_PIP_CACHE_DIR,
TAR_PY311_CACHE_DIR,
DENO_CACHE_DIR,
DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM,
@@ -1028,7 +1045,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
let rx = killpill_rxs.pop().unwrap();
let tx = tx.clone();
let base_internal_url = base_internal_url.clone();
let rsmq2 = rsmq.clone();
let hostname = hostname.clone();
handles.push(tokio::spawn(async move {
@@ -1046,7 +1062,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
rx,
tx,
&base_internal_url,
rsmq2,
agent_mode,
);

View File

@@ -11,8 +11,8 @@ use std::{
};
use chrono::{NaiveDateTime, Utc};
use rsmq_async::MultiplexedRsmq;
use serde::de::DeserializeOwned;
use futures::{stream::FuturesUnordered, StreamExt};
use serde::{de::DeserializeOwned, Deserializer};
use sqlx::{Pool, Postgres};
use tokio::{
join,
@@ -27,7 +27,7 @@ use windmill_api::{
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
};
#[cfg(feature = "enterprise")]
use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts, LICENSE_KEY_VALID};
use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
use windmill_common::{
auth::JWT_SECRET,
ee::CriticalErrorChannel,
@@ -38,8 +38,9 @@ use windmill_common::{
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -53,10 +54,12 @@ use windmill_common::{
worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG,
SMTP_CONFIG, WORKER_CONFIG, WORKER_GROUP,
SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP,
},
BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -83,12 +86,12 @@ lazy_static::lazy_static! {
static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
.ok()
.and_then(|x| x.parse::<String>().ok())
.unwrap_or_else(|| "30".to_string());
.unwrap_or_else(|| "60".to_string());
static ref FLOW_ZOMBIE_TRANSITION_TIMEOUT: String = std::env::var("FLOW_ZOMBIE_TRANSITION_TIMEOUT")
.ok()
.and_then(|x| x.parse::<String>().ok())
.unwrap_or_else(|| "30".to_string());
.unwrap_or_else(|| "60".to_string());
pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
@@ -180,6 +183,7 @@ pub async fn initial_load(
}
if worker_mode {
reload_job_default_timeout_setting(&db).await;
reload_extra_pip_index_url_setting(&db).await;
reload_pip_index_url_setting(&db).await;
reload_npm_config_registry_setting(&db).await;
@@ -196,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
Ok(())
}
fn empty_string_as_none<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
#[derive(serde::Deserialize)]
struct OtelSetting {
metrics_enabled: Option<bool>,
logs_enabled: Option<bool>,
tracing_enabled: Option<bool>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_endpoint: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_headers: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_protocol: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_compression: Option<String>,
}
pub async fn load_otel(db: &DB) {
let otel = load_value_from_global_settings(db, OTEL_SETTING).await;
if let Ok(v) = otel {
if let Some(v) = v {
let deser = serde_json::from_value::<OtelSetting>(v);
if let Ok(o) = deser {
let metrics_enabled = o.metrics_enabled.unwrap_or(false);
let logs_enabled = o.logs_enabled.unwrap_or(false);
let tracing_enabled = o.tracing_enabled.unwrap_or(false);
OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed);
OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed);
OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed);
if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
}
if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers);
}
if let Some(protocol) = o.otel_exporter_otlp_protocol {
std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
}
if let Some(compression) = o.otel_exporter_otlp_compression {
std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression);
}
println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})",
o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some());
} else {
tracing::error!("Error deserializing otel settings");
}
}
} else {
tracing::error!("Error loading otel settings: {}", otel.unwrap_err());
}
}
pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled =
load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await;
@@ -568,6 +631,11 @@ pub async fn load_require_preexisting_user(db: &DB) {
};
}
struct LogFile {
file_path: String,
hostname: String,
}
pub async fn delete_expired_items(db: &DB) -> () {
let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
"DELETE FROM token WHERE expiration <= now()
@@ -630,6 +698,25 @@ pub async fn delete_expired_items(db: &DB) -> () {
Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
}
match sqlx::query_as!(
LogFile,
"DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname",
SERVICE_LOG_RETENTION_SECS,
)
.fetch_all(db)
.await
{
Ok(log_files_to_delete) => {
let paths = log_files_to_delete
.iter()
.map(|f| format!("{}/{}", f.hostname, f.file_path))
.collect();
delete_log_files_from_disk_and_store(paths, TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await;
}
Err(e) => tracing::error!("Error deleting log file: {:?}", e),
}
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
if job_retention_secs > 0 {
match db.begin().await {
@@ -659,14 +746,22 @@ pub async fn delete_expired_items(db: &DB) -> () {
{
tracing::error!("Error deleting job stats: {:?}", e);
}
if let Err(e) = sqlx::query!(
"DELETE FROM job_logs WHERE job_id = ANY($1)",
match sqlx::query_scalar!(
"DELETE FROM job_logs WHERE job_id = ANY($1) RETURNING log_file_index",
&deleted_jobs
)
.execute(&mut *tx)
.fetch_all(&mut *tx)
.await
{
tracing::error!("Error deleting job stats: {:?}", e);
Ok(log_file_index) => {
let paths = log_file_index
.into_iter()
.filter_map(|opt| opt)
.flat_map(|inner_vec| inner_vec.into_iter())
.collect();
delete_log_files_from_disk_and_store(paths, TMP_DIR, "").await;
}
Err(e) => tracing::error!("Error deleting job stats: {:?}", e),
}
if let Err(e) = sqlx::query!(
"DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval ",
@@ -677,21 +772,11 @@ pub async fn delete_expired_items(db: &DB) -> () {
{
tracing::error!("Error deleting custom concurrency key: {:?}", e);
}
if let Err(e) = sqlx::query!(
"DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
job_retention_secs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting log file: {:?}", e);
}
if let Err(e) = sqlx::query!(
"DELETE FROM job WHERE id = ANY($1)",
&deleted_jobs
)
.execute(&mut *tx)
.await
if let Err(e) =
sqlx::query!("DELETE FROM job WHERE id = ANY($1)", &deleted_jobs)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job: {:?}", e);
}
@@ -714,6 +799,59 @@ pub async fn delete_expired_items(db: &DB) -> () {
}
}
async fn delete_log_files_from_disk_and_store(
paths_to_delete: Vec<String>,
tmp_dir: &str,
_s3_prefix: &str,
) {
#[cfg(feature = "parquet")]
let os = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
#[cfg(not(feature = "parquet"))]
let os: Option<()> = None;
let _should_del_from_store = MONITOR_LOGS_ON_OBJECT_STORE.read().await.clone();
let delete_futures = FuturesUnordered::new();
for path in paths_to_delete {
let _os2 = &os;
delete_futures.push(async move {
let disk_path = std::path::Path::new(tmp_dir).join(&path);
if tokio::fs::metadata(&disk_path).await.is_ok() {
if let Err(e) = tokio::fs::remove_file(&disk_path).await {
tracing::error!(
"Failed to delete from disk {}: {e}",
disk_path.to_string_lossy()
);
} else {
tracing::debug!(
"Succesfully deleted {} from disk",
disk_path.to_string_lossy()
);
}
}
#[cfg(feature = "parquet")]
if _should_del_from_store {
if let Some(os) = _os2 {
let p = object_store::path::Path::from(format!("{}{}", _s3_prefix, path));
if let Err(e) = os.delete(&p).await {
tracing::error!("Failed to delete from object store {}: {e}", p.to_string())
} else {
tracing::debug!("Succesfully deleted {} from object store", p.to_string());
}
}
}
});
}
let _: Vec<_> = delete_futures.collect().await;
}
pub async fn reload_scim_token_setting(db: &DB) {
reload_option_setting_with_tracing(db, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone())
.await;
@@ -792,6 +930,20 @@ pub async fn reload_retention_period_setting(db: &DB) {
tracing::error!("Error reloading retention period: {:?}", e)
}
}
pub async fn reload_delete_logs_periodically_setting(db: &DB) {
if let Err(e) = reload_setting(
db,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
"MONITOR_LOGS_ON_OBJECT_STORE",
false,
MONITOR_LOGS_ON_OBJECT_STORE.clone(),
|x| x,
)
.await
{
tracing::error!("Error reloading retention period: {:?}", e)
}
}
#[cfg(feature = "parquet")]
pub async fn reload_s3_cache_setting(db: &DB) {
@@ -1031,7 +1183,6 @@ pub async fn monitor_pool(db: &DB) {
pub async fn monitor_db(
db: &Pool<Postgres>,
base_internal_url: &str,
rsmq: Option<MultiplexedRsmq>,
server_mode: bool,
_worker_mode: bool,
initial_load: bool,
@@ -1039,8 +1190,8 @@ pub async fn monitor_db(
) {
let zombie_jobs_f = async {
if server_mode && !initial_load {
handle_zombie_jobs(db, base_internal_url, rsmq.clone(), "server").await;
match handle_zombie_flows(db, rsmq.clone()).await {
handle_zombie_jobs(db, base_internal_url, "server").await;
match handle_zombie_flows(db).await {
Err(err) => {
tracing::error!("Error handling zombie flows: {:?}", err);
}
@@ -1058,13 +1209,6 @@ pub async fn monitor_db(
#[cfg(feature = "enterprise")]
if !initial_load {
verify_license_key().await;
if _worker_mode {
let valid_key = *LICENSE_KEY_VALID.read().await;
if !valid_key {
tracing::error!("Invalid license key, exiting...");
_killpill_tx.send(()).expect("send");
}
}
}
};
@@ -1253,7 +1397,7 @@ pub async fn reload_worker_config(
}
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
pub async fn load_base_url(db: &DB) -> error::Result<String> {
let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?;
let std_base_url = std::env::var("BASE_URL")
@@ -1277,6 +1421,14 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
std_base_url
};
{
let mut l = BASE_URL.write().await;
*l = base_url.clone();
}
Ok(base_url)
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
let oauths = if let Some(q) = q_oauth {
@@ -1290,6 +1442,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
None
};
let base_url = load_base_url(db).await?;
let is_secure = base_url.starts_with("https://");
{
@@ -1299,11 +1452,6 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
.unwrap();
}
{
let mut l = BASE_URL.write().await;
*l = base_url
}
{
let mut l = IS_SECURE.write().await;
*l = is_secure;
@@ -1312,12 +1460,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
Ok(())
}
async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
db: &Pool<Postgres>,
base_internal_url: &str,
rsmq: Option<R>,
worker_name: &str,
) {
async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker_name: &str) {
if *RESTART_ZOMBIE_JOBS {
let restarted = sqlx::query!(
"UPDATE queue SET running = false, started_at = null
@@ -1426,7 +1569,6 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
true,
same_worker_tx_never_used,
"",
rsmq.clone(),
worker_name,
send_result_never_used,
#[cfg(feature = "benchmark")]
@@ -1436,10 +1578,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
}
}
async fn handle_zombie_flows(
db: &DB,
rsmq: Option<rsmq_async::MultiplexedRsmq>,
) -> error::Result<()> {
async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
let flows = sqlx::query_as::<_, QueuedJob>(
r#"
SELECT *
@@ -1486,7 +1625,7 @@ async fn handle_zombie_flows(
}
);
report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
cancel_zombie_flow_job(db, flow, &rsmq, reason).await?;
cancel_zombie_flow_job(db, flow, reason).await?;
}
}
@@ -1516,7 +1655,7 @@ async fn handle_zombie_flows(
job.workspace_id,
flow.last_ping
);
cancel_zombie_flow_job(db, job, &rsmq,
cancel_zombie_flow_job(db, job,
format!("Flow {} cancelled as one of the parallel branch {} was unable to make the last transition ", flow.parent_flow_id, flow.job_id))
.await?;
} else {
@@ -1529,7 +1668,6 @@ async fn handle_zombie_flows(
async fn cancel_zombie_flow_job(
db: &Pool<Postgres>,
flow: QueuedJob,
rsmq: &Option<MultiplexedRsmq>,
message: String,
) -> Result<(), error::Error> {
let tx = db.begin().await.unwrap();
@@ -1545,7 +1683,6 @@ async fn cancel_zombie_flow_job(
flow.workspace_id.as_str(),
tx,
db,
rsmq.clone(),
true,
false,
)

View File

@@ -77,7 +77,11 @@ async fn initialize_tracing() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let _ = windmill_common::tracing_init::initialize_tracing("test");
let _ = windmill_common::tracing_init::initialize_tracing(
"test",
&windmill_common::utils::Mode::Standalone,
"test",
);
});
}
@@ -126,7 +130,6 @@ impl ApiServer {
db.clone(),
None,
None,
None,
addr,
rx,
port_tx,
@@ -899,8 +902,8 @@ impl RunJob {
hm_args.insert(k, windmill_common::worker::to_raw_value(&v));
}
let tx = PushIsolationLevel::IsolatedRoot(db.clone(), None);
let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
let (uuid, tx) = windmill_queue::push(
&db,
tx,
"test-workspace",
@@ -1018,7 +1021,7 @@ fn spawn_test_worker(
windmill_common::worker::make_suspended_pull_query(&wc).await;
windmill_common::worker::make_pull_query(&wc).await;
}
windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
windmill_worker::run_worker(
&db,
worker_instance,
worker_name,
@@ -1028,7 +1031,6 @@ fn spawn_test_worker(
rx,
tx2,
&base_internal_url,
None,
false,
)
.await
@@ -1172,6 +1174,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
continue_on_error: None,
skip_if: None,
}],
modules_node: None,
}
.into(),
stop_after_if: Default::default(),
@@ -1388,6 +1391,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
skip_if: None,
},
],
modules_node: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),

View File

@@ -77,7 +77,6 @@ async-stripe = { workspace = true, optional = true }
lazy_static.workspace = true
prometheus = { workspace = true, optional = true }
async_zip.workspace = true
rsmq_async.workspace = true
regex.workspace = true
bytes.workspace = true
samael = { workspace = true, optional = true }
@@ -98,6 +97,7 @@ jsonwebtoken = { workspace = true }
matchit.workspace = true
tokio-tungstenite.workspace = true
rdkafka = { workspace = true, optional = true }
const_format.workspace = true
pin-project.workspace = true
http.workspace = true

View File

@@ -8709,6 +8709,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
@@ -9267,6 +9270,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.430.1
version: 1.437.1
title: Windmill API
contact:
@@ -872,9 +872,20 @@ paths:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
type: object
properties:
alerts:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
total_rows:
type: integer
description: Total number of rows matching the query.
example: 100
total_pages:
type: integer
description: Total number of pages based on the page size.
example: 10
/settings/critical_alerts/{id}/acknowledge:
post:
@@ -2757,9 +2768,20 @@ paths:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
type: object
properties:
alerts:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
total_rows:
type: integer
description: Total number of rows matching the query.
example: 100
total_pages:
type: integer
description: Total number of pages based on the page size.
example: 10
/w/{workspace}/workspaces/critical_alerts/{id}/acknowledge:
post:
@@ -3733,6 +3755,27 @@ paths:
required:
- app
/apps_u/public_app_by_custom_path/{custom_path}:
get:
summary: get public app by custom path
operationId: getPublicAppByCustomPath
tags:
- app
parameters:
- $ref: "#/components/parameters/CustomPath"
responses:
"200":
description: app details
content:
application/json:
schema:
allOf:
- $ref: "#/components/schemas/AppWithLastVersion"
- type: object
properties:
workspace_id:
type: string
/scripts/hub/get/{path}:
get:
summary: get hub script content by path
@@ -5371,6 +5414,8 @@ paths:
type: boolean
deployment_message:
type: string
custom_path:
type: string
required:
- path
- value
@@ -5422,6 +5467,23 @@ paths:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/lite/{path}:
get:
summary: get app lite by path
operationId: getAppLiteByPath
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: app lite details
content:
application/json:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/draft/{path}:
get:
summary: get app by path with draft
@@ -5696,6 +5758,8 @@ paths:
$ref: "#/components/schemas/Policy"
deployment_message:
type: string
custom_path:
type: string
responses:
"200":
description: app updated
@@ -5704,6 +5768,23 @@ paths:
schema:
type: string
/w/{workspace}/apps/custom_path_exists/{custom_path}:
get:
summary: check if custom path exists
operationId: customPathExists
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/CustomPath"
responses:
"200":
description: custom path exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/apps_u/execute_component/{path}:
post:
summary: executeComponent
@@ -5727,6 +5808,8 @@ paths:
#flow: flow/<path>
path:
type: string
version:
type: integer
args: {}
raw_code:
type: object
@@ -5744,6 +5827,8 @@ paths:
required:
- content
- language
id:
type: integer
force_viewer_static_fields:
type: object
force_viewer_one_of_fields:
@@ -6852,6 +6937,43 @@ paths:
- resume
- cancel
/w/{workspace}/jobs/slack_approval/{id}/{resume_id}:
get:
summary: get interactive slack approval payload given the job_id, resume_id and a nonce to resume a flow
operationId: getSlackApprovalPayload
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/JobId"
- name: resume_id
in: path
required: true
schema:
type: integer
- name: approver
in: query
schema:
type: string
- name: message
in: query
schema:
type: string
responses:
"200":
description: Blocks array for posting interactive slack approval
content:
application/json:
schema:
type: object
properties:
blocks:
type: array
items:
type: object
required:
- blocks
/w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}:
get:
summary: resume a job for a suspended flow
@@ -10098,11 +10220,6 @@ paths:
required: true
schema:
type: string
- name: hosts
in: query
required: true
schema:
type: string
- name: min_ts
in: query
required: false
@@ -10132,6 +10249,29 @@ paths:
description: count of log lines that matched the query per hostname
type: object
/srch/index/delete/{idx_name}:
delete:
summary: Restart container and delete the index to recreate it.
operationId: clearIndex
tags:
- indexSearch
parameters:
- name: idx_name
in: path
required: true
schema:
type: string
enum:
- JobIndex
- ServiceLogIndex
responses:
"200":
description: idx to be deleted and container restarting
content:
text/plain:
schema:
type: string
components:
securitySchemes:
bearerAuth:
@@ -10204,6 +10344,12 @@ components:
required: true
schema:
type: string
CustomPath:
name: custom_path
in: path
required: true
schema:
type: string
PathId:
name: id
in: path
@@ -10873,6 +11019,9 @@ components:
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -10992,6 +11141,9 @@ components:
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -11101,6 +11253,8 @@ components:
type: string
is_admin:
type: boolean
name:
type: string
is_super_admin:
type: boolean
created_at:
@@ -12840,6 +12994,8 @@ components:
draft_only:
type: boolean
draft: {}
custom_path:
type: string
AppHistory:
type: object

View File

@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
/*
* Author: Ruben Fiszel
@@ -31,6 +31,7 @@ use axum::{
routing::{delete, get, post},
Router,
};
use futures::future::{FutureExt, TryFutureExt};
use hyper::StatusCode;
#[cfg(feature = "parquet")]
use itertools::Itertools;
@@ -50,27 +51,30 @@ use windmill_audit::ActionKind;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::{
apps::ListAppQuery,
apps::{AppScriptId, ListAppQuery},
cache::{self, future::FutureCachedExt},
db::UserDB,
error::{to_anyhow, Error, JsonResult, Result},
jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
users::username_to_permissioned_as,
utils::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
Pagination, StripPath,
},
variables::{build_crypt, build_crypt_with_key_suffix},
worker::to_raw_value,
worker::{to_raw_value, CLOUD_HOSTED},
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel, QueueTransaction};
use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_apps))
.route("/list_search", get(list_search_apps))
.route("/get/p/*path", get(get_app))
.route("/get/lite/*path", get(get_app_lite))
.route("/get/draft/*path", get(get_app_w_draft))
.route("/secret_of/*path", get(get_secret_id))
.route("/get/v/*id", get(get_app_by_id))
@@ -81,6 +85,7 @@ pub fn workspaced_service() -> Router {
.route("/history/p/*path", get(get_app_history))
.route("/get_latest_version/*path", get(get_latest_version))
.route("/history_update/a/:id/v/:version", post(update_app_history))
.route("/custom_path_exists/*custom_path", get(custom_path_exists))
}
pub fn unauthed_service() -> Router {
@@ -90,13 +95,17 @@ pub fn unauthed_service() -> Router {
.route("/public_app/:secret", get(get_public_app_by_secret))
.route("/public_resource/*path", get(get_public_resource))
}
pub fn global_service() -> Router {
Router::new()
.route("/hub/list", get(list_hub_apps))
.route("/hub/get/:id", get(get_hub_app_by_id))
}
#[cfg(not(feature = "enterprise"))]
pub fn global_unauthed_service() -> Router {
Router::new()
}
#[derive(FromRow, Deserialize, Serialize)]
pub struct ListableApp {
pub id: i64,
@@ -147,21 +156,26 @@ pub struct AppWithLastVersionAndStarred {
pub starred: Option<bool>,
}
#[cfg(feature = "enterprise")]
#[derive(Serialize, FromRow)]
pub struct AppWithLastVersionAndWorkspace {
#[sqlx(flatten)]
#[serde(flatten)]
pub app: AppWithLastVersion,
pub workspace_id: String,
}
#[derive(Serialize, Deserialize, FromRow)]
pub struct AppWithLastVersionAndDraft {
pub id: i64,
pub path: String,
pub summary: String,
pub policy: sqlx::types::Json<Box<RawValue>>,
pub versions: Vec<i64>,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: serde_json::Value,
#[sqlx(flatten)]
#[serde(flatten)]
pub app: AppWithLastVersion,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<sqlx::types::Json<Box<RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub custom_path: Option<String>,
}
#[derive(Serialize)]
@@ -181,15 +195,16 @@ pub type StaticFields = HashMap<String, Box<RawValue>>;
pub type OneOfFields = HashMap<String, Vec<Box<RawValue>>>;
pub type AllowUserResources = Vec<String>;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
#[serde(rename_all = "lowercase")]
pub enum ExecutionMode {
#[default]
Anonymous,
Publisher,
Viewer,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct PolicyTriggerableInputs {
static_inputs: StaticFields,
one_of_inputs: OneOfFields,
@@ -205,7 +220,7 @@ pub struct S3Input {
file_key_regex: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct Policy {
pub on_behalf_of: Option<String>,
pub on_behalf_of_email: Option<String>,
@@ -229,6 +244,7 @@ pub struct CreateApp {
pub policy: Policy,
pub draft_only: Option<bool>,
pub deployment_message: Option<String>,
pub custom_path: Option<String>,
}
#[derive(Deserialize)]
@@ -238,6 +254,7 @@ pub struct EditApp {
pub value: Option<sqlx::types::Json<Box<RawValue>>>,
pub policy: Option<Policy>,
pub deployment_message: Option<String>,
pub custom_path: Option<String>,
}
#[derive(Serialize, FromRow)]
@@ -398,6 +415,33 @@ async fn get_app(
Ok(Json(app))
}
async fn get_app_lite(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<AppWithLastVersion> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value,
app_version.created_at, app_version.created_by, NULL as starred
FROM app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let app = not_found_if_none(app_o, "App", path)?;
Ok(Json(app))
}
async fn get_app_w_draft(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -408,7 +452,7 @@ async fn get_app_w_draft(
let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>(
r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app.extra_perms, app_version.value, app.custom_path,
app_version.created_at, app_version.created_by,
app.draft_only, draft.value as "draft"
from app
@@ -515,6 +559,21 @@ async fn update_app_history(
return Ok(());
}
async fn custom_path_exists(
Extension(db): Extension<DB>,
Path((w_id, custom_path)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists =
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
custom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None }
)
.fetch_one(&db)
.await?.unwrap_or(false);
Ok(Json(exists))
}
async fn get_app_by_id(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -555,8 +614,9 @@ async fn get_public_app_by_secret(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
null as extra_perms, app_version.value,
null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
app_version.created_at, app_version.created_by from app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
.bind(&w_id)
@@ -651,12 +711,11 @@ async fn create_app(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(webhook): Extension<WebhookShared>,
Path(w_id): Path<String>,
Json(mut app): Json<CreateApp>,
) -> Result<(StatusCode, String)> {
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
let mut tx = user_db.clone().begin(&authed).await?;
app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
app.policy.on_behalf_of_email = Some(authed.email.clone());
@@ -670,7 +729,7 @@ async fn create_app(
&app.path,
w_id
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
@@ -681,25 +740,47 @@ async fn create_app(
)));
}
if let Some(custom_path) = &app.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
custom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None }
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
custom_path
)));
}
}
sqlx::query!(
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
&app.path,
&w_id
)
.execute(&mut tx)
.execute(&mut *tx)
.await?;
let id = sqlx::query_scalar!(
"INSERT INTO app
(workspace_id, path, summary, policy, versions, draft_only)
VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
(workspace_id, path, summary, policy, versions, draft_only, custom_path)
VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id",
w_id,
app.path,
app.summary,
json!(app.policy),
app.draft_only,
app.custom_path
.map(|s| if s.is_empty() { None } else { Some(s) })
.flatten()
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?;
let v_id = sqlx::query_scalar!(
@@ -711,7 +792,7 @@ async fn create_app(
serde_json::to_string(&app.value).unwrap(),
authed.username,
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?;
sqlx::query!(
@@ -719,11 +800,11 @@ async fn create_app(
v_id,
id
)
.execute(&mut tx)
.execute(&mut *tx)
.await?;
audit_log(
&mut tx,
&mut *tx,
&authed,
"apps.create",
ActionKind::Create,
@@ -809,7 +890,6 @@ async fn delete_app(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(webhook): Extension<WebhookShared>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
@@ -862,7 +942,6 @@ async fn delete_app(
version: 0, // dummy version as it will not get inserted in db
},
Some(format!("App '{}' deleted", path)),
rsmq,
true,
)
.await?;
@@ -893,7 +972,6 @@ async fn update_app(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(webhook): Extension<WebhookShared>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(ns): Json<EditApp>,
) -> Result<String> {
@@ -901,9 +979,13 @@ async fn update_app(
let path = path.to_path();
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
let mut tx = user_db.clone().begin(&authed).await?;
let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() {
let npath = if ns.policy.is_some()
|| ns.path.is_some()
|| ns.summary.is_some()
|| ns.custom_path.is_some()
{
let mut sqlb = SqlBuilder::update_table("app");
sqlb.and_where_eq("path", "?".bind(&path));
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
@@ -918,7 +1000,7 @@ async fn update_app(
npath,
w_id
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
@@ -936,6 +1018,32 @@ async fn update_app(
sqlb.set_str("summary", nsummary);
}
if let Some(ncustom_path) = &ns.custom_path {
require_admin(authed.is_admin, &authed.username)?;
if ncustom_path.is_empty() {
sqlb.set("custom_path", "NULL");
} else {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
ncustom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None },
path,
w_id
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
ncustom_path
)));
}
sqlb.set_str("custom_path", ncustom_path);
}
}
if let Some(mut npolicy) = ns.policy {
npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
npolicy.on_behalf_of_email = Some(authed.email.clone());
@@ -950,7 +1058,7 @@ async fn update_app(
sqlb.returning("path");
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?;
not_found_if_none(npath_o, "App", path)?
} else {
path.to_owned()
@@ -961,7 +1069,7 @@ async fn update_app(
npath,
w_id
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?;
let v_id = sqlx::query_scalar!(
@@ -973,7 +1081,7 @@ async fn update_app(
serde_json::to_string(&nvalue).unwrap(),
authed.username,
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?;
sqlx::query!(
@@ -982,7 +1090,7 @@ async fn update_app(
npath,
w_id
)
.execute(&mut tx)
.execute(&mut *tx)
.await?;
v_id
} else {
@@ -991,7 +1099,7 @@ async fn update_app(
npath,
w_id
)
.fetch_one(&mut tx)
.fetch_one(&mut *tx)
.await?;
if let Some(v_id) = v_id {
v_id
@@ -1008,11 +1116,11 @@ async fn update_app(
path,
&w_id
)
.execute(&mut tx)
.execute(&mut *tx)
.await?;
audit_log(
&mut tx,
&mut *tx,
&authed,
"apps.update",
ActionKind::Update,
@@ -1022,8 +1130,7 @@ async fn update_app(
)
.await?;
let tx: PushIsolationLevel<'_, rsmq_async::MultiplexedRsmq> =
PushIsolationLevel::Transaction(tx);
let tx = PushIsolationLevel::Transaction(tx);
let mut args: HashMap<String, Box<serde_json::value::RawValue>> = HashMap::new();
if let Some(dm) = ns.deployment_message {
args.insert("deployment_message".to_string(), to_raw_value(&dm));
@@ -1072,6 +1179,10 @@ async fn update_app(
#[derive(Debug, Deserialize, Clone)]
pub struct ExecuteApp {
/// The app version to execute. Fallback to `path` if not provided.
pub version: Option<i64>,
/// The app script id (from the `app_script` table) to execute.
pub id: Option<i64>,
pub args: HashMap<String, Box<RawValue>>,
// - script: script/<path>
// - flow: flow/<path>
@@ -1134,11 +1245,26 @@ async fn get_on_behalf_details_from_policy_and_authed(
Ok((username, permissioned_as, email))
}
/// Convert the triggerables from the old format to the new format.
fn empty_triggerables(mut policy: Policy) -> Policy {
use std::mem::take;
if let Some(triggerables) = take(&mut policy.triggerables) {
let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default();
for (k, static_inputs) in triggerables.into_iter() {
triggerables_v2.insert(
k,
PolicyTriggerableInputs { static_inputs, ..Default::default() },
);
}
policy.triggerables_v2 = Some(triggerables_v2);
}
policy
}
async fn execute_component(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<ExecuteApp>,
) -> Result<String> {
@@ -1157,102 +1283,132 @@ async fn execute_component(
};
let path = path.to_path();
let (arc_policy, policy): (Arc<Policy>, Policy);
let policy_triggerables_default = Default::default();
let policy = match payload.clone() {
// Two cases here:
// 1. The component is executed from the editor (i.e. in "preview" mode), then:
// - The policy is set to default (in `Viewer` execution mode).
// - The policy triggerables are built by the frontend and retrieved from the request
// payload.
// - In case of inline script, the `RawCode` from the request is pushed as is to the
// job queue.
// 2. Otherwise (i.e. "run" mode):
// - The policy and triggerables are fetched from the database.
// - In case of inline script, if an entry exists in the `app_script` table, push
// an `AppScript` job payload, as in (.1) otherwise.
let (policy, policy_triggerables) = match payload {
// 1. "preview" mode.
ExecuteApp {
force_viewer_static_fields: Some(static_fields),
force_viewer_one_of_fields: Some(one_of_fields),
force_viewer_static_fields: Some(static_inputs),
force_viewer_one_of_fields: Some(one_of_inputs),
force_viewer_allow_user_resources: Some(allow_user_resources),
..
} => {
let mut hm = HashMap::new();
if let Some(path) = payload.path.clone() {
hm.insert(
format!("{}:{path}", payload.component),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
} else {
hm.insert(
format!(
"{}:{}",
payload.component,
digest(payload.raw_code.clone().unwrap().content.as_str())
),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
}
Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: Some(hm),
on_behalf_of: None,
on_behalf_of_email: None,
s3_inputs: None,
}
}
} => (
&Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() },
&PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources },
),
// 2. "run" mode.
_ => {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
// Policy is fetched from the database on app `path` and `workspace_id`.
let policy_fut = sqlx::query_scalar!(
"SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"
FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
path,
&w_id
&w_id,
)
.fetch_optional(&db)
.await?;
.map_err(Into::<Error>::into)
.map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?))
.map(|policy| Result::Ok(serde_json::from_str(policy?.get())?))
.map_ok(empty_triggerables);
let policy = not_found_if_none(policy_o, "App", path)?;
// 1. The app `version` is provided: cache the fetched policy.
// 2. Otherwise, always fetch the policy from the database.
let policy = if let Some(id) = payload.version {
let cache = cache::anon!({ u64 => Arc<Policy> } in "policy" <= 1000);
arc_policy = policy_fut
.map_ok(Arc::new)
.cached(cache, &(id as u64))
.await?;
&*arc_policy
} else {
policy = policy_fut.await?;
&policy
};
serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
// Compute the path for the triggerables map:
// - flow: `flow/<payload.path>`
// - script: `script/<payload.path>`
// - inline script: `rawscript/<sha256(raw_code.content)>`
let path = match &payload {
// flow or script: just use the `payload.path`.
ExecuteApp { path: Some(path), .. } => path,
// inline script: without entry in the `app_script` table.
ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content),
// inline script: with an entry in the `app_script` table.
ExecuteApp { raw_code: Some(_), id: Some(id), .. } => {
let cache = cache::anon!({ u64 => Arc<String> } in "appscriptpath" <= 10000);
// `id` is unique, cache the result.
&*sqlx::query_scalar!(
"SELECT format('rawscript/%s', code_sha256) as \"path!: String\"
FROM app_script WHERE id = $1 LIMIT 1",
id
)
.fetch_one(&db)
.map_err(Into::<Error>::into)
.map_ok(Arc::new)
.cached(cache, &(*id as u64))
.await?
}
_ => unreachable!(),
};
// Retrieve the triggerables from the policy on `path` or `<component>:<path>`.
let triggerables_v2 = policy
.triggerables_v2
.as_ref()
.ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?;
let policy_triggerables = triggerables_v2
.get(path) // start with `path` in case we can avoid the next` format!`.
.or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path)))
.or(match policy.execution_mode {
ExecutionMode::Viewer => Some(&policy_triggerables_default),
_ => None,
})
.ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?;
(policy, policy_triggerables)
}
};
let (username, permissioned_as, email) =
get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
let (job_payload, (args, job_id), tag) = match payload {
ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
let content = &raw_code.content;
let payload = JobPayload::Code(raw_code.clone());
let path = digest(content);
let args = build_args(
policy,
&component,
path,
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, None)
}
ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?;
let args = build_args(
policy,
&component,
path.to_string(),
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, tag)
}
let (args, job_id) = build_args(
policy,
policy_triggerables,
payload.args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) {
// flow or script:
(Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
// inline script: in "preview" mode or without entry in the `app_script` table.
(None, Some(raw_code), None) => (JobPayload::Code(raw_code), None),
// inline script: in "run" mode and with an entry in the `app_script` table.
(None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => (
JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
None,
),
_ => unreachable!(),
};
let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);
let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone());
let (uuid, tx) = push(
&db,
@@ -1584,9 +1740,12 @@ async fn exists_app(
}
async fn build_args(
policy: Policy,
component: &str,
path: String,
policy: &Policy,
PolicyTriggerableInputs {
static_inputs,
one_of_inputs,
allow_user_resources,
}: &PolicyTriggerableInputs,
mut args: HashMap<String, Box<RawValue>>,
authed: Option<&ApiAuthed>,
user_db: &UserDB,
@@ -1594,54 +1753,6 @@ async fn build_args(
w_id: &str,
) -> Result<(PushArgsOwned, Option<Uuid>)> {
let mut job_id: Option<Uuid> = None;
let key = format!("{}:{}", component, &path);
let (static_inputs, one_of_inputs, allow_user_resources) = match policy {
Policy { triggerables_v2: Some(t), .. } => {
let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(PolicyTriggerableInputs {
static_inputs: HashMap::new(),
one_of_inputs: HashMap::new(),
allow_user_resources: Vec::new(),
})
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, one_of_inputs, allow_user_resources)
}
Policy { triggerables: Some(t), .. } => {
let static_inputs = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(HashMap::new())
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, HashMap::new(), Vec::new())
}
_ => Err(Error::BadRequest(format!(
"Policy is missing triggerables for {}",
key
)))?,
};
let mut safe_args = HashMap::<String, Box<RawValue>>::new();
// tracing::error!("{:?}", allow_user_resources);
@@ -1690,16 +1801,16 @@ async fn build_args(
}
for (k, v) in one_of_inputs {
if safe_args.contains_key(&k) {
if safe_args.contains_key(k) {
continue;
}
if let Some(arg_val) = args.get(&k) {
if let Some(arg_val) = args.get(k) {
let arg_str = arg_val.get();
let options_str_vec = v.iter().map(|x| x.get()).collect::<Vec<&str>>();
if options_str_vec.contains(&arg_str) {
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
@@ -1710,7 +1821,7 @@ async fn build_args(
.all(|x| options_str_vec.contains(&x.get()))
{
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
}

View File

@@ -0,0 +1,5 @@
use axum::Router;
pub fn global_unauthed_service() -> Router {
Router::new()
}

Some files were not shown because too many files have changed in this diff Show More