Compare commits

...

39 Commits

Author SHA1 Message Date
Faton Ramadani
5131ee8b89 feat(frontend): add a dev docker-compose file + update readme 2023-04-12 21:46:13 +02:00
Ruben Fiszel
a5f6d73f7d fix(backend): do not fail on schedule not existing anymore 2023-04-12 20:06:50 +02:00
Ruben Fiszel
670c84b901 exclude deno/gen/file from global cache 2023-04-12 19:18:48 +02:00
Oliver Veal
92a293488e feat: inputs library on run page
* display previous script inputs on script run page

* parallelise loading

* parallelise loading

* also working for flows

* separate endpoints for scripts and flows

* Splitpanes and Saved Inputs (UI)

* Saved inputs API endpoints

* Editable Input name

* Narrow width styling

* feat(frontend): Add a toggle to open the saved inputs (#1401)

* feat(frontend): Add a toggle to open the saved inputs

* feat(frontend): Add a toggle to open the saved inputs

* feat(frontend): Move toggle

* update all

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
Co-authored-by: Faton Ramadani <faton.ramadani14@gmail.com>
2023-04-12 17:29:06 +02:00
Ruben Fiszel
0cd1e65e46 frontend apps rename improvements 2023-04-12 14:38:13 +02:00
Faton Ramadani
6aa1008933 fix(frontend): Remove output when deleting a component (#1397) 2023-04-12 10:28:54 +02:00
Ruben Fiszel
9434bbb18b fix script explorer 2023-04-11 22:12:28 +02:00
Ruben Fiszel
e6632a32c9 chore(main): release 1.87.0 (#1375)
* chore(main): release 2.0.0

* Apply automatic changes

* Update version.txt

* Update CHANGELOG.md

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
2023-04-11 21:11:11 +02:00
Ruben Fiszel
54c4d03173 fix search input for app propagation 2023-04-11 20:04:34 +02:00
Ruben Fiszel
3932e5dfb9 add GOPRIVATE and NETRC for private modules 2023-04-11 19:55:38 +02:00
Ruben Fiszel
9b3d6a3dd9 fix hub compatible export 2023-04-11 18:33:45 +02:00
Faton Ramadani
58d4b556eb fix(frontend): Add missing stopPropagation (#1394)
* fix(frontend): Add missing stopPropagation

* fix(frontend): fix component selection
2023-04-11 16:22:22 +02:00
Ruben Fiszel
8552c92837 fix relative url deno loading2 2023-04-11 15:57:59 +02:00
Ruben Fiszel
955a213a50 fix(backend): nested deno relative imports 2023-04-11 15:16:55 +02:00
Ruben Fiszel
e82f5388b6 compile 2023-04-11 13:40:58 +02:00
Ádám Kovács
8a44f8e779 fix(frontend): Fix icon picker input (#1389) 2023-04-11 13:29:55 +02:00
Kai Jellinghaus
d45e6c94ab feat(backend): Redis based queue (#1324)
* Merge?

* Fix V8 breaking change

* WIP

* WIP

* Cleanup

* Move to git reference

* Fix Merge conflict

* update

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
2023-04-11 12:24:13 +02:00
Ruben Fiszel
60da67a725 restore int default argument parsing in dneo 2023-04-11 12:15:16 +02:00
Ruben Fiszel
0718931616 fix tests + scheduled_for uses now instead of system time 2023-04-11 11:48:14 +02:00
Faton Ramadani
41831d58ed fix(frontend): Fix mac shortcuts (#1381)
* fix(frontend): fix app init issue

* fix(frontend): Fix mac shortcuts

* Update NonRunnableComponent.svelte

---------

Co-authored-by: Ruben Fiszel <ruben@rubenfiszel.com>
2023-04-11 08:52:40 +02:00
Ruben Fiszel
36816877b4 remove initialized 2023-04-11 08:51:09 +02:00
dependabot[bot]
0040e15805 chore(deps-dev): bump @types/vscode from 1.74.0 to 1.77.0 in /frontend (#1358)
Bumps [@types/vscode](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/vscode) from 1.74.0 to 1.77.0.
- [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases)
- [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/vscode)

---
updated-dependencies:
- dependency-name: "@types/vscode"
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-04-11 08:24:34 +02:00
dependabot[bot]
72317e9b54 chore(deps-dev): bump svelte2tsx from 0.6.10 to 0.6.11 in /frontend (#1382)
Bumps [svelte2tsx](https://github.com/sveltejs/language-tools) from 0.6.10 to 0.6.11.
- [Release notes](https://github.com/sveltejs/language-tools/releases)
- [Commits](https://github.com/sveltejs/language-tools/compare/svelte2tsx-0.6.10...svelte2tsx-0.6.11)

---
updated-dependencies:
- dependency-name: svelte2tsx
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-04-11 08:24:19 +02:00
dependabot[bot]
4cdad376b3 chore(deps): bump python from 3.11.2-slim-buster to 3.11.3-slim-buster (#1383)
Bumps python from 3.11.2-slim-buster to 3.11.3-slim-buster.

---
updated-dependencies:
- dependency-name: python
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-04-11 08:24:02 +02:00
dependabot[bot]
866228b663 chore(deps-dev): bump eslint from 8.37.0 to 8.38.0 in /frontend (#1384)
Bumps [eslint](https://github.com/eslint/eslint) from 8.37.0 to 8.38.0.
- [Release notes](https://github.com/eslint/eslint/releases)
- [Changelog](https://github.com/eslint/eslint/blob/main/CHANGELOG.md)
- [Commits](https://github.com/eslint/eslint/compare/v8.37.0...v8.38.0)

---
updated-dependencies:
- dependency-name: eslint
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-04-11 08:23:42 +02:00
dependabot[bot]
7eacca4caa chore(deps-dev): bump pdfjs-dist from 3.4.120 to 3.5.141 in /frontend (#1385)
Bumps [pdfjs-dist](https://github.com/mozilla/pdfjs-dist) from 3.4.120 to 3.5.141.
- [Release notes](https://github.com/mozilla/pdfjs-dist/releases)
- [Commits](https://github.com/mozilla/pdfjs-dist/commits)

---
updated-dependencies:
- dependency-name: pdfjs-dist
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-04-11 08:23:21 +02:00
dependabot[bot]
1526537f56 chore(deps): bump svelte-select from 5.5.2 to 5.6.0 in /frontend (#1386)
Bumps [svelte-select](https://github.com/rob-balfre/svelte-select) from 5.5.2 to 5.6.0.
- [Release notes](https://github.com/rob-balfre/svelte-select/releases)
- [Changelog](https://github.com/rob-balfre/svelte-select/blob/master/CHANGELOG.md)
- [Commits](https://github.com/rob-balfre/svelte-select/compare/v5.5.2...v5.6.0)

---
updated-dependencies:
- dependency-name: svelte-select
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-04-11 08:22:03 +02:00
Ruben Fiszel
1f705cab2c fix(cli): overwrite archived scripts 2023-04-11 08:21:37 +02:00
Ruben Fiszel
f2d3c8208b feat(backend): add instance events webhook 2023-04-11 01:45:56 +02:00
Ruben Fiszel
1b04537c9a more app fixes 2023-04-10 22:34:09 +02:00
Ruben Fiszel
23e374b10d more app fixes 2023-04-10 21:57:07 +02:00
Ruben Fiszel
5fc72ea2e6 fix app first debounce for improve trigger on load 2023-04-10 20:27:37 +02:00
Ruben Fiszel
3f5df1ee41 fix home + refreshOnStart 2023-04-10 18:02:27 +02:00
Ádám Kovács
3dabac153f feat(frontend)!: Add option to return file names (#1380) 2023-04-10 17:32:03 +02:00
Ruben Fiszel
d0e0e1fdf2 fix(frontend): fix app init issue 2023-04-10 17:27:48 +02:00
Faton Ramadani
8e9c491650 fix(frontend): Fix frontend dependencies (#1379)
* fix(frontend): Fix frontend dependencies

* fix(frontend): Fix frontend dependencies

* fix(frontend): Fix frontend dependencies
2023-04-10 15:34:34 +02:00
Ruben Fiszel
df4c6289ac improve metrics 2023-04-10 11:58:44 +02:00
Ruben Fiszel
72c8d3921d improve metrics 2023-04-10 11:53:32 +02:00
Ruben Fiszel
e911869d99 perf: parallelize more operations for deno jobs 2023-04-10 10:55:56 +02:00
125 changed files with 3118 additions and 1187 deletions

View File

@@ -1,6 +1,39 @@
# Changelog
## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)
### ⚠ BREAKING CHANGES
* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380))
### Features
* **backend:** add instance events webhook ([f2d3c82](https://github.com/windmill-labs/windmill/commit/f2d3c8208b6daa49f304f355752145de47138a3c))
* **backend:** extend cached resolution for go ([dac61d1](https://github.com/windmill-labs/windmill/commit/dac61d1c982576d7589e16ab01c8cc8bad6e1686))
* **backend:** Redis based queue ([#1324](https://github.com/windmill-labs/windmill/issues/1324)) ([d45e6c9](https://github.com/windmill-labs/windmill/commit/d45e6c94abed609357b18d4daa7de6b2ea0ba978))
* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380)) ([3dabac1](https://github.com/windmill-labs/windmill/commit/3dabac153f302f48210d15ebaec514e72717300f))
* **python:** cache dependency resolution ([facb670](https://github.com/windmill-labs/windmill/commit/facb67093ce7d3b0874d0d559fb272ed822ce360))
### Bug Fixes
* **backend:** nested deno relative imports ([955a213](https://github.com/windmill-labs/windmill/commit/955a213a504c1f3b8811c930823e87fe7dba101a))
* **cli:** overwrite archived scripts ([1f705ca](https://github.com/windmill-labs/windmill/commit/1f705cab2ce8c79829f22fc6af9e06ecba7450b1))
* **frontend:** Add missing stopPropagation ([#1394](https://github.com/windmill-labs/windmill/issues/1394)) ([58d4b55](https://github.com/windmill-labs/windmill/commit/58d4b556ebbd76c6f07f1a16d601a9d824b99f7e))
* **frontend:** fix app init issue ([d0e0e1f](https://github.com/windmill-labs/windmill/commit/d0e0e1fdf27d9a7fb86c66e43398786b64d8b6b7))
* **frontend:** Fix frontend dependencies ([#1379](https://github.com/windmill-labs/windmill/issues/1379)) ([8e9c491](https://github.com/windmill-labs/windmill/commit/8e9c49165060a4a7f831b8be075593f89d867784))
* **frontend:** Fix icon picker input ([#1389](https://github.com/windmill-labs/windmill/issues/1389)) ([8a44f8e](https://github.com/windmill-labs/windmill/commit/8a44f8e7796f13698e2a99af9f3772f5e676604b))
* **frontend:** Fix mac shortcuts ([#1381](https://github.com/windmill-labs/windmill/issues/1381)) ([41831d5](https://github.com/windmill-labs/windmill/commit/41831d58ed593bb283600b76170f6e76783e0eae))
* **frontend:** fix popover configuration to avoid content shift ([#1377](https://github.com/windmill-labs/windmill/issues/1377)) ([2031e1e](https://github.com/windmill-labs/windmill/commit/2031e1ebd0dc020da104ee84a0294c86babcefaf))
* **frontend:** remove stopPropagation that was preventing components dnd ([#1378](https://github.com/windmill-labs/windmill/issues/1378)) ([de8dc1e](https://github.com/windmill-labs/windmill/commit/de8dc1e9cd7beea2ce62656e9e7676214f77a110))
### Performance Improvements
* parallelize more operations for deno jobs ([e911869](https://github.com/windmill-labs/windmill/commit/e911869d990956463834ac9ff35c52ba8236e362))
## [1.86.0](https://github.com/windmill-labs/windmill/compare/v1.85.0...v1.86.0) (2023-04-08)

View File

@@ -85,7 +85,7 @@ COPY .git/ .git/
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
FROM python:3.11.2-slim-buster
FROM python:3.11.3-slim-buster
ARG TARGETPLATFORM
ARG APP=/usr/src/app

View File

@@ -70,10 +70,10 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
3. Make it flow! You can chain your scripts or scripts made by the community
shared on [WindmillHub](https://hub.windmill.dev).
![Step 4](./imgs/windmill-flow.png)
![Step 3](./imgs/windmill-flow.png)
4. Build complex UI on top of your scripts and flows.
![Step 5](./imgs/windmill-builder.png)
![Step 4](./imgs/windmill-builder.png)
Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
through webhooks.
@@ -123,7 +123,7 @@ instance from local commands. See
### Running scripts locally
You can run your script locally easily, you simply need to pass the right
environment variables for the `wmill` client library to fetch resource and
environment variables for the `wmill` client library to fetch resources and
variables from your instance if necessary. See more:
<https://docs.windmill.dev/docs/advanced/local_development/>
@@ -315,6 +315,8 @@ it being synced automatically everyday.
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
| NETRC | | The netrc content to use a private go registry | Worker |
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
@@ -332,7 +334,7 @@ it being synced automatically everyday.
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
| NEW_USER_WEBHOOK | None | Webhook to notify of a new user added, signup/invite. Can hook back to windmill to send emails | Server |
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails | Server |
## Run a local dev setup

View File

@@ -18,4 +18,4 @@ rustflags = [
rustflags = [
"-C", "link-arg=-undefined",
"-C", "link-arg=dynamic_lookup",
]
]

354
backend/Cargo.lock generated
View File

@@ -178,6 +178,55 @@ dependencies = [
"zstd-safe",
]
[[package]]
name = "async-executor"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fa3dc5f2a8564f07759c008b9109dc0d39de92a88d5588b8a5036d286383afb"
dependencies = [
"async-lock",
"async-task",
"concurrent-queue",
"fastrand",
"futures-lite",
"slab",
]
[[package]]
name = "async-global-executor"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776"
dependencies = [
"async-channel",
"async-executor",
"async-io",
"async-lock",
"blocking",
"futures-lite",
"once_cell",
]
[[package]]
name = "async-io"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af"
dependencies = [
"async-lock",
"autocfg",
"cfg-if",
"concurrent-queue",
"futures-lite",
"log",
"parking",
"polling",
"rustix",
"slab",
"socket2",
"waker-fn",
]
[[package]]
name = "async-lock"
version = "2.7.0"
@@ -214,7 +263,33 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
name = "async-std"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d"
dependencies = [
"async-channel",
"async-global-executor",
"async-io",
"async-lock",
"crossbeam-utils",
"futures-channel",
"futures-core",
"futures-io",
"futures-lite",
"gloo-timers",
"kv-log-macro",
"log",
"memchr",
"once_cell",
"pin-project-lite",
"pin-utils",
"slab",
"wasm-bindgen-futures",
]
[[package]]
@@ -240,6 +315,12 @@ dependencies = [
"uuid 0.8.2",
]
[[package]]
name = "async-task"
version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc7ab41815b3c653ccd2978ec3255c81349336702dfdf62ee6f7069b12a3aae"
[[package]]
name = "async-timer"
version = "0.7.4"
@@ -259,7 +340,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -286,6 +367,12 @@ dependencies = [
"num-traits",
]
[[package]]
name = "atomic-waker"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3"
[[package]]
name = "autocfg"
version = "1.1.0"
@@ -294,9 +381,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
version = "0.6.12"
version = "0.6.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491"
checksum = "6e5abc76e6ef57bf438c067d2898481475994102eeb5d662a0b2162d0c2fdcf1"
dependencies = [
"async-trait",
"axum-core",
@@ -327,9 +414,9 @@ dependencies = [
[[package]]
name = "axum-core"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
dependencies = [
"async-trait",
"bytes",
@@ -360,6 +447,19 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "bb8"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e9f4fa9768efd269499d8fba693260cfc670891cf6de3adc935588447a77cc8"
dependencies = [
"async-trait",
"futures-channel",
"futures-util",
"parking_lot 0.11.2",
"tokio",
]
[[package]]
name = "better_scoped_tls"
version = "0.1.0"
@@ -434,6 +534,21 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae"
[[package]]
name = "blocking"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77231a1c8f801696fc0123ec6150ce92cffb8e164a02afb9c8ddee0e9b65ad65"
dependencies = [
"async-channel",
"async-lock",
"async-task",
"atomic-waker",
"fastrand",
"futures-lite",
"log",
]
[[package]]
name = "bstr"
version = "0.2.17"
@@ -601,7 +716,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -620,6 +735,20 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "combine"
version = "4.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
dependencies = [
"bytes",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
"tokio-util",
]
[[package]]
name = "concolor-override"
version = "1.0.0"
@@ -792,6 +921,16 @@ dependencies = [
"typenum",
]
[[package]]
name = "ctor"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
dependencies = [
"quote",
"syn 1.0.109",
]
[[package]]
name = "cxx"
version = "1.0.94"
@@ -816,7 +955,7 @@ dependencies = [
"proc-macro2",
"quote",
"scratch",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -833,7 +972,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -1287,7 +1426,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -1397,10 +1536,22 @@ dependencies = [
]
[[package]]
name = "gosyn"
version = "0.2.2"
name = "gloo-timers"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1584c4cbcb6f1d97481c19d1951c64f6e1256e5a03c28159ae45431c00511ee7"
checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c"
dependencies = [
"futures-channel",
"futures-core",
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "gosyn"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "248a4b26b282ec2e90272f7e7bd4764eaa6378dee13497ef09aa0efebd80cc85"
dependencies = [
"strum",
"unic-ucd-category",
@@ -1794,6 +1945,15 @@ version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
[[package]]
name = "kv-log-macro"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f"
dependencies = [
"log",
]
[[package]]
name = "lalrpop"
version = "0.19.9"
@@ -1976,6 +2136,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
"value-bag",
]
[[package]]
@@ -2254,9 +2415,9 @@ dependencies = [
[[package]]
name = "openssl"
version = "0.10.49"
version = "0.10.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33"
checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1"
dependencies = [
"bitflags",
"cfg-if",
@@ -2275,7 +2436,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -2286,9 +2447,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.84"
version = "0.9.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa"
checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0"
dependencies = [
"cc",
"libc",
@@ -2565,6 +2726,22 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "polling"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4be1c66a6add46bff50935c313dae30a5030cf8385c5206e8a95e9e9def974aa"
dependencies = [
"autocfg",
"bitflags",
"cfg-if",
"concurrent-queue",
"libc",
"log",
"pin-project-lite",
"windows-sys 0.48.0",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@@ -2650,7 +2827,7 @@ dependencies = [
"schemars",
"serde",
"serde_json",
"syn 2.0.13",
"syn 2.0.14",
"thiserror",
"typify",
"unicode-ident",
@@ -2670,7 +2847,7 @@ dependencies = [
"serde_json",
"serde_tokenstream",
"serde_yaml",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -2711,6 +2888,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radix_fmt"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce082a9940a7ace2ad4a8b7d0b1eac6aa378895f18be598230c5f2284ac05426"
[[package]]
name = "rand"
version = "0.7.3"
@@ -2782,6 +2965,27 @@ dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "redis"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f"
dependencies = [
"async-std",
"async-trait",
"bytes",
"combine",
"futures-util",
"itoa",
"percent-encoding",
"pin-project-lite",
"ryu",
"sha1_smol",
"tokio",
"tokio-util",
"url",
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
@@ -2934,6 +3138,21 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rsmq_async"
version = "5.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "086675eeb88ca55fa3b25c0075615d31dd9eb0adee219a9dcd999090cb371cab"
dependencies = [
"async-trait",
"bb8",
"lazy_static",
"radix_fmt",
"rand 0.8.5",
"redis",
"thiserror",
]
[[package]]
name = "rust-embed"
version = "6.6.1"
@@ -3043,7 +3262,7 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython#dde92d1269b5fbd09cf5dba5e7817f0e2a96c49f"
source = "git+https://github.com/RustPython/RustPython#c9546c2419f2e4db7e2987f30f0508dc29b30035"
dependencies = [
"num-bigint",
"rustpython-compiler-core",
@@ -3052,7 +3271,7 @@ dependencies = [
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython#dde92d1269b5fbd09cf5dba5e7817f0e2a96c49f"
source = "git+https://github.com/RustPython/RustPython#c9546c2419f2e4db7e2987f30f0508dc29b30035"
dependencies = [
"bitflags",
"bstr",
@@ -3065,7 +3284,7 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython#dde92d1269b5fbd09cf5dba5e7817f0e2a96c49f"
source = "git+https://github.com/RustPython/RustPython#c9546c2419f2e4db7e2987f30f0508dc29b30035"
dependencies = [
"ahash 0.7.6",
"anyhow",
@@ -3127,7 +3346,7 @@ dependencies = [
"schemars_derive",
"serde",
"serde_json",
"uuid 1.3.0",
"uuid 1.3.1",
]
[[package]]
@@ -3237,9 +3456,9 @@ dependencies = [
[[package]]
name = "serde"
version = "1.0.159"
version = "1.0.160"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
dependencies = [
"serde_derive",
]
@@ -3266,13 +3485,13 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.159"
version = "1.0.160"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -3338,7 +3557,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -3392,6 +3611,12 @@ dependencies = [
"digest 0.10.6",
]
[[package]]
name = "sha1_smol"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sha2"
version = "0.9.9"
@@ -3618,7 +3843,7 @@ dependencies = [
"thiserror",
"tokio-stream",
"url",
"uuid 1.3.0",
"uuid 1.3.1",
"webpki-roots",
"whoami",
]
@@ -3904,9 +4129,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.13"
version = "2.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5"
dependencies = [
"proc-macro2",
"quote",
@@ -3969,7 +4194,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -4083,7 +4308,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.13",
"syn 2.0.14",
]
[[package]]
@@ -4396,7 +4621,7 @@ dependencies = [
"regress",
"schemars",
"serde_json",
"syn 2.0.13",
"syn 2.0.14",
"thiserror",
"unicode-ident",
]
@@ -4412,7 +4637,7 @@ dependencies = [
"serde",
"serde_json",
"serde_tokenstream",
"syn 2.0.13",
"syn 2.0.14",
"typify-impl",
]
@@ -4429,7 +4654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13a3aaa69b04e5b66cc27309710a569ea23593612387d67daaf102e73aa974fd"
dependencies = [
"rand 0.8.5",
"uuid 1.3.0",
"uuid 1.3.1",
]
[[package]]
@@ -4617,9 +4842,9 @@ dependencies = [
[[package]]
name = "uuid"
version = "1.3.0"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79"
checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb"
dependencies = [
"getrandom 0.2.9",
"serde",
@@ -4643,6 +4868,16 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "value-bag"
version = "1.0.0-alpha.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55"
dependencies = [
"ctor",
"version_check",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
@@ -4861,7 +5096,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"axum",
@@ -4873,6 +5108,7 @@ dependencies = [
"rand 0.8.5",
"reqwest",
"rsa",
"rsmq_async",
"serde",
"serde_json",
"sha2 0.10.6",
@@ -4880,6 +5116,7 @@ dependencies = [
"tokio",
"tokio-metrics",
"tracing",
"url",
"windmill-api",
"windmill-api-client",
"windmill-common",
@@ -4889,7 +5126,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"argon2",
@@ -4916,6 +5153,7 @@ dependencies = [
"regex",
"reqwest",
"retainer",
"rsmq_async",
"rust-embed",
"serde",
"serde_json",
@@ -4946,7 +5184,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"base64 0.21.0",
"chrono",
@@ -4956,12 +5194,12 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
"uuid 1.3.0",
"uuid 1.3.1",
]
[[package]]
name = "windmill-audit"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"chrono",
"serde",
@@ -4974,7 +5212,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"axum",
@@ -4994,12 +5232,12 @@ dependencies = [
"tokio",
"tracing",
"tracing-subscriber",
"uuid 1.3.0",
"uuid 1.3.1",
]
[[package]]
name = "windmill-parser"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"serde",
"serde_json",
@@ -5007,7 +5245,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"itertools",
@@ -5022,7 +5260,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"gosyn",
@@ -5035,7 +5273,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"itertools",
@@ -5050,7 +5288,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"deno_core",
@@ -5064,31 +5302,34 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"chrono",
"chrono-tz",
"cron",
"futures-core",
"hex",
"hmac",
"lazy_static",
"prometheus",
"reqwest",
"rsmq_async",
"serde",
"serde_json",
"sql-builder",
"sqlx",
"tokio",
"tracing",
"ulid",
"uuid 1.3.0",
"uuid 1.3.1",
"windmill-audit",
"windmill-common",
]
[[package]]
name = "windmill-worker"
version = "1.86.0"
version = "1.87.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -5105,12 +5346,13 @@ dependencies = [
"prometheus",
"rand 0.8.5",
"regex",
"rsmq_async",
"serde",
"serde_json",
"sqlx",
"tokio",
"tracing",
"uuid 1.3.0",
"uuid 1.3.1",
"windmill-api-client",
"windmill-audit",
"windmill-common",

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.86.0"
version = "1.87.0"
authors.workspace = true
edition.workspace = true
@@ -19,7 +19,7 @@ members = [
]
[workspace.package]
version = "1.86.0"
version = "1.87.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -48,6 +48,8 @@ git-version.workspace = true
rsa.workspace = true
base64.workspace = true
sha2.workspace = true
rsmq_async.workspace = true
url.workspace = true
[dev-dependencies]
@@ -141,6 +143,7 @@ sqlx = { version = "^0", features = [
dotenv = "^0"
ulid = { version = "^1", features = ["uuid"] }
futures = "^0"
futures-core = "^0"
tokio-metrics = "0.1.0"
lazy_static = "1.4.0"
serde_derive = "1.0.147"
@@ -153,4 +156,7 @@ async-stripe = { version = "0.14", features = [
] }
async_zip = { version = "0.0.11", features = ["full"] }
once_cell = "1.17.1"
rsmq_async = { version = "5.1.5" }
gosyn = "0.2.2"
[patch.crates-io]

View File

@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS input;
DROP TYPE RUNNABLE_TYPE;

View File

@@ -0,0 +1,13 @@
CREATE TYPE RUNNABLE_TYPE AS ENUM ('ScriptHash', 'ScriptPath', 'FlowPath');
CREATE TABLE IF NOT EXISTS input (
id UUID PRIMARY KEY,
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
runnable_id VARCHAR(255) NOT NULL,
runnable_type RUNNABLE_TYPE NOT NULL,
name TEXT NOT NULL,
args JSONB NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
created_by VARCHAR(50) NOT NULL,
is_public BOOLEAN NOT NULL DEFAULT FALSE
);

View File

@@ -103,6 +103,11 @@ fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<
Expr::Lit(Lit::Str(Str { value, .. })) => {
Some(Value::String(value.to_string()))
}
Expr::Lit(Lit::Num(Number { value, .. }))
if (value == (value as u64) as f64) =>
{
Some(serde_json::json!(value as u64))
}
Expr::Lit(Lit::Num(Number { value, .. })) => Some(serde_json::json!(value)),
Expr::Lit(Lit::BigInt(BigInt { value, .. })) => Some(serde_json::json!(value)),
Expr::Lit(Lit::Bool(Bool { value, .. })) => Some(Value::Bool(value)),

View File

@@ -3957,76 +3957,6 @@
},
"query": "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND workspace_id = $2)"
},
"a1c41bbeb2d64fa1e7dfd2ed053191a1de5d786ae8c22e225e450865ecac94e9": {
"describe": {
"columns": [
{
"name": "id",
"ordinal": 0,
"type_info": "Uuid"
}
],
"nullable": [
false
],
"parameters": {
"Left": [
"Varchar",
"Uuid",
"Bool",
"Uuid",
"Varchar",
"Varchar",
"Timestamptz",
"Int8",
"Varchar",
"Text",
"Text",
"Jsonb",
{
"Custom": {
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
]
},
"name": "job_kind"
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
{
"Custom": {
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash"
]
},
"name": "script_lang"
}
},
"Bool",
"Text",
"Varchar",
"Bool",
"Uuid"
]
}
},
"query": "INSERT INTO queue\n (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) RETURNING id"
},
"a227548b6604c56bfc15eb780bd8ee72a89dc6701a50f5048e928bd87baa7b9a": {
"describe": {
"columns": [
@@ -5036,6 +4966,26 @@
},
"query": "\n INSERT INTO resume_job\n (id, resume_id, job, flow, value, approver)\n VALUES ($1, $2, $3, $4, $5, $6)\n "
},
"c4fb68ce79c1cec23cadc515285578ca18398504cf703091d06191ff68d95399": {
"describe": {
"columns": [
{
"name": "database_length!",
"ordinal": 0,
"type_info": "Int8"
}
],
"nullable": [
null
],
"parameters": {
"Left": [
"Text"
]
}
},
"query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1"
},
"c59dd666b9a316c027e8c319b80ccbab3a220d93b64357981bd4a03324dad1d0": {
"describe": {
"columns": [
@@ -6122,6 +6072,76 @@
},
"query": "SELECT EXISTS(SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2)"
},
"f014bbaecaa09840c6b66d8b58b942fcf9827a7dfd9450f225fddd2b404c4f36": {
"describe": {
"columns": [
{
"name": "id",
"ordinal": 0,
"type_info": "Uuid"
}
],
"nullable": [
false
],
"parameters": {
"Left": [
"Varchar",
"Uuid",
"Bool",
"Uuid",
"Varchar",
"Varchar",
"Timestamptz",
"Int8",
"Varchar",
"Text",
"Text",
"Jsonb",
{
"Custom": {
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies"
]
},
"name": "job_kind"
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
{
"Custom": {
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash"
]
},
"name": "script_lang"
}
},
"Bool",
"Text",
"Varchar",
"Bool",
"Uuid"
]
}
},
"query": "INSERT INTO queue\n (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, \n script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)\n VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) RETURNING id"
},
"f056b5f3e66a764748925f1bfd3180923fde8c7fdf69088d0e4a5555cc049545": {
"describe": {
"columns": [

View File

@@ -49,8 +49,40 @@ async fn main() -> anyhow::Result<()> {
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
let rsmq_config = std::env::var("REDIS_URL").ok().map(|x| {
let url = x.parse::<url::Url>().unwrap();
let mut config = rsmq_async::RsmqOptions { ..Default::default() };
config.host = url.host_str().expect("redis host required").to_owned();
config.password = url.password().map(|s| s.to_owned());
config.db = url
.path_segments()
.and_then(|mut segments| segments.next())
.and_then(|segment| segment.parse().ok())
.unwrap_or(0);
config.ns = url
.query_pairs()
.find(|s| s.0 == "rsmq_namespace")
.map(|s| s.1)
.unwrap_or(std::borrow::Cow::Borrowed("rsmq"))
.into_owned();
config.port = url.port().unwrap_or(6379).to_string();
config
});
let db = windmill_common::connect_db(server_mode).await?;
let rsmq = if let Some(config) = rsmq_config {
let mut rsmq = rsmq_async::MultiplexedRsmq::new(config).await.unwrap();
let _ = rsmq_async::RsmqConnection::create_queue(&mut rsmq, "main_queue", None, None, None)
.await;
Some(rsmq)
} else {
None
};
if server_mode {
windmill_api::migrate_db(&db).await?;
}
@@ -96,6 +128,8 @@ Windmill Community Edition {GIT_VERSION}
"PYTHON_PATH",
"DENO_PATH",
"GO_PATH",
"GOPRIVATE",
"NETRC",
"PIP_INDEX_URL",
"PIP_EXTRA_INDEX_URL",
"PIP_TRUSTED_HOST",
@@ -112,16 +146,17 @@ Windmill Community Edition {GIT_VERSION}
"INCLUDE_HEADERS",
"WHITELIST_WORKSPACES",
"BLACKLIST_WORKSPACES",
"NEW_USER_WEBHOOK",
"INSTANCE_EVENTS_WEBHOOK",
"CLOUD_HOSTED",
]);
if server_mode || num_workers > 0 {
let addr = SocketAddr::from((server_bind_address, port));
let rsmq2 = rsmq.clone();
let server_f = async {
if server_mode {
windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe()).await?;
}
Ok(()) as anyhow::Result<()>
};
@@ -133,15 +168,17 @@ Windmill Community Edition {GIT_VERSION}
rx.resubscribe(),
num_workers,
base_internal_url.clone(),
rsmq.clone(),
)
.await?;
}
Ok(()) as anyhow::Result<()>
};
let rsmq2 = rsmq.clone();
let monitor_f = async {
if server_mode {
monitor_db(&db, rx.resubscribe(), &base_internal_url);
monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2);
}
Ok(()) as anyhow::Result<()>
};
@@ -181,10 +218,11 @@ fn display_config(envs: Vec<&str>) {
)
}
pub fn monitor_db(
pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
db: &Pool<Postgres>,
rx: tokio::sync::broadcast::Receiver<()>,
base_internal_url: &str,
rsmq: Option<R>,
) {
let db1 = db.clone();
let db2 = db.clone();
@@ -192,16 +230,17 @@ pub fn monitor_db(
let rx2 = rx.resubscribe();
let base_internal_url = base_internal_url.to_string();
tokio::spawn(async move {
windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url).await
windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq).await
});
tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
}
pub async fn run_workers(
pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
db: Pool<Postgres>,
rx: tokio::sync::broadcast::Receiver<()>,
num_workers: i32,
base_internal_url: String,
rsmq: Option<R>,
) -> anyhow::Result<()> {
let license_key = std::env::var("LICENSE_KEY").ok();
#[cfg(feature = "enterprise")]
@@ -231,6 +270,7 @@ pub async fn run_workers(
let ip = ip.clone();
let rx = rx.resubscribe();
let base_internal_url = base_internal_url.clone();
let rsmq2 = rsmq.clone();
handles.push(tokio::spawn(monitor.instrument(async move {
tracing::info!(worker = %worker_name, "starting worker");
windmill_worker::run_worker(
@@ -241,6 +281,7 @@ pub async fn run_workers(
&ip,
rx,
&base_internal_url,
rsmq2,
)
.await
})));

View File

@@ -88,7 +88,7 @@ impl ApiServer {
let addr = sock.local_addr().unwrap();
drop(sock);
let task = tokio::task::spawn(windmill_api::run_server(db.clone(), addr, rx));
let task = tokio::task::spawn(windmill_api::run_server(db.clone(), None, addr, rx));
return Self { addr, tx, task };
}
@@ -656,7 +656,7 @@ def main(error, port):
},
}))
.unwrap();
let (attempts, responses) = [
let (_attempts, responses) = [
/* fail the first step twice */
(0x00, None),
(0x00, None),
@@ -678,14 +678,20 @@ def main(error, port):
_ => panic!("expected failure module"),
}
assert_eq!(server.close().await, attempts);
println!("result: {:#?}", result);
assert_eq!(
result,
json!({
"recv": 42,
"from failure module": {"error": {"name": "IndexError", "stack": " File \"/tmp/tmp/main/step_0.py\", line 5, in main\n return sock.recv(1)[0]\n", "message": "index out of range"}},
})
result
.get("from failure module")
.unwrap()
.get("error")
.unwrap()
.get("name")
.unwrap()
.clone(),
json!("IndexError")
);
assert_eq!(result.get("recv").unwrap().clone(), json!(42));
}
}
@@ -821,9 +827,8 @@ impl RunJob {
async fn push(self, db: &Pool<Postgres>) -> Uuid {
let RunJob { payload, args } = self;
let tx = db.begin().await.unwrap();
let (uuid, tx) = windmill_queue::push(
tx,
let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
(None, db.begin().await.unwrap()).into(),
"test-workspace",
payload,
args,
@@ -841,8 +846,7 @@ impl RunJob {
)
.await
.expect("push has to succeed");
tx.commit().await.expect("push has to commit");
tx.commit().await.unwrap();
uuid
}
@@ -912,7 +916,7 @@ fn spawn_test_worker(
let ip: &str = Default::default();
let future = async move {
let base_internal_url = format!("http://localhost:{}", port);
windmill_worker::run_worker(
windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
&db,
worker_instance,
worker_name,
@@ -920,6 +924,7 @@ fn spawn_test_worker(
ip,
rx,
&base_internal_url,
None,
)
.await
};
@@ -2068,7 +2073,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
.unwrap();
assert_eq!(
serde_json::from_value::<ErrorResult>(result)
serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
.unwrap()
.error
.name,
@@ -2105,7 +2110,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
.unwrap();
assert_eq!(
serde_json::from_value::<ErrorResult>(result)
serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
.unwrap()
.error
.name,

View File

@@ -70,4 +70,5 @@ async-stripe = { workspace = true, optional = true }
lazy_static.workspace = true
prometheus.workspace = true
async_zip.workspace = true
rsmq_async.workspace = true
regex.workspace = true

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.86.0
version: 1.87.0
title: Windmill API
contact:
@@ -2782,6 +2782,27 @@ paths:
schema:
type: string
/w/{workspace}/flows/input_history/p/{path}:
get:
summary: list inputs for previous completed flow jobs
operationId: getFlowInputHistoryByPath
tags:
- flow
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
responses:
"200":
description: input history for completed jobs with this flow path
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Input"
/w/{workspace}/apps/list:
get:
summary: list all available apps
@@ -3322,23 +3343,6 @@ paths:
schema:
$ref: "#/components/schemas/Job"
# /w/{workspace}/jobs/flow/current_state/{id}:
# get:
# summary: get flow current step state
# operationId: getJob
# tags:
# - job
# parameters:
# - $ref: "#/components/parameters/WorkspaceId"
# - $ref: "#/components/parameters/JobId"
# responses:
# "200":
# description: state details
# content:
# application/json:
# schema:
# type: string
/w/{workspace}/jobs_u/getupdate/{id}:
get:
summary: get job updates
@@ -4558,6 +4562,118 @@ paths:
"200":
description: unstar item
/w/{workspace}/inputs/history:
get:
summary: List Inputs used in previously completed jobs
operationId: getInputHistory
tags:
- input
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/RunnableId"
- $ref: "#/components/parameters/RunnableType"
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
responses:
"200":
description: Input history for completed jobs
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Input"
/w/{workspace}/inputs/list:
get:
summary: List saved Inputs for a Runnable
operationId: listInputs
tags:
- input
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/RunnableId"
- $ref: "#/components/parameters/RunnableType"
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
responses:
"200":
description: Saved Inputs for a Runnable
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/Input"
/w/{workspace}/inputs/create:
post:
summary: Create an Input for future use in a script or flow
operationId: createInput
tags:
- input
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/RunnableId"
- $ref: "#/components/parameters/RunnableType"
requestBody:
description: Input
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/CreateInput"
responses:
"201":
description: Input created
content:
text/plain:
schema:
type: string
format: uuid
/w/{workspace}/inputs/update:
post:
summary: Update an Input
operationId: updateInput
tags:
- input
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: UpdateInput
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/UpdateInput"
responses:
"201":
description: Input updated
content:
text/plain:
schema:
type: string
format: uuid
/w/{workspace}/inputs/delete/{input}:
post:
summary: Delete a Saved Input
operationId: deleteInput
tags:
- input
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/InputId"
responses:
"200":
description: Input deleted
content:
text/plain:
schema:
type: string
format: uuid
components:
securitySchemes:
bearerAuth:
@@ -4567,6 +4683,7 @@ components:
type: apiKey
in: cookie
name: token
parameters:
WorkspaceId:
name: workspace
@@ -4685,7 +4802,6 @@ components:
in: query
schema:
type: string
ScriptStartPath:
name: script_path_start
description: mask to filter matching starting path
@@ -4742,7 +4858,6 @@ components:
in: query
schema:
type: string
ResultFilter:
name: result
description: filter on jobs containing those result as a json subset (@> in postgres)
@@ -4802,9 +4917,26 @@ components:
# type: string
# enum: ["preview", "script", "dependencies"]
# explode: false
RunnableId:
name: runnable_id
in: query
schema:
type: string
RunnableType:
name: runnable_type
in: query
schema:
$ref: "#/components/schemas/RunnableType"
InputId:
name: input
in: path
required: true
schema:
type: string
schemas:
$ref: "../../openflow.openapi.yaml#/components/schemas"
Script:
type: object
properties:
@@ -4875,6 +5007,60 @@ components:
type: object
additionalProperties: {}
Input:
type: object
properties:
id:
type: string
name:
type: string
args:
type: object
created_by:
type: string
created_at:
type: string
format: date-time
is_public:
type: boolean
required:
- id
- name
- args
- created_by
- created_at
- is_public
CreateInput:
type: object
properties:
name:
type: string
args:
type: object
required:
- name
- args
- created_by
UpdateInput:
type: object
properties:
id:
type: string
name:
type: string
is_public:
type: boolean
required:
- id
- name
- is_public
RunnableType:
type: string
enum: ["ScriptHash", "ScriptPath", "FlowPath"]
QueuedJob:
type: object
properties:
@@ -5896,6 +6082,7 @@ components:
- extra_perms
- edited_at
- execution_mode
AppWithLastVersion:
type: object
properties:

View File

@@ -37,7 +37,7 @@ use windmill_common::{
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
},
};
use windmill_queue::{push, JobPayload, RawCode};
use windmill_queue::{push, JobPayload, QueueTransaction, RawCode};
pub fn workspaced_service() -> Router {
Router::new()
@@ -565,6 +565,7 @@ fn digest(code: &str) -> String {
async fn execute_component(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<ExecuteApp>,
) -> Result<String> {
@@ -583,7 +584,7 @@ async fn execute_component(
};
let path = path.to_path();
let mut tx = db.begin().await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
let mut hm = HashMap::new();
@@ -652,8 +653,12 @@ async fn execute_component(
}
ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
let payload = if path.starts_with("script/") {
script_path_to_payload(path.strip_prefix("script/").unwrap(), &mut tx, &w_id)
.await?
script_path_to_payload(
path.strip_prefix("script/").unwrap(),
tx.transaction_mut(),
&w_id,
)
.await?
} else if path.starts_with("flow/") {
JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
} else {
@@ -686,8 +691,8 @@ async fn execute_component(
true,
)
.await?;
tx.commit().await?;
Ok(uuid.to_string())
}

View File

@@ -6,14 +6,20 @@
* LICENSE-AGPL for a copy of the license.
*/
use hyper::StatusCode;
use sql_builder::prelude::*;
use crate::{
db::{UserDB, DB},
schedule::clear_schedule,
users::{maybe_refresh_folders, require_owner_of_path, Authed},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use hyper::StatusCode;
use sql_builder::prelude::*;
use sql_builder::SqlBuilder;
use sqlx::{Postgres, Transaction};
use windmill_audit::{audit_log, ActionKind};
@@ -25,15 +31,7 @@ use windmill_common::{
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
},
};
use windmill_queue::{push, schedule::push_scheduled_job, JobPayload};
use crate::{
db::{UserDB, DB},
schedule::clear_schedule,
users::{maybe_refresh_folders, require_owner_of_path, Authed},
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
use windmill_queue::{push, schedule::push_scheduled_job, JobPayload, QueueTransaction};
pub fn workspaced_service() -> Router {
Router::new()
@@ -180,6 +178,7 @@ async fn create_flow(
authed: Authed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(webhook): Extension<WebhookShared>,
Path(w_id): Path<String>,
Json(nf): Json<NewFlow>,
@@ -187,10 +186,10 @@ async fn create_flow(
// cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await;
let mut tx = user_db.clone().begin(&authed).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
check_path_conflict(&mut tx, &w_id, &nf.path).await?;
check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
check_path_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
sqlx::query!(
"INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
@@ -222,13 +221,11 @@ async fn create_flow(
)
.await?;
tx.commit().await?;
webhook.send_message(
w_id.clone(),
WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
);
let tx = user_db.begin(&authed).await?;
let (dependency_job_uuid, mut tx) = push(
tx,
&w_id,
@@ -247,6 +244,7 @@ async fn create_flow(
true,
)
.await?;
sqlx::query!(
"UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3",
dependency_job_uuid,
@@ -286,6 +284,7 @@ async fn check_schedule_conflict<'c>(
async fn update_flow(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(db): Extension<DB>,
Extension(webhook): Extension<WebhookShared>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
@@ -294,9 +293,9 @@ async fn update_flow(
let flow_path = flow_path.to_path();
let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await;
let mut tx = user_db.clone().begin(&authed).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
check_schedule_conflict(tx.transaction_mut(), &w_id, flow_path).await?;
let schema = nf.schema.map(|x| x.0);
let old_dep_job = sqlx::query_scalar!(
@@ -323,13 +322,13 @@ async fn update_flow(
.await?;
if nf.path != flow_path {
check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
if !authed.is_admin {
require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
}
let mut schedulables = sqlx::query_as!(
let mut schedulables: Vec<Schedule> = sqlx::query_as!(
Schedule,
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
nf.path,
@@ -352,32 +351,32 @@ async fn update_flow(
schedulables.push(schedule);
}
for schedule in schedulables {
clear_schedule(&mut tx, flow_path, true).await?;
for schedule in schedulables.into_iter() {
// TODO: Why is this in the loop in the first place? Seems like it's just doing nothing after the first iteration? Should this use schedule.path?
clear_schedule(tx.transaction_mut(), flow_path, true).await?;
if schedule.enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
}
audit_log(
&mut tx,
&authed.username,
"flows.update",
ActionKind::Create,
&w_id,
Some(&nf.path.to_string()),
Some(
[Some(("flow", nf.path.as_str()))]
.into_iter()
.flatten()
.collect(),
),
)
.await?;
}
audit_log(
&mut tx,
&authed.username,
"flows.update",
ActionKind::Create,
&w_id,
Some(&nf.path.to_string()),
Some(
[Some(("flow", nf.path.as_str()))]
.into_iter()
.flatten()
.collect(),
),
)
.await?;
tx.commit().await?;
webhook.send_message(
w_id.clone(),
WebhookMessage::UpdateFlow {
@@ -387,7 +386,6 @@ async fn update_flow(
},
);
let tx = user_db.begin(&authed).await?;
let (dependency_job_uuid, mut tx) = push(
tx,
&w_id,

View File

@@ -0,0 +1,282 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{db::UserDB, jobs::CompletedJob, users::Authed};
use axum::{
extract::{Path, Query},
routing::{get, post},
Extension, Json, Router,
};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sqlx::types::Uuid;
use std::{
fmt::{Display, Formatter},
vec,
};
use windmill_common::{
error::JsonResult,
scripts::to_i64,
utils::{paginate, Pagination},
};
use windmill_queue::JobKind;
pub fn workspaced_service() -> Router {
Router::new()
.route("/history", get(get_input_history))
.route("/list", get(list_saved_inputs))
.route("/create", post(create_input))
.route("/update", post(update_input))
.route("/delete/:id", post(delete_input))
}
#[derive(Debug, sqlx::FromRow, Serialize, Deserialize)]
pub struct InputRow {
pub id: Uuid,
pub workspace_id: String,
pub runnable_id: String,
pub runnable_type: RunnableType,
pub name: String,
pub args: Value,
pub created_at: DateTime<Utc>,
pub created_by: String,
pub is_public: bool,
}
#[derive(Debug, Serialize, Deserialize, sqlx::Type)]
#[sqlx(type_name = "runnable_type")]
pub enum RunnableType {
ScriptHash,
ScriptPath,
FlowPath,
}
impl Display for RunnableType {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
RunnableType::ScriptHash => write!(f, "ScriptHash"),
RunnableType::ScriptPath => write!(f, "ScriptPath"),
RunnableType::FlowPath => write!(f, "FlowPath"),
}
}
}
impl RunnableType {
fn job_kind(&self) -> JobKind {
match self {
RunnableType::ScriptHash => JobKind::Script,
RunnableType::ScriptPath => JobKind::Script,
RunnableType::FlowPath => JobKind::Flow,
}
}
fn column_name(&self) -> &'static str {
match self {
RunnableType::ScriptHash => "script_hash",
RunnableType::ScriptPath => "script_path",
RunnableType::FlowPath => "script_path",
}
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct RunnableParams {
pub runnable_id: String,
pub runnable_type: RunnableType,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Input {
id: Uuid,
name: String,
created_at: chrono::DateTime<chrono::Utc>,
args: serde_json::Value,
created_by: String,
is_public: bool,
}
async fn get_input_history(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(r): Query<RunnableParams>,
) -> JsonResult<Vec<Input>> {
let (per_page, offset) = paginate(pagination);
let mut tx = user_db.begin(&authed).await?;
let sql = &format!(
"select distinct on (args) * from completed_job \
where {} = $1 and job_kind = $2 and workspace_id = $3 \
order by args, started_at desc limit $4 offset $5",
r.runnable_type.column_name()
);
let query = sqlx::query_as::<_, CompletedJob>(sql);
let query = match r.runnable_type {
RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),
_ => query.bind(&r.runnable_id),
};
let rows = query
.bind(r.runnable_type.job_kind())
.bind(&w_id)
.bind(per_page as i32)
.bind(offset as i32)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
let mut inputs = vec![];
for row in rows {
inputs.push(Input {
id: row.id,
name: format!(
"{} {}",
row.created_at.format("%H:%M %-d/%-m"),
row.created_by
),
created_at: row.created_at,
args: row.args.unwrap_or(serde_json::json!({})),
created_by: row.created_by,
is_public: true,
});
}
Ok(Json(inputs))
}
async fn list_saved_inputs(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(r): Query<RunnableParams>,
) -> JsonResult<Vec<Input>> {
let (per_page, offset) = paginate(pagination);
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, InputRow>(
"select * from input \
where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
and is_public IS true OR created_by = $4 \
order by created_at desc limit $5 offset $6",
)
.bind(&r.runnable_id)
.bind(&r.runnable_type)
.bind(&w_id)
.bind(&authed.username)
.bind(per_page as i32)
.bind(offset as i32)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
let mut inputs: Vec<Input> = Vec::new();
for row in rows {
inputs.push(Input {
id: row.id,
name: row.name,
args: row.args,
created_by: row.created_by,
created_at: row.created_at,
is_public: row.is_public,
})
}
Ok(Json(inputs))
}
#[derive(Debug, Serialize, Deserialize)]
pub struct CreateInput {
name: String,
args: serde_json::Value,
}
async fn create_input(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(r): Query<RunnableParams>,
Json(input): Json<CreateInput>,
) -> JsonResult<String> {
let mut tx = user_db.begin(&authed).await?;
let id = Uuid::new_v4();
sqlx::query(
"INSERT INTO input (id, workspace_id, runnable_id, runnable_type, name, args, created_by) VALUES ($1, $2, $3, $4, $5, $6, $7)",
)
.bind(&id)
.bind(&w_id)
.bind(&r.runnable_id)
.bind(&r.runnable_type)
.bind(&input.name)
.bind(&input.args)
.bind(&authed.username)
.execute(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(id.to_string()))
}
#[derive(Debug, Serialize, Deserialize)]
pub struct UpdateInput {
id: Uuid,
name: String,
is_public: bool,
}
async fn update_input(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(input): Json<UpdateInput>,
) -> JsonResult<String> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4")
.bind(&input.name)
.bind(&input.is_public)
.bind(&input.id)
.bind(&w_id)
.execute(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(input.id.to_string()))
}
async fn delete_input(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, i_id)): Path<(String, Uuid)>,
) -> JsonResult<String> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2")
.bind(&i_id)
.bind(&w_id)
.execute(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(i_id.to_string()))
}

View File

@@ -6,6 +6,12 @@
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
db::{UserDB, DB},
users::{require_owner_of_path, Authed, OptAuthed},
variables::get_workspace_key,
BASE_URL,
};
use anyhow::Context;
use axum::{
extract::{FromRequest, Json, Path, Query},
@@ -30,13 +36,8 @@ use windmill_common::{
users::username_to_permissioned_as,
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
};
use windmill_queue::{get_queued_job, push, JobKind, JobPayload, QueuedJob, RawCode};
use crate::{
db::{UserDB, DB},
users::{require_owner_of_path, Authed, OptAuthed},
variables::get_workspace_key,
BASE_URL,
use windmill_queue::{
get_queued_job, push, JobKind, JobPayload, QueueTransaction, QueuedJob, RawCode,
};
pub fn workspaced_service() -> Router {
@@ -64,6 +65,7 @@ pub fn workspaced_service() -> Router {
.route("/run/preview_flow", post(run_preview_flow_job))
.route("/list", get(list_jobs))
.route("/queue/list", get(list_queue_jobs))
.route("/queue/count", get(count_queue_jobs))
.route("/completed/list", get(list_completed_jobs))
.route("/completed/get/:id", get(get_completed_job))
.route("/completed/get_result/:id", get(get_completed_job_result))
@@ -114,6 +116,7 @@ async fn get_result_by_id(
}
async fn cancel_job_api(
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
@@ -127,7 +130,7 @@ async fn cancel_job_api(
};
let (mut tx, job_option) =
windmill_queue::cancel_job(&username, reason, id, &w_id, tx, false).await?;
windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, false).await?;
if let Some(id) = job_option {
audit_log(
@@ -156,6 +159,7 @@ async fn cancel_job_api(
}
async fn force_cancel(
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
@@ -169,7 +173,7 @@ async fn force_cancel(
};
let (mut tx, job_option) =
windmill_queue::cancel_job(&username, reason, id, &w_id, tx, true).await?;
windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, true).await?;
if let Some(id) = job_option {
audit_log(
@@ -496,6 +500,26 @@ async fn list_queue_jobs(
Ok(Json(jobs))
}
#[derive(Serialize, Debug, FromRow)]
struct QueueStats {
database_length: i64,
}
async fn count_queue_jobs(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> error::JsonResult<QueueStats> {
Ok(Json(
sqlx::query_as!(
QueueStats,
"SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1",
w_id
)
.fetch_one(&db)
.await?,
))
}
async fn list_jobs(
authed: Authed,
Extension(user_db): Extension<UserDB>,
@@ -793,6 +817,7 @@ async fn get_suspended_flow_info<'c>(
pub async fn cancel_suspended_job(
/* unauthed */
Extension(db): Extension<DB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, job, resume_id, secret)): Path<(String, Uuid, u32, String)>,
Query(approver): Query<QueryApprover>,
) -> error::Result<String> {
@@ -815,6 +840,7 @@ pub async fn cancel_suspended_job(
parent_flow,
&w_id,
tx,
rsmq,
false,
)
.await?;
@@ -1208,14 +1234,15 @@ fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
pub async fn run_flow_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
) -> error::Result<(StatusCode, String)> {
let flow_path = flow_path.to_path();
let mut tx = user_db.begin(&authed).await?;
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1243,15 +1270,16 @@ pub async fn run_flow_by_path(
pub async fn run_job_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
) -> error::Result<(StatusCode, String)> {
let script_path = script_path.to_path();
let mut tx = user_db.begin(&authed).await?;
let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1393,6 +1421,7 @@ lazy_static::lazy_static! {
pub async fn run_wait_result_job_by_path_get(
authed: Authed,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
@@ -1411,8 +1440,8 @@ pub async fn run_wait_result_job_by_path_get(
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
let mut tx = user_db.clone().begin(&authed).await?;
let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
let (uuid, tx) = push(
tx,
@@ -1447,6 +1476,7 @@ pub async fn run_wait_result_job_by_path_get(
pub async fn run_wait_result_job_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(db): Extension<DB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
@@ -1455,8 +1485,8 @@ pub async fn run_wait_result_job_by_path(
) -> error::JsonResult<serde_json::Value> {
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
let mut tx = user_db.clone().begin(&authed).await?;
let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
@@ -1493,6 +1523,7 @@ pub async fn run_wait_result_job_by_path(
pub async fn run_wait_result_job_by_hash(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(db): Extension<DB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
@@ -1502,8 +1533,9 @@ pub async fn run_wait_result_job_by_hash(
check_queue_too_long(db, run_query.queue_limit).await?;
let hash = script_hash.0;
let mut tx = user_db.clone().begin(&authed).await?;
let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1539,6 +1571,7 @@ pub async fn run_wait_result_job_by_hash(
pub async fn run_wait_result_flow_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(db): Extension<DB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
@@ -1548,8 +1581,8 @@ pub async fn run_wait_result_flow_by_path(
check_queue_too_long(db, run_query.queue_limit).await?;
let flow_path = flow_path.to_path();
let mut tx = user_db.clone().begin(&authed).await?;
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1570,7 +1603,6 @@ pub async fn run_wait_result_flow_by_path(
!run_query.invisible_to_owner.unwrap_or(false),
)
.await?;
tx.commit().await?;
run_wait_result(
@@ -1601,13 +1633,14 @@ pub async fn script_path_to_payload<'c>(
async fn run_preview_job(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path(w_id): Path<String>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
Json(preview): Json<Preview>,
) -> error::Result<(StatusCode, String)> {
let mut tx = user_db.begin(&authed).await?;
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1634,19 +1667,21 @@ async fn run_preview_job(
)
.await?;
tx.commit().await?;
Ok((StatusCode::CREATED, uuid.to_string()))
}
async fn run_preview_flow_job(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path(w_id): Path<String>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
Json(raw_flow): Json<PreviewFlow>,
) -> error::Result<(StatusCode, String)> {
let mut tx = user_db.begin(&authed).await?;
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1668,21 +1703,23 @@ async fn run_preview_flow_job(
)
.await?;
tx.commit().await?;
Ok((StatusCode::CREATED, uuid.to_string()))
}
pub async fn run_job_by_hash(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
headers: HeaderMap,
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
) -> error::Result<(StatusCode, String)> {
let hash = script_hash.0;
let mut tx = user_db.begin(&authed).await?;
let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
let (uuid, tx) = push(
@@ -1704,6 +1741,7 @@ pub async fn run_job_by_hash(
)
.await?;
tx.commit().await?;
Ok((StatusCode::CREATED, uuid.to_string()))
}

View File

@@ -7,6 +7,13 @@
*/
use crate::oauth2::AllClients;
use crate::{
db::UserDB,
oauth2::{build_oauth_clients, SlackVerifier},
tracing_init::{MyMakeSpan, MyOnResponse},
users::{Authed, OptAuthed},
webhook_util::WebhookShared,
};
use argon2::Argon2;
use axum::{middleware::from_extractor, routing::get, Extension, Router};
use db::DB;
@@ -22,14 +29,6 @@ use tower_http::{
};
use windmill_common::utils::rd_string;
use crate::{
db::UserDB,
oauth2::{build_oauth_clients, SlackVerifier},
tracing_init::{MyMakeSpan, MyOnResponse},
users::{Authed, OptAuthed},
webhook_util::WebhookShared,
};
mod apps;
mod audit;
mod capture;
@@ -39,6 +38,7 @@ mod flows;
mod folders;
mod granular_acls;
mod groups;
mod inputs;
pub mod jobs;
mod oauth2;
mod resources;
@@ -81,6 +81,7 @@ lazy_static::lazy_static! {
pub async fn run_server(
db: DB,
rsmq: Option<rsmq_async::MultiplexedRsmq>,
addr: SocketAddr,
mut rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<()> {
@@ -100,6 +101,7 @@ pub async fn run_server(
.on_request(()),
)
.layer(Extension(db.clone()))
.layer(Extension(rsmq))
.layer(Extension(user_db))
.layer(Extension(auth_cache.clone()))
.layer(CookieManagerLayer::new())
@@ -117,25 +119,27 @@ pub async fn run_server(
.nest(
"/w/:workspace_id",
Router::new()
.nest("/scripts", scripts::workspaced_service())
// Reordered alphabetically
.nest("/acls", granular_acls::workspaced_service())
.nest("/apps", apps::workspaced_service())
.nest("/audit", audit::workspaced_service())
.nest("/capture", capture::workspaced_service())
.nest("/favorites", favorite::workspaced_service())
.nest("/flows", flows::workspaced_service())
.nest("/folders", folders::workspaced_service())
.nest("/groups", groups::workspaced_service())
.nest("/inputs", inputs::workspaced_service())
.nest("/jobs", jobs::workspaced_service().layer(cors.clone()))
.nest("/oauth", oauth2::workspaced_service())
.nest("/resources", resources::workspaced_service())
.nest("/schedules", schedule::workspaced_service())
.nest("/scripts", scripts::workspaced_service())
.nest(
"/users",
users::workspaced_service().layer(Extension(argon2.clone())),
)
.nest("/variables", variables::workspaced_service())
.nest("/oauth", oauth2::workspaced_service())
.nest("/resources", resources::workspaced_service())
.nest("/schedules", schedule::workspaced_service())
.nest("/groups", groups::workspaced_service())
.nest("/audit", audit::workspaced_service())
.nest("/acls", granular_acls::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/apps", apps::workspaced_service())
.nest("/flows", flows::workspaced_service())
.nest("/capture", capture::workspaced_service())
.nest("/favorites", favorite::workspaced_service())
.nest("/folders", folders::workspaced_service()),
.nest("/workspaces", workspaces::workspaced_service()),
)
.nest("/workspaces", workspaces::global_service())
.nest(

View File

@@ -32,7 +32,8 @@ use windmill_audit::{audit_log, ActionKind};
use windmill_common::users::username_to_permissioned_as;
use windmill_common::utils::{not_found_if_none, now_from_db};
use crate::users::{truncate_token, Authed, NEW_USER_WEBHOOK};
use crate::users::{truncate_token, Authed};
use crate::webhook_util::{InstanceEvent, WebhookShared};
use crate::workspaces::invite_user_to_all_auto_invite_worspaces;
use crate::{
db::{UserDB, DB},
@@ -43,7 +44,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
use windmill_common::error::{self, to_anyhow, Error};
use windmill_common::oauth2::*;
use windmill_queue::JobPayload;
use windmill_queue::{JobPayload, QueueTransaction};
use std::{fs, str};
@@ -728,6 +729,7 @@ where
async fn slack_command(
SlackSig { sig, ts }: SlackSig,
Extension(db): Extension<DB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
body: Bytes,
) -> error::Result<String> {
let form: SlackCommand = serde_urlencoded::from_bytes(&body)
@@ -740,7 +742,7 @@ async fn slack_command(
}
}
let mut tx = db.begin().await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
let settings = sqlx::query_as!(
WorkspaceSettings,
"SELECT * FROM workspace_settings WHERE slack_team_id = $1",
@@ -756,7 +758,7 @@ async fn slack_command(
} else {
let path = path.strip_prefix("script/").unwrap_or_else(|| path);
let script_hash = windmill_common::get_latest_hash_for_path(
&mut tx,
tx.transaction_mut(),
&settings.workspace_id,
path,
)
@@ -788,14 +790,15 @@ async fn slack_command(
true,
)
.await?;
tx.commit().await?;
let url = BASE_URL.to_owned();
tx.commit().await?;
return Ok(format!(
"Job launched. See details at {url}/run/{uuid}?workspace={}",
&settings.workspace_id
));
}
}
tx.commit().await?;
return Ok(format!(
"workspace not properly configured (did you set the script to trigger in the settings?)"
@@ -815,6 +818,7 @@ async fn login_callback(
Path(client_name): Path<String>,
cookies: Cookies,
Extension(db): Extension<DB>,
Extension(webhook): Extension<WebhookShared>,
Json(callback): Json<OAuthCallback>,
) -> error::Result<String> {
let client_w_config = &OAUTH_CLIENTS
@@ -940,14 +944,7 @@ async fn login_callback(
}
tx.commit().await?;
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
let _ = HTTP_CLIENT
.post(&new_user_webhook)
.json(&serde_json::json!({"email" : &email, "event": "oauth_signup"}))
.send()
.await
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
}
webhook.send_instance_event(InstanceEvent::UserSignupOAuth { email: email.clone() });
Ok("Successfully logged in".to_string())
} else {

View File

@@ -25,7 +25,7 @@ use windmill_common::{
schedule::Schedule,
utils::{not_found_if_none, paginate, Pagination, StripPath},
};
use windmill_queue::{self, schedule::push_scheduled_job, JobKind};
use windmill_queue::{self, schedule::push_scheduled_job, JobKind, QueueTransaction};
pub fn workspaced_service() -> Router {
Router::new()
@@ -79,15 +79,23 @@ async fn create_schedule(
authed: Authed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path(w_id): Path<String>,
Json(ns): Json<NewSchedule>,
) -> Result<String> {
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let mut tx = user_db.begin(&authed).await?;
cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
check_path_conflict(&mut tx, &w_id, &ns.path).await?;
check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
check_flow_conflict(
tx.transaction_mut(),
&w_id,
&ns.path,
ns.is_flow,
&ns.script_path,
)
.await?;
let schedule = sqlx::query_as!(
Schedule,
@@ -139,14 +147,15 @@ async fn edit_schedule(
authed: Authed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(es): Json<EditSchedule>,
) -> Result<String> {
let path = path.to_path();
let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
let mut tx = user_db.begin(&authed).await?;
let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
(rsmq, user_db.begin(&authed).await?).into();
cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
@@ -158,7 +167,7 @@ async fn edit_schedule(
.fetch_one(&mut tx)
.await?;
clear_schedule(&mut tx, path, is_flow).await?;
clear_schedule(tx.transaction_mut(), path, is_flow).await?;
let schedule = sqlx::query_as!(
Schedule,
"UPDATE schedule SET schedule = $1, timezone = $2, args = $3 WHERE path \
@@ -173,10 +182,6 @@ async fn edit_schedule(
.await
.map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
if schedule.enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
audit_log(
&mut tx,
&authed.username,
@@ -192,6 +197,10 @@ async fn edit_schedule(
),
)
.await?;
if schedule.enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
tx.commit().await?;
Ok(path.to_string())
@@ -270,10 +279,12 @@ pub async fn preview_schedule(
pub async fn set_enabled(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<SetEnabled>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
(rsmq, user_db.begin(&authed).await?).into();
let path = path.to_path();
let schedule_o = sqlx::query_as!(
Schedule,
@@ -288,11 +299,8 @@ pub async fn set_enabled(
let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
clear_schedule(&mut tx, path, schedule.is_flow).await?;
clear_schedule(tx.transaction_mut(), path, schedule.is_flow).await?;
if payload.enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
audit_log(
&mut tx,
&authed.username,
@@ -303,7 +311,12 @@ pub async fn set_enabled(
Some([("enabled", payload.enabled.to_string().as_ref())].into()),
)
.await?;
if payload.enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
tx.commit().await?;
Ok(format!(
"succesfully updated schedule at path {} to status {}",
path, payload.enabled

View File

@@ -6,10 +6,6 @@
* LICENSE-AGPL for a copy of the license.
*/
use sql_builder::prelude::*;
use windmill_audit::{audit_log, ActionKind};
use windmill_parser::MainArgSignature;
use crate::{
db::{UserDB, DB},
schedule::clear_schedule,
@@ -25,6 +21,7 @@ use axum::{
use hyper::StatusCode;
use serde::Serialize;
use serde_json::json;
use sql_builder::prelude::*;
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Postgres, Transaction};
use std::{
@@ -32,6 +29,7 @@ use std::{
hash::{Hash, Hasher},
sync::Arc,
};
use windmill_audit::{audit_log, ActionKind};
use windmill_common::{
error::{Error, JsonResult, Result},
schedule::Schedule,
@@ -44,7 +42,8 @@ use windmill_common::{
list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
},
};
use windmill_queue::{self, schedule::push_scheduled_job};
use windmill_parser::MainArgSignature;
use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
@@ -85,6 +84,7 @@ pub fn workspaced_service() -> Router {
.route("/deployment_status/h/:hash", get(get_deployment_status))
.route("/list_paths", get(list_paths))
}
async fn list_scripts(
authed: Authed,
Extension(user_db): Extension<UserDB>,
@@ -193,6 +193,7 @@ fn hash_script(ns: &NewScript) -> i64 {
async fn create_script(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Extension(webhook): Extension<WebhookShared>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
@@ -200,14 +201,14 @@ async fn create_script(
) -> Result<(StatusCode, String)> {
let hash = ScriptHash(hash_script(&ns));
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
let mut tx = user_db.begin(&authed).await?;
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
if sqlx::query_scalar!(
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
hash.0,
&w_id
)
.fetch_optional(&mut tx)
.fetch_optional(tx.transaction_mut())
.await?
.is_some()
{
@@ -268,7 +269,7 @@ async fn create_script(
)));
};
let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
if ps.path != ns.path {
if !authed.is_admin {
@@ -363,7 +364,7 @@ async fn create_script(
.await?;
for schedule in schedulables {
clear_schedule(&mut tx, &schedule.path, false).await?;
clear_schedule(tx.transaction_mut(), &schedule.path, false).await?;
if schedule.enabled {
tx = push_scheduled_job(tx, schedule).await?;
@@ -371,36 +372,6 @@ async fn create_script(
}
}
let mut tx = if needs_lock_gen {
let dependencies = match ns.language {
ScriptLang::Python3 => {
windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
}
_ => ns.content,
};
let (_, tx) = windmill_queue::push(
tx,
&w_id,
windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
serde_json::Map::new(),
&authed.username,
&authed.email,
username_to_permissioned_as(&authed.username),
None,
None,
None,
None,
false,
false,
None,
true,
)
.await?;
tx
} else {
tx
};
if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
audit_log(
&mut tx,
@@ -415,7 +386,7 @@ async fn create_script(
webhook.send_message(
w_id.clone(),
WebhookMessage::UpdateScript {
workspace: w_id,
workspace: w_id.clone(),
path: ns.path.clone(),
hash: hash.to_string(),
},
@@ -440,13 +411,41 @@ async fn create_script(
webhook.send_message(
w_id.clone(),
WebhookMessage::CreateScript {
workspace: w_id,
workspace: w_id.clone(),
path: ns.path.clone(),
hash: hash.to_string(),
},
);
}
if needs_lock_gen {
let dependencies = match ns.language {
ScriptLang::Python3 => {
windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
}
_ => ns.content,
};
let (_, new_tx) = windmill_queue::push(
tx,
&w_id,
windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
serde_json::Map::new(),
&authed.username,
&authed.email,
username_to_permissioned_as(&authed.username),
None,
None,
None,
None,
false,
false,
None,
true,
)
.await?;
tx = new_tx;
}
tx.commit().await?;
Ok((StatusCode::CREATED, format!("{}", hash)))

View File

@@ -12,8 +12,9 @@ use crate::{
db::{UserDB, DB},
folders::get_folders_for_user,
utils::require_super_admin,
webhook_util::{InstanceEvent, WebhookShared},
workspaces::invite_user_to_all_auto_invite_worspaces,
COOKIE_DOMAIN, HTTP_CLIENT, IS_SECURE,
COOKIE_DOMAIN, IS_SECURE,
};
use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
use axum::{
@@ -1037,6 +1038,7 @@ lazy_static! {
async fn accept_invite(
Authed { email, .. }: Authed,
Extension(webhook): Extension<WebhookShared>,
Extension(db): Extension<DB>,
Json(nu): Json<AcceptInvite>,
) -> Result<(StatusCode, String)> {
@@ -1082,6 +1084,11 @@ async fn accept_invite(
}
if is_some {
webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace {
email: email.clone(),
workspace: nu.workspace_id.clone(),
username: nu.username.clone(),
});
Ok((
StatusCode::CREATED,
format!(
@@ -1322,6 +1329,7 @@ lazy_static::lazy_static! {
async fn create_user(
Authed { email, .. }: Authed,
Extension(db): Extension<DB>,
Extension(webhook): Extension<WebhookShared>,
Extension(argon2): Extension<Arc<Argon2<'_>>>,
Json(mut nu): Json<NewUser>,
) -> Result<(StatusCode, String)> {
@@ -1362,19 +1370,9 @@ async fn create_user(
.await?;
tx.commit().await?;
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
let _ = HTTP_CLIENT
.post(&new_user_webhook)
.json(
&serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "global_add"}),
)
.send()
.await
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
}
invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
webhook.send_instance_event(InstanceEvent::UserAdded { email: nu.email.clone() });
Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
}

View File

@@ -13,6 +13,26 @@ lazy_static::lazy_static! {
"Histogram of webhook requests made"
)
.unwrap();
pub static ref INSTANCE_EVENTS_WEBHOOK: Option<String> = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok();
}
pub enum WebhookPayload {
WorkspaceEvent(String, WebhookMessage),
InstanceEvent(InstanceEvent),
}
#[derive(Serialize)]
#[serde(tag = "type")]
pub enum InstanceEvent {
UserSignupOAuth { email: String },
UserAdded { email: String },
// UserDeleted { email: String },
// UserDeletedWorkspace { workspace: String, email: String },
UserAddedWorkspace { workspace: String, email: String },
UserInvitedWorkspace { workspace: String, email: String },
UserJoinedWorkspace { workspace: String, email: String, username: String },
}
#[derive(Serialize)]
@@ -46,12 +66,12 @@ pub enum WebhookMessage {
#[derive(Clone)]
pub struct WebhookShared {
pub channel: mpsc::UnboundedSender<(String, WebhookMessage)>,
pub channel: mpsc::UnboundedSender<WebhookPayload>,
}
impl WebhookShared {
pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self {
let (tx, mut rx) = mpsc::unbounded_channel::<(String, WebhookMessage)>();
let (tx, mut rx) = mpsc::unbounded_channel::<WebhookPayload>();
let _process = tokio::spawn(async move {
let client = reqwest::Client::builder()
// TODO: investigate pool timeouts and such if TCP load is high
@@ -66,7 +86,7 @@ impl WebhookShared {
biased;
_ = shutdown_rx.recv() => break,
r = rx.recv() => match r {
Some((workspace_id, message)) => {
Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
let url_guard = match cache.get(&workspace_id).await {
Some(guard) => {
guard
@@ -96,6 +116,13 @@ impl WebhookShared {
drop(url_guard);
}
},
Some(WebhookPayload::InstanceEvent(event)) => {
if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await;
if let Err(e) = r {
tracing::error!("Error sending instance event: {}", e);
}
},
None => break,
},
_ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
@@ -110,6 +137,16 @@ impl WebhookShared {
}
pub fn send_message(&self, workspace_id: String, message: WebhookMessage) {
let _ = self.channel.send((workspace_id.clone(), message));
let _ = self.channel.send(WebhookPayload::WorkspaceEvent(
workspace_id.clone(),
message,
));
}
pub fn send_instance_event(&self, event: InstanceEvent) {
if INSTANCE_EVENTS_WEBHOOK.is_none() {
return;
}
let _ = self.channel.send(WebhookPayload::InstanceEvent(event));
}
}

View File

@@ -16,10 +16,10 @@ use crate::{
db::{UserDB, DB},
folders::Folder,
resources::{Resource, ResourceType},
users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK, VALID_USERNAME},
users::{Authed, WorkspaceInvite, VALID_USERNAME},
utils::require_super_admin,
variables::build_crypt,
HTTP_CLIENT,
webhook_util::{InstanceEvent, WebhookShared},
};
#[cfg(feature = "enterprise")]
use axum::response::Redirect;
@@ -943,6 +943,7 @@ pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> R
async fn invite_user(
Authed { username, is_admin, .. }: Authed,
Extension(db): Extension<DB>,
Extension(webhook): Extension<WebhookShared>,
Path(w_id): Path<String>,
Json(mut nu): Json<NewWorkspaceInvite>,
) -> Result<(StatusCode, String)> {
@@ -966,14 +967,10 @@ async fn invite_user(
tx.commit().await?;
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
let _ = &HTTP_CLIENT
.post(&new_user_webhook)
.json(&serde_json::json!({"email" : &nu.email, "event": "workspace_invite"}))
.send()
.await
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
}
webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace {
email: nu.email.clone(),
workspace: w_id,
});
Ok((
StatusCode::CREATED,
@@ -984,6 +981,7 @@ async fn invite_user(
async fn add_user(
Authed { username, is_admin, .. }: Authed,
Extension(db): Extension<DB>,
Extension(webhook): Extension<WebhookShared>,
Path(w_id): Path<String>,
Json(mut nu): Json<NewWorkspaceUser>,
) -> Result<(StatusCode, String)> {
@@ -1022,14 +1020,10 @@ async fn add_user(
tx.commit().await?;
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
let _ = HTTP_CLIENT
.post(&new_user_webhook)
.json(&serde_json::json!({"email" : &nu.email, "event": "workspace_add"}))
.send()
.await
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
}
webhook.send_instance_event(InstanceEvent::UserAddedWorkspace {
workspace: w_id.clone(),
email: nu.email.clone(),
});
Ok((
StatusCode::CREATED,

View File

@@ -41,8 +41,8 @@ pub struct AuditLog {
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn audit_log<'c>(
db: &mut Transaction<'c, Postgres>,
pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
db: E,
username: &str,
operation: &str,
action_kind: ActionKind,

View File

@@ -17,7 +17,7 @@ use serde_json::to_string_pretty;
use crate::utils::StripPath;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq)]
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
#[cfg_attr(
feature = "sqlx",

View File

@@ -6,11 +6,10 @@
* LICENSE-AGPL for a copy of the license.
*/
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use crate::error::{Error, Result};
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
pub const MAX_PER_PAGE: usize = 10000;
pub const DEFAULT_PER_PAGE: usize = 1000;
@@ -20,7 +19,8 @@ pub struct Pagination {
pub page: Option<usize>,
pub per_page: Option<usize>,
}
#[derive(Deserialize)]
#[derive(Debug, Serialize, Deserialize)]
pub struct StripPath(pub String);
impl StripPath {
@@ -52,8 +52,8 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
}
#[cfg(feature = "sqlx")]
pub async fn now_from_db<'c>(
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
db: E,
) -> Result<chrono::DateTime<chrono::Utc>> {
Ok(sqlx::query_scalar!("SELECT now()")
.fetch_one(db)

View File

@@ -31,3 +31,6 @@ reqwest.workspace = true
lazy_static.workspace = true
prometheus.workspace = true
cron.workspace = true
rsmq_async.workspace = true
tokio.workspace = true
futures-core.workspace = true

View File

@@ -25,6 +25,8 @@ use windmill_common::{
METRICS_ENABLED,
};
use crate::QueueTransaction;
lazy_static::lazy_static! {
pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
@@ -52,13 +54,15 @@ lazy_static::lazy_static! {
const MAX_FREE_EXECS: i32 = 1000;
const MAX_FREE_CONCURRENT_RUNS: i32 = 15;
const RSMQ_MAIN_QUEUE: &'static str = "main_queue";
pub async fn cancel_job<'c>(
pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
username: &str,
reason: Option<String>,
id: Uuid,
w_id: &str,
mut tx: Transaction<'c, Postgres>,
rsmq: Option<R>,
force_rerun: bool,
) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
let job_option = sqlx::query_scalar!(
@@ -72,6 +76,12 @@ pub async fn cancel_job<'c>(
)
.fetch_optional(&mut tx)
.await?;
if let Some(mut rsmq) = rsmq {
rsmq.change_message_visibility(RSMQ_MAIN_QUEUE, &id.to_string(), 0)
.await
.map_err(|e| anyhow::anyhow!(e))?;
}
let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default();
while !jobs.is_empty() {
let p_job = jobs.pop();
@@ -91,10 +101,11 @@ pub async fn cancel_job<'c>(
Ok((tx, job_option))
}
pub async fn pull(
pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
db: &Pool<Postgres>,
whitelist_workspaces: Option<Vec<String>>,
blacklist_workspaces: Option<Vec<String>>,
rsmq: Option<R>,
) -> windmill_common::error::Result<Option<QueuedJob>> {
let mut workspaces_filter = String::new();
if let Some(whitelist) = whitelist_workspaces {
@@ -106,6 +117,9 @@ pub async fn pull(
.collect::<Vec<String>>()
.join(",")
));
if let Some(_rsmq) = rsmq {
todo!("REDIS: Implement workspace filters for redis");
}
}
if let Some(blacklist) = blacklist_workspaces {
workspaces_filter.push_str(&format!(
@@ -116,16 +130,50 @@ pub async fn pull(
.collect::<Vec<String>>()
.join(",")
));
if let Some(_rsmq) = rsmq {
todo!("REDIS: Implement workspace filters for redis");
}
}
/* Jobs can be started if they:
* - haven't been started before,
* running = false
* - are flows with a step that needed resume,
* suspend_until is non-null
* and suspend = 0 when the resume messages are received
* or suspend_until <= now() if it has timed out */
let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(&format!(
"UPDATE queue
let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
// TODO: REDIS: Race conditions / replace last_ping
let msg = rsmq
.pop_message::<Vec<u8>>(RSMQ_MAIN_QUEUE)
.await
.map_err(|e| anyhow::anyhow!(e))?;
if let Some(msg) = msg {
let uuid = Uuid::from_bytes_le(
msg.message
.try_into()
.map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?,
);
sqlx::query_as::<_, QueuedJob>(
"UPDATE queue
SET running = true
, started_at = coalesce(started_at, now())
, last_ping = now()
, suspend_until = null
WHERE id = $1
RETURNING *",
)
.bind(uuid)
.fetch_optional(db)
.await?
} else {
None
}
} else {
/* Jobs can be started if they:
* - haven't been started before,
* running = false
* - are flows with a step that needed resume,
* suspend_until is non-null
* and suspend = 0 when the resume messages are received
* or suspend_until <= now() if it has timed out */
sqlx::query_as::<_, QueuedJob>(&format!(
"UPDATE queue
SET running = true
, started_at = coalesce(started_at, now())
, last_ping = now()
@@ -143,9 +191,10 @@ pub async fn pull(
LIMIT 1
)
RETURNING *"
))
.fetch_optional(db)
.await?;
))
.fetch_optional(db)
.await?
};
if job.is_some() && *METRICS_ENABLED {
QUEUE_PULL_COUNT.inc();
@@ -207,11 +256,11 @@ pub async fn get_result_by_id(
}
#[instrument(level = "trace", skip_all)]
pub async fn delete_job(
db: &Pool<Postgres>,
pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
mut tx: QueueTransaction<'c, R>,
w_id: &str,
job_id: Uuid,
) -> windmill_common::error::Result<()> {
) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
if *METRICS_ENABLED {
QUEUE_DELETE_COUNT.inc();
}
@@ -220,13 +269,13 @@ pub async fn delete_job(
w_id,
job_id
)
.fetch_one(db)
.fetch_one(&mut tx)
.await
.map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
.unwrap_or(0)
== 1;
tracing::debug!("Job {job_id} deleted: {job_removed}");
Ok(())
Ok(tx)
}
pub async fn get_queued_job<'c>(
@@ -245,9 +294,9 @@ pub async fn get_queued_job<'c>(
Ok(r)
}
#[instrument(level = "trace", skip_all)]
pub async fn push<'c>(
mut tx: Transaction<'c, Postgres>,
// #[instrument(level = "trace", skip_all)]
pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
mut tx: QueueTransaction<'c, R>,
workspace_id: &str,
job_payload: JobPayload,
args: serde_json::Map<String, serde_json::Value>,
@@ -262,8 +311,7 @@ pub async fn push<'c>(
mut same_worker: bool,
pre_run_error: Option<&windmill_common::error::Error>,
visible_to_owner: bool,
) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now);
) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
let args_json = serde_json::Value::Object(args);
let job_id: Uuid = Ulid::new().into();
@@ -515,7 +563,7 @@ pub async fn push<'c>(
(workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for,
script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
RETURNING id",
workspace_id,
job_id,
@@ -523,7 +571,7 @@ pub async fn push<'c>(
parent_job,
user,
permissioned_as,
scheduled_for,
scheduled_for_o,
script_hash,
script_path.clone(),
raw_code,
@@ -582,6 +630,10 @@ pub async fn push<'c>(
.instrument(tracing::info_span!("job_run", email = &email))
.await?;
}
if let Some(ref mut rsmq) = tx.rsmq {
rsmq.send_message(job_id.to_bytes_le().to_vec(), scheduled_for_o);
}
Ok((uuid, tx))
}

View File

@@ -7,6 +7,8 @@
*/
mod jobs;
mod queue_transaction;
pub mod schedule;
pub use jobs::*;
pub use queue_transaction::*;

View File

@@ -0,0 +1,161 @@
use std::fmt::Debug;
use futures_core::{future::BoxFuture, stream::BoxStream};
use rsmq_async::{RedisBytes, RsmqConnection};
use sqlx::{Postgres, Transaction};
pub enum RedisOp {
SendMessage(RedisBytes, Option<chrono::DateTime<chrono::Utc>>),
DeleteMessage(String),
}
impl RedisOp {
pub async fn apply<R: RsmqConnection>(self, rsmq: &mut R) -> Result<(), rsmq_async::RsmqError> {
match self {
RedisOp::SendMessage(bytes, time) => {
rsmq.send_message(
"main_queue",
bytes,
time.map(|t| (t - chrono::Utc::now()).num_seconds())
.and_then(|e| e.try_into().ok()),
)
.await?;
}
RedisOp::DeleteMessage(id) => {
rsmq.delete_message("main_queue", &id).await?;
}
};
Ok(())
}
}
pub struct RedisTransaction<R: RsmqConnection> {
rsmq: R,
queued_ops: Vec<RedisOp>,
}
impl<R: RsmqConnection> From<R> for RedisTransaction<R> {
fn from(value: R) -> Self {
Self { rsmq: value, queued_ops: Vec::new() }
}
}
impl<R: RsmqConnection> RedisTransaction<R> {
pub async fn commit(self) -> Result<(), rsmq_async::RsmqError> {
let mut rsmq = self.rsmq;
for op in self.queued_ops {
op.apply(&mut rsmq).await?;
}
Ok(())
}
pub fn send_message<E: Into<RedisBytes>>(
&mut self,
bytes: E,
delay_until: Option<chrono::DateTime<chrono::Utc>>,
) {
self.queued_ops
.push(RedisOp::SendMessage(bytes.into(), delay_until))
}
pub fn delete_message(&mut self, id: String) {
self.queued_ops.push(RedisOp::DeleteMessage(id))
}
}
pub struct QueueTransaction<'c, R: RsmqConnection> {
pub rsmq: Option<RedisTransaction<R>>,
transaction: Transaction<'c, Postgres>,
}
impl<'c, R: RsmqConnection> From<(Option<R>, Transaction<'c, Postgres>)>
for QueueTransaction<'c, R>
{
fn from(value: (Option<R>, Transaction<'c, Postgres>)) -> Self {
Self { rsmq: value.0.map(|e| e.into()), transaction: value.1 }
}
}
impl<'c, R: RsmqConnection> Debug for QueueTransaction<'c, R> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("QueueTransaction")
.field("rsmq", &self.rsmq.as_ref().map(|_| ())) // do not require R: Debug
.field("transaction", &self.transaction)
.finish()
}
}
impl<'c, R: RsmqConnection> QueueTransaction<'c, R> {
pub async fn commit(self) -> Result<(), windmill_common::error::Error> {
self.transaction.commit().await?;
if let Some(rsmq) = self.rsmq {
rsmq.commit().await.map_err(|e| anyhow::anyhow!(e))?;
}
Ok(())
}
pub fn transaction_mut<'a>(&'a mut self) -> &'a mut Transaction<'c, Postgres> {
&mut self.transaction
}
}
impl<'c, 'b, R: RsmqConnection + Send> sqlx::Executor<'b> for &'b mut QueueTransaction<'c, R> {
type Database = Postgres;
fn fetch_many<'e, 'q: 'e, E: 'q>(
self,
query: E,
) -> BoxStream<
'e,
Result<
sqlx::Either<
<Self::Database as sqlx::Database>::QueryResult,
<Self::Database as sqlx::Database>::Row,
>,
sqlx::Error,
>,
>
where
'b: 'e,
E: sqlx::Execute<'q, Self::Database>,
{
self.transaction.fetch_many(query)
}
fn fetch_optional<'e, 'q: 'e, E: 'q>(
self,
query: E,
) -> BoxFuture<'e, Result<Option<<Self::Database as sqlx::Database>::Row>, sqlx::Error>>
where
'b: 'e,
E: sqlx::Execute<'q, Self::Database>,
{
self.transaction.fetch_optional(query)
}
fn prepare_with<'e, 'q: 'e>(
self,
sql: &'q str,
parameters: &'e [<Self::Database as sqlx::Database>::TypeInfo],
) -> BoxFuture<
'e,
Result<<Self::Database as sqlx::database::HasStatement<'q>>::Statement, sqlx::Error>,
>
where
'b: 'e,
{
self.transaction.prepare_with(sql, parameters)
}
fn describe<'e, 'q: 'e>(
self,
sql: &'q str,
) -> BoxFuture<'e, Result<sqlx::Describe<Self::Database>, sqlx::Error>>
where
'b: 'e,
{
self.transaction.describe(sql)
}
}

View File

@@ -15,11 +15,12 @@ use windmill_common::{
users::username_to_permissioned_as,
utils::{now_from_db, StripPath},
};
use crate::{QueueTransaction};
pub async fn push_scheduled_job<'c>(
mut tx: Transaction<'c, Postgres>,
pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
mut tx: QueueTransaction<'c, R>,
schedule: Schedule,
) -> Result<Transaction<'c, Postgres>> {
) -> Result<QueueTransaction<'c, R>> {
let sched = cron::Schedule::from_str(&schedule.schedule)
.map_err(|e| error::Error::BadRequest(e.to_string()))?;
@@ -70,7 +71,7 @@ pub async fn push_scheduled_job<'c>(
} else {
JobPayload::ScriptHash {
hash: windmill_common::get_latest_hash_for_path(
&mut tx,
tx.transaction_mut(),
&schedule.workspace_id,
&schedule.script_path,
)
@@ -79,7 +80,15 @@ pub async fn push_scheduled_job<'c>(
}
};
let (_, mut tx) = push(
sqlx::query!(
"UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
&schedule.workspace_id,
&schedule.path
)
.execute(&mut tx)
.await?;
let (_, tx) = push(
tx,
&schedule.workspace_id,
payload,
@@ -97,14 +106,7 @@ pub async fn push_scheduled_job<'c>(
true,
)
.await?;
sqlx::query!(
"UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
&schedule.workspace_id,
&schedule.path
)
.execute(&mut tx)
.await?;
Ok(tx)
Ok(tx) // TODO: Bubble up pushed UUID from here
}
pub async fn get_schedule_opt<'c>(

View File

@@ -48,4 +48,5 @@ deno_core.workspace = true
const_format.workspace = true
git-version.workspace = true
dyn-iter.workspace = true
once_cell.workspace = true
once_cell.workspace = true
rsmq_async.workspace = true

View File

@@ -6,27 +6,30 @@
* LICENSE-AGPL for a copy of the license.
*/
use sqlx::{Pool, Postgres, Transaction};
use sqlx::{Pool, Postgres};
use tracing::instrument;
use uuid::Uuid;
use windmill_common::{
error::Error, flow_status::FlowStatusModule, schedule::Schedule, METRICS_ENABLED,
};
use windmill_queue::{delete_job, schedule::get_schedule_opt, JobKind, QueuedJob, CLOUD_HOSTED};
use windmill_queue::{
delete_job, schedule::get_schedule_opt, JobKind, QueueTransaction, QueuedJob, CLOUD_HOSTED,
};
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job_error(
pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
logs: String,
e: serde_json::Value,
metrics: Option<crate::worker::Metrics>,
rsmq: Option<R>,
) -> Result<serde_json::Value, Error> {
if *METRICS_ENABLED {
metrics.map(|m| m.worker_execution_failed.inc());
}
let result = serde_json::json!({ "error": e });
let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs).await?;
let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs, rsmq).await?;
Ok(result)
}
@@ -49,13 +52,14 @@ fn flatten_jobs(modules: Vec<FlowStatusModule>) -> Vec<Uuid> {
}
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job(
pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
success: bool,
skipped: bool,
result: serde_json::Value,
logs: String,
rsmq: Option<R>,
) -> Result<Uuid, Error> {
let duration =
if queued_job.job_kind == JobKind::Flow || queued_job.job_kind == JobKind::FlowPreview {
@@ -87,7 +91,7 @@ pub async fn add_completed_job(
.ok()
.flatten()
.flatten();
let mut tx = db.begin().await?;
let mut tx: QueueTransaction<'_, R> = (rsmq, db.begin().await?).into();
let job_id = queued_job.id.clone();
sqlx::query!(
"INSERT INTO completed_job AS cj
@@ -157,7 +161,7 @@ pub async fn add_completed_job(
.execute(&mut tx)
.await
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
let _ = delete_job(db, &queued_job.workspace_id, job_id).await?;
tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
if !queued_job.is_flow_step
&& queued_job.job_kind != JobKind::Flow
&& queued_job.job_kind != JobKind::FlowPreview
@@ -212,21 +216,24 @@ pub async fn add_completed_job(
}
#[instrument(level = "trace", skip_all)]
pub async fn schedule_again_if_scheduled<'c>(
mut tx: Transaction<'c, Postgres>,
pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
mut tx: QueueTransaction<'c, R>,
db: &Pool<Postgres>,
schedule_path: &str,
script_path: &str,
w_id: &str,
) -> windmill_common::error::Result<Transaction<'c, Postgres>> {
let schedule = get_schedule_opt(&mut tx, w_id, schedule_path)
.await?
.ok_or_else(|| {
Error::InternalErr(format!(
"Could not find schedule {:?} for workspace {}",
schedule_path, w_id
))
})?;
) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;
if schedule.is_none() {
tracing::error!(
"Schedule {schedule_path} in {w_id} not found. Impossible to schedule again"
);
return Ok(tx);
}
let schedule = schedule.unwrap();
if schedule.enabled && script_path == schedule.script_path {
let res = windmill_queue::schedule::push_scheduled_job(
tx,

View File

@@ -17,7 +17,8 @@ use windmill_api_client::Client;
use windmill_parser_go::parse_go_imports;
use std::{
borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic,
process::Stdio, time::Duration, sync::{atomic::Ordering, Arc},
process::Stdio, time::{Duration}, sync::atomic::Ordering,
sync::{Arc},
};
use tracing::{trace_span, Instrument};
use uuid::Uuid;
@@ -78,7 +79,7 @@ async fn copy_cache_from_bucket(bucket: &str, tx: Option<Sender<()>>) -> Option:
.arg("--size-only")
.arg("--fast-list")
.arg("--exclude")
.arg(format!("\"{TAR_CACHE_FILENAME}\""))
.arg(format!("\"{TAR_CACHE_FILENAME},/deno/gen/file/**\""))
.stdin(Stdio::null())
.stdout(Stdio::null())
.spawn()
@@ -127,7 +128,7 @@ async fn copy_cache_to_bucket(bucket: &str) {
.arg("--size-only")
.arg("--fast-list")
.arg("--exclude")
.arg(format!("\"{TAR_CACHE_FILENAME}\""))
.arg(format!("\"{TAR_CACHE_FILENAME},/deno/gen/file/**\""))
.stdin(Stdio::null())
.stdout(Stdio::null())
.spawn()
@@ -410,6 +411,8 @@ lazy_static::lazy_static! {
static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new());
static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| String::new());
static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
static ref GOPRIVATE: Option<String> = std::env::var("GOPRIVATE").ok();
static ref NETRC: Option<String> = std::env::var("NETRC").ok();
static ref PIP_EXTRA_INDEX_URL: Option<String> = std::env::var("PIP_EXTRA_INDEX_URL").ok();
static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
static ref DENO_AUTH_TOKENS: String = std::env::var("DENO_AUTH_TOKENS")
@@ -532,8 +535,8 @@ impl AuthedClient {
}
#[tracing::instrument(level = "trace")]
pub async fn run_worker(
#[tracing::instrument(skip(rsmq), level = "trace")]
pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
db: &Pool<Postgres>,
worker_instance: &str,
worker_name: String,
@@ -541,8 +544,8 @@ pub async fn run_worker(
ip: &str,
mut rx: tokio::sync::broadcast::Receiver<()>,
base_internal_url: &str,
rsmq: Option<R>,
) {
#[cfg(not(feature = "enterprise"))]
if !*DISABLE_NSJAIL {
tracing::warn!(
@@ -580,30 +583,33 @@ pub async fn run_worker(
.unwrap();
let worker_execution_duration = prometheus::register_histogram_vec!(
let all_langs = [
None,
Some(ScriptLang::Python3),
Some(ScriptLang::Deno),
Some(ScriptLang::Go),
Some(ScriptLang::Bash)];
let worker_execution_duration: HashMap<_, _> = all_langs.clone().into_iter().map(|x| (x.clone(), prometheus::register_histogram!(
prometheus::HistogramOpts::new(
"worker_execution_duration",
"Duration between receiving a job and completing it",
)
.const_label("name", &worker_name),
&["workspace_id", "language"],
)
.expect("register prometheus metric");
let worker_execution_duration_counter = prometheus::register_counter!(prometheus::opts!(
"worker_execution_duration_counter",
"Total number of seconds spent executing jobs"
)
.const_label("name", &worker_name))
.expect("register prometheus metric");
.const_label("name", &worker_name)
.const_label("language", x.map(|x| x.as_str()).unwrap_or("none")))
.expect("register prometheus metric"))
).collect();
let worker_sleep_duration = prometheus::register_histogram!(prometheus::HistogramOpts::new(
"worker_sleep_duration",
"Duration sleeping waiting for job",
)
.const_label("name", &worker_name),)
.expect("register prometheus metric");
let worker_execution_duration_counter: HashMap<_, _> = all_langs.clone().into_iter().map(|x| (x.clone(), prometheus::register_counter!(
prometheus::Opts::new(
"worker_execution_duration_counter",
"Total number of seconds spent executing jobs"
)
.const_label("name", &worker_name)
.const_label("language", x.map(|x| x.as_str()).unwrap_or("none")))
.expect("register prometheus metric"))
).collect();
let worker_sleep_duration_counter = prometheus::register_counter!(prometheus::opts!(
@@ -628,19 +634,24 @@ pub async fn run_worker(
.const_label("name", &worker_name))
.expect("register prometheus metric");
let worker_execution_failed = prometheus::register_int_counter_vec!(
prometheus::Opts::new("worker_execution_failed", "Number of failed jobs",)
.const_label("name", &worker_name),
&["workspace_id", "language"],
)
.expect("register prometheus metric");
let worker_execution_failed: HashMap<_, _> = all_langs.clone().into_iter().map(|x| (x.clone(), prometheus::register_int_counter!(
prometheus::Opts::new(
"worker_execution_failed", "Number of failed jobs",
)
.const_label("name", &worker_name)
.const_label("language", x.map(|x| x.as_str()).unwrap_or("none")))
.expect("register prometheus metric"))
).collect();
let worker_execution_count = prometheus::register_int_counter_vec!(
prometheus::Opts::new("worker_execution_count", "Number of executed jobs",)
.const_label("name", &worker_name),
&["workspace_id", "language"],
)
.expect("register prometheus metric");
let worker_execution_count: HashMap<_, _> = all_langs.into_iter().map(|x| (x.clone(), prometheus::register_int_counter!(
prometheus::Opts::new(
"worker_execution_count", "Number of executed jobs"
)
.const_label("name", &worker_name)
.const_label("language", x.map(|x| x.as_str()).unwrap_or("none")))
.expect("register prometheus metric"))
).collect();
let worker_busy: prometheus::IntGauge = prometheus::register_int_gauge!(prometheus::Opts::new(
"worker_busy",
@@ -689,7 +700,7 @@ pub async fn run_worker(
if *METRICS_ENABLED {
worker_busy.set(0);
uptime_metric.inc_by(
(((Instant::now() - start_time).as_millis() as f64)/1000.0 - uptime_metric.get())
((start_time.elapsed().as_millis() as f64)/1000.0 - uptime_metric.get())
.try_into()
.unwrap(),
);
@@ -751,7 +762,8 @@ pub async fn run_worker(
},
(job, timer) = {
let timer = if *METRICS_ENABLED { Some(worker_pull_duration.start_timer()) } else { None };
pull(&db, WHITELIST_WORKSPACES.clone(), BLACKLIST_WORKSPACES.clone()).map(|x| (x, timer)) } => {
pull(&db, WHITELIST_WORKSPACES.clone(), BLACKLIST_WORKSPACES.clone(), rsmq.clone()).map(|x| (x, timer))
} => {
timer.map(|timer| {
let duration_pull_s = timer.stop_and_record();
worker_pull_duration_counter.inc_by(duration_pull_s);
@@ -760,6 +772,7 @@ pub async fn run_worker(
},
}
}.instrument(trace_span!("worker_get_next_job")).await;
if do_break {
return true;
}
@@ -768,28 +781,30 @@ pub async fn run_worker(
}
match next_job {
Ok(Some(job)) => {
// println!("{:?}", SystemTime::now());
let token = create_token_for_owner_in_bg(&db, &job).await;
let label_values = [
&job.workspace_id,
job.language.as_ref().map(|l| l.as_str()).unwrap_or(""),
];
let language = job.language.clone();
let _timer = worker_execution_duration
.with_label_values(label_values.as_slice())
.get(&language)
.expect("no timer found")
.start_timer();
jobs_executed += 1;
if *METRICS_ENABLED {
worker_execution_count
.with_label_values(label_values.as_slice())
.get(&language)
.expect("no timer found")
.inc();
}
let metrics = Metrics {
worker_execution_failed: worker_execution_failed
.with_label_values(label_values.as_slice()),
};
let metrics = if *METRICS_ENABLED {
Some(Metrics {
worker_execution_failed: worker_execution_failed
.get(&language)
.expect("no timer found").clone(),
})
} else { None };
let job_root = job.root_job.map(|x| x.to_string()).unwrap_or_else(|| "none".to_string());
tracing::info!(worker = %worker_name, id = %job.id, root_id = %job_root, "fetched job {}, root job: {}", job.id, job_root);
@@ -837,7 +852,8 @@ pub async fn run_worker(
&job_dir,
metrics.clone(),
same_worker_tx.clone(),
base_internal_url
base_internal_url,
rsmq.clone()
)
.await
.err()
@@ -847,27 +863,30 @@ pub async fn run_worker(
&authed_client.get_authed().await,
job,
err,
Some(metrics),
metrics,
false,
same_worker_tx.clone(),
&worker_dir,
base_internal_url
base_internal_url,
rsmq.clone()
)
.await;
};
let duration = _timer.stop_and_record();
worker_execution_duration_counter.inc_by(duration);
worker_execution_duration_counter
.get(&language)
.expect("no timer found").inc_by(duration);
if !*KEEP_JOB_DIR && !(is_flow && same_worker) {
let _ = tokio::fs::remove_dir_all(job_dir).await;
}
}
Ok(None) => {
let _timer = if *METRICS_ENABLED { Some(worker_sleep_duration.start_timer()) } else { None };
let _timer = if *METRICS_ENABLED { Some(Instant::now()) } else { None };
tokio::time::sleep(Duration::from_millis(*SLEEP_QUEUE)).await;
_timer.map(|timer| {
let duration = timer.stop_and_record();
let duration = timer.elapsed().as_secs_f64();
worker_sleep_duration_counter.inc_by(duration);
});
}
@@ -886,7 +905,7 @@ pub async fn run_worker(
}
}
async fn handle_job_error(
async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
db: &Pool<Postgres>,
client: &AuthedClient,
job: QueuedJob,
@@ -896,12 +915,14 @@ async fn handle_job_error(
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
base_internal_url: &str,
rsmq: Option<R>,
) {
let err = match err {
Error::JsonErr(err) => err,
_ => json!({"message": err.to_string(), "name": "InternalErr"}),
};
let rsmq_2 = rsmq.clone();
let update_job_future = || {
add_completed_job_error(
db,
@@ -909,15 +930,16 @@ async fn handle_job_error(
format!("Unexpected error during job execution:\n{err}"),
err.clone(),
metrics.clone(),
rsmq_2,
)
};
if job.is_flow_step || job.job_kind == JobKind::FlowPreview || job.job_kind == JobKind::Flow {
let (flow, job_status_to_update) = if let Some(parent_job_id) = job.parent_job {
let update_job_future = if job.is_flow_step || job.job_kind == JobKind::FlowPreview || job.job_kind == JobKind::Flow {
let (flow, job_status_to_update, update_job_future) = if let Some(parent_job_id) = job.parent_job {
let _ = update_job_future().await;
(parent_job_id, job.id)
(parent_job_id, job.id, None)
} else {
(job.id, Uuid::nil())
(job.id, Uuid::nil(), Some(update_job_future))
};
let updated_flow = update_flow_status_after_job_completion(
@@ -934,7 +956,7 @@ async fn handle_job_error(
worker_dir,
None,
base_internal_url,
rsmq.clone(),
)
.await;
@@ -950,15 +972,20 @@ async fn handle_job_error(
format!("Unexpected error during flow job error handling:\n{err}"),
json!({"message": err.to_string(), "name": "InternalErr"}),
metrics.clone(),
rsmq,
)
.await;
}
}
}
}
}
if job.parent_job.is_none() {
let _ = update_job_future().await;
update_job_future
} else {
Some(update_job_future)
};
if let Some(f) = update_job_future {
let _ = f().await;
}
tracing::error!(job_id = %job.id, "error handling job: {err:#?} {} {} {}", job.id, job.workspace_id, job.created_by);
}
@@ -982,19 +1009,20 @@ async fn insert_initial_ping(
fn extract_error_value(log_lines: &str, i: i32) -> serde_json::Value {
return json!({"message": format!("ExitCode: {i}, last log lines: {}", log_lines.to_string().trim().to_string()), "name": "ExecutionErr"});
return json!({"message": format!("ExitCode: {i}, last log lines:\n{}", log_lines.to_string().trim().to_string()), "name": "ExecutionErr"});
}
#[tracing::instrument(level = "trace", skip_all)]
async fn handle_queued_job(
async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
job: QueuedJob,
db: &sqlx::Pool<sqlx::Postgres>,
client: &AuthedClientBackgroundTask,
worker_name: &str,
worker_dir: &str,
job_dir: &str,
metrics: Metrics,
metrics: Option<Metrics>,
same_worker_tx: Sender<Uuid>,
base_internal_url: &str,
rsmq: Option<R>,
) -> windmill_common::error::Result<()> {
if job.canceled {
return Err(Error::JsonErr(canceled_job_to_result(&job)))?;
@@ -1013,12 +1041,14 @@ async fn handle_queued_job(
args,
same_worker_tx,
worker_dir,
base_internal_url
base_internal_url,
rsmq,
)
.await?;
}
_ => {
let mut logs = "".to_string();
// println!("handle queue {:?}", SystemTime::now());
if let Some(log_str) = &job.logs {
logs.push_str(&log_str);
}
@@ -1078,7 +1108,8 @@ async fn handle_queued_job(
let client = &client.get_authed().await;
match result {
Ok(r) => {
add_completed_job(db, &job, true, false, r.clone(), logs).await?;
// println!("bef completed job{:?}", SystemTime::now());
add_completed_job(db, &job, true, false, r.clone(), logs, rsmq.clone()).await?;
if job.is_flow_step {
if let Some(parent_job) = job.parent_job {
update_flow_status_after_job_completion(
@@ -1089,13 +1120,13 @@ async fn handle_queued_job(
&job.workspace_id,
true,
r,
Some(metrics.clone()),
metrics.clone(),
false,
same_worker_tx.clone(),
worker_dir,
None,
base_internal_url,
rsmq.clone()
)
.await?;
}
@@ -1130,7 +1161,7 @@ async fn handle_queued_job(
};
let result =
add_completed_job_error(db, &job, logs, error_value, Some(metrics.clone()))
add_completed_job_error(db, &job, logs, error_value, metrics.clone(), rsmq.clone())
.await?;
if job.is_flow_step {
if let Some(parent_job) = job.parent_job {
@@ -1142,13 +1173,13 @@ async fn handle_queued_job(
&job.workspace_id,
false,
result,
Some(metrics),
metrics,
false,
same_worker_tx,
worker_dir,
None,
base_internal_url,
rsmq
)
.await?;
}
@@ -1277,6 +1308,8 @@ mount {{
"".to_string()
};
// println!("handle lang job {:?}", SystemTime::now());
let result: error::Result<serde_json::Value> = match language {
None => {
return Err(Error::ExecutionErr(
@@ -1351,6 +1384,8 @@ mount {{
&lang_str,
job.id
);
// println!("handled job: {:?}", SystemTime::now());
result
}
@@ -1388,6 +1423,7 @@ async fn handle_go_job(
} else {
(false, false)
};
logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
set_logs(logs, &job.id, db).await;
@@ -1533,6 +1569,9 @@ func Run(req Req) (interface{{}}, error){{
.stderr(Stdio::piped())
.spawn()?
} else {
if let Some(ref netrc) = *NETRC {
write_file(&HOME_ENV, ".netrc", netrc).await?;
}
Command::new(GO_PATH.as_str())
.current_dir(job_dir)
.env_clear()
@@ -1540,6 +1579,7 @@ func Run(req Req) (interface{{}}, error){{
.env("PATH", PATH_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("GOPATH", GO_CACHE_DIR)
.env("GOPRIVATE", GOPRIVATE.as_ref().unwrap_or(&String::new()))
.env("HOME", HOME_ENV.as_str())
.args(vec!["run", "main.go"])
.stdout(Stdio::piped())
@@ -1672,67 +1712,101 @@ async fn handle_deno_job(
base_internal_url: &str,
worker_name: &str
) -> error::Result<serde_json::Value> {
// let mut start = Instant::now();
logs.push_str("\n\n--- DENO CODE EXECUTION ---\n");
set_logs(logs, &job.id, db).await;
let logs_to_set = logs.clone();
let id = job.id.clone();
let db2 = db.clone();
let set_logs_f = async {
set_logs(&logs_to_set, &id, &db2).await;
Ok(()) as error::Result<()>
};
// logs.push_str(format!("st: {:?}\n", start.elapsed().as_millis()).as_str());
// start = Instant::now();
let _ = write_file(job_dir, "main.ts", inner_content).await?;
let write_main_f = write_file(job_dir, "main.ts", inner_content);
let sig = trace_span!("parse_deno_signature")
.in_scope(|| windmill_parser_ts::parse_deno_signature(inner_content, true))?;
let client = client.get_authed().await;
create_args_and_out_file(&client, job, job_dir).await?;
let write_wrapper_f = async {
// let mut start = Instant::now();
let spread = windmill_parser_ts::parse_deno_signature(inner_content, true)?.args.into_iter().map(|x| x.name).join(",");
// logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
let wrapper_content: String = format!(
r#"
import {{ main }} from "./main.ts";
let spread = sig.args.into_iter().map(|x| x.name).join(",");
let wrapper_content: String = format!(
r#"
import {{ main }} from "./main.ts";
const args = await Deno.readTextFile("args.json")
.then(JSON.parse)
.then(({{ {spread} }}) => [ {spread} ])
const args = await Deno.readTextFile("args.json")
.then(JSON.parse)
.then(({{ {spread} }}) => [ {spread} ])
async function run() {{
let res: any = await main(...args);
const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
await Deno.writeTextFile("result.json", res_json);
Deno.exit(0);
}}
run().catch(async (e) => {{
await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }}));
Deno.exit(1);
}});
"#,
);
let w_id = job.workspace_id.clone();
let script_path_split = job.script_path().split("/");
let script_path_parts_len = script_path_split.clone().count();
let mut relative_mounts = "".to_string();
for c in 0..script_path_parts_len {
relative_mounts += ",\n ";
relative_mounts += &format!("\"./{}\": \"{base_internal_url}/api/w/{w_id}/scripts/raw/p/{}{}\"",
(0..c).map(|_| "../").join(""),
&script_path_split.clone().take(script_path_parts_len - c - 1).join("/"),
if c == script_path_parts_len - 1 { "" } else { "/" },
async function run() {{
let res: any = await main(...args);
const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
await Deno.writeTextFile("result.json", res_json);
Deno.exit(0);
}}
run().catch(async (e) => {{
await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }}));
Deno.exit(1);
}});
"#,
);
}
write_file(job_dir, "wrapper.ts", &wrapper_content).await?;
let import_map = format!(
r#"{{
"imports": {{
"/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
"./wrapper.ts": "./wrapper.ts",
"./main.ts": "./main.ts"{relative_mounts}
}}
}}"#,
);
write_file(job_dir, "import_map.json", &import_map).await?;
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
write_file(job_dir, "wrapper.ts", &wrapper_content).await?;
Ok(()) as error::Result<()>
};
let write_import_map_f = async {
let w_id = job.workspace_id.clone();
let script_path_split = job.script_path().split("/");
let script_path_parts_len = script_path_split.clone().count();
let mut relative_mounts = "".to_string();
for c in 0..script_path_parts_len {
relative_mounts += ",\n ";
relative_mounts += &format!("\"./{}\": \"{base_internal_url}/api/w/{w_id}/scripts/raw/p/{}{}\"",
(0..c).map(|_| "../").join(""),
&script_path_split.clone().take(script_path_parts_len - c - 1).join("/"),
if c == script_path_parts_len - 1 { "" } else { "/" },
);
}
let import_map = format!(
r#"{{
"imports": {{
"{base_internal_url}/api/w/{w_id}/scripts/raw/p/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
"{base_internal_url}": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
"/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
"./wrapper.ts": "./wrapper.ts",
"./main.ts": "./main.ts"{relative_mounts}
}}
}}"#,
);
write_file(job_dir, "import_map.json", &import_map).await?;
Ok(()) as error::Result<()>
};
let reserved_variables_args_out_f = async {
let client = client.get_authed().await;
let args_and_out_f = async {
create_args_and_out_file(&client, job, job_dir).await?;
Ok(()) as Result<()>
};
let reserved_variables_f = async {
let mut vars = get_reserved_variables(job, &client.token, db).await?;
vars.insert("RUST_LOG".to_string(), "info".to_string());
Ok(vars) as Result<HashMap<String, String>>
};
let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?;
Ok((reserved_variables, client.token)) as error::Result<(HashMap<String, String>, String)>
};
let (_, (reserved_variables, token), _, _, _) = tokio::try_join!(
set_logs_f,
reserved_variables_args_out_f,
write_main_f,
write_wrapper_f,
write_import_map_f)?;
let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url);
let common_deno_proc_envs = get_common_deno_proc_envs(&client.token, base_internal_url);
//do not cache local dependencies
let reload = format!("--reload={base_internal_url}");
let child = async {
@@ -1768,16 +1842,12 @@ run().catch(async (e) => {{
.stderr(Stdio::piped())
.spawn()
}
.instrument(trace_span!("create_deno_jail"))
.await?;
// logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_millis()).as_str());
// logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
// start = Instant::now();
handle_child(&job.id, db, logs, child, false, worker_name, &job.workspace_id).await?;
// logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
// start = Instant::now();
let r = read_result(job_dir).await;
// logs.push_str(format!("rr: {:?}\n", start.elapsed().as_millis()).as_str());
r
read_result(job_dir).await
}
#[tracing::instrument(level = "trace", skip_all)]
@@ -2435,6 +2505,7 @@ async fn install_go_dependencies(
new_lockfile = true;
}
}
let mod_command = if skip_tidy {
"download"
} else {
@@ -2461,8 +2532,11 @@ async fn install_go_dependencies(
let mut file = File::open(format!("{job_dir}/go.mod")).await?;
file.read_to_string(&mut req_content).await?;
req_content.push_str(GO_REQ_SPLITTER);
let mut file = File::open(format!("{job_dir}/go.sum")).await?;
file.read_to_string(&mut req_content).await?;
let sum_path = format!("{job_dir}/go.sum");
if tokio::fs::metadata(&sum_path).await.is_ok() {
let mut file = File::open(sum_path).await?;
file.read_to_string(&mut req_content).await?;
}
if non_dep_job {
sqlx::query!(
@@ -2917,13 +2991,14 @@ async fn append_logs(job_id: uuid::Uuid, logs: impl AsRef<str>, db: impl Borrow<
}
}
pub async fn handle_zombie_jobs_periodically(
pub async fn handle_zombie_jobs_periodically<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
db: &Pool<Postgres>,
mut rx: tokio::sync::broadcast::Receiver<()>,
base_internal_url: &str,
rsmq: Option<R>,
) {
loop {
handle_zombie_jobs(db, base_internal_url).await;
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(30)) => (),
@@ -2935,7 +3010,7 @@ pub async fn handle_zombie_jobs_periodically(
}
}
async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str) {
async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(db: &Pool<Postgres>, base_internal_url: &str, rsmq: Option<R>) {
if *RESTART_ZOMBIE_JOBS {
let restarted = sqlx::query!(
"UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
@@ -3013,6 +3088,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str) {
same_worker_tx_never_used,
"",
base_internal_url,
rsmq.clone()
)
.await;
}

View File

@@ -31,11 +31,13 @@ use windmill_common::{
type DB = sqlx::Pool<sqlx::Postgres>;
use windmill_queue::{
canceled_job_to_result, get_queued_job, push, JobPayload, QueuedJob, RawCode,
canceled_job_to_result, get_queued_job, push, JobPayload, QueueTransaction, QueuedJob, RawCode,
};
// #[instrument(level = "trace", skip_all)]
pub async fn update_flow_status_after_job_completion(
pub async fn update_flow_status_after_job_completion<
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
>(
db: &DB,
client: &AuthedClient,
flow: uuid::Uuid,
@@ -49,6 +51,7 @@ pub async fn update_flow_status_after_job_completion(
worker_dir: &str,
stop_early_override: Option<bool>,
base_internal_url: &str,
rsmq: Option<R>,
) -> error::Result<()> {
// this is manual tailrecursion because async_recursion blows up the stack
let mut depth = 0;
@@ -67,6 +70,7 @@ pub async fn update_flow_status_after_job_completion(
stop_early_override,
base_internal_url,
depth,
rsmq.clone(),
)
.await?;
while let Some(nrec) = rec {
@@ -86,6 +90,7 @@ pub async fn update_flow_status_after_job_completion(
nrec.stop_early_override,
base_internal_url,
depth,
rsmq.clone(),
)
.await?;
}
@@ -99,7 +104,9 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
stop_early_override: Option<bool>,
}
// #[instrument(level = "trace", skip_all)]
pub async fn update_flow_status_after_job_completion_internal(
pub async fn update_flow_status_after_job_completion_internal<
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
>(
db: &DB,
client: &AuthedClient,
flow: uuid::Uuid,
@@ -114,11 +121,12 @@ pub async fn update_flow_status_after_job_completion_internal(
stop_early_override: Option<bool>,
base_internal_url: &str,
depth: u8,
rsmq: Option<R>,
) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
let (should_continue_flow, flow_job, stop_early, skip_if_stop_early, nresult) = {
tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
let mut tx = db.begin().await?;
let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
let old_status_json = sqlx::query_scalar!(
"SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
@@ -154,7 +162,7 @@ pub async fn update_flow_status_after_job_completion_internal(
module_status,
FlowStatusModule::InProgress { iterator: Some(_), .. }
) {
compute_skip_loop_failures(flow, old_status.step, &mut tx)
compute_skip_loop_failures(flow, old_status.step, tx.transaction_mut())
.await?
.unwrap_or(false)
} else {
@@ -197,9 +205,11 @@ pub async fn update_flow_status_after_job_completion_internal(
FlowStatusModule::InProgress {
branchall: Some(BranchAllStatus { branch, .. }),
..
} => compute_skip_branchall_failure(flow, old_status.step, *branch, &mut tx)
.await?
.unwrap_or(false),
} => {
compute_skip_branchall_failure(flow, old_status.step, *branch, tx.transaction_mut())
.await?
.unwrap_or(false)
}
_ => false,
};
@@ -461,7 +471,7 @@ pub async fn update_flow_status_after_job_completion_internal(
.context("remove flow status retry")?;
}
let flow_job = get_queued_job(flow, w_id, &mut tx)
let flow_job = get_queued_job(flow, w_id, tx.transaction_mut())
.await?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
@@ -503,7 +513,9 @@ pub async fn update_flow_status_after_job_completion_internal(
{
true
}
false if has_failure_module(flow, &mut tx).await? && !is_failure_step => true,
false if has_failure_module(flow, tx.transaction_mut()).await? && !is_failure_step => {
true
}
false => false,
};
@@ -548,6 +560,7 @@ pub async fn update_flow_status_after_job_completion_internal(
logs,
canceled_job_to_result(&flow_job),
metrics.clone(),
rsmq.clone(),
)
.await?;
} else {
@@ -558,6 +571,7 @@ pub async fn update_flow_status_after_job_completion_internal(
stop_early && skip_if_stop_early,
nresult.clone(),
logs,
rsmq.clone(),
)
.await?;
}
@@ -571,6 +585,7 @@ pub async fn update_flow_status_after_job_completion_internal(
same_worker_tx.clone(),
worker_dir,
base_internal_url,
rsmq.clone(),
)
.await
{
@@ -581,6 +596,7 @@ pub async fn update_flow_status_after_job_completion_internal(
"Unexpected error during flow chaining:\n".to_string(),
json!({"message": err.to_string(), "name": "InternalError"}),
metrics.clone(),
rsmq.clone(),
)
.await;
true
@@ -831,7 +847,7 @@ async fn transform_input(
}
#[instrument(level = "trace", skip_all)]
pub async fn handle_flow(
pub async fn handle_flow<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
flow_job: &QueuedJob,
db: &sqlx::Pool<sqlx::Postgres>,
client: &AuthedClient,
@@ -839,6 +855,7 @@ pub async fn handle_flow(
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
base_internal_url: &str,
rsmq: Option<R>,
) -> anyhow::Result<()> {
let value = flow_job
.raw_flow
@@ -862,6 +879,7 @@ pub async fn handle_flow(
same_worker_tx,
worker_dir,
base_internal_url,
rsmq,
)
.await?;
Ok(())
@@ -869,7 +887,7 @@ pub async fn handle_flow(
#[async_recursion]
// #[instrument(level = "trace", skip_all)]
async fn push_next_flow_job(
async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
flow_job: &QueuedJob,
mut status: FlowStatus,
flow: FlowValue,
@@ -879,6 +897,7 @@ async fn push_next_flow_job(
same_worker_tx: Sender<Uuid>,
worker_dir: &str,
base_internal_url: &str,
rsmq: Option<R>,
) -> error::Result<()> {
let job_root = flow_job
.root_job
@@ -915,6 +934,7 @@ async fn push_next_flow_job(
worker_dir,
None,
base_internal_url,
rsmq,
)
.await;
}
@@ -1085,7 +1105,7 @@ async fn push_next_flow_job(
let logs = "Timed out waiting to be resumed".to_string();
let result = json!({ "error": {"message": logs, "name": "SuspendedTimeout"}});
let _uuid =
add_completed_job(db, &flow_job, success, skipped, result, logs).await?;
add_completed_job(db, &flow_job, success, skipped, result, logs, rsmq).await?;
return Ok(());
}
@@ -1286,7 +1306,7 @@ async fn push_next_flow_job(
flow.same_worker && module.suspend.is_none() && module.sleep.is_none();
/* Finally, push the job into the queue */
let mut tx = db.begin().await?;
let mut tx = (rsmq.clone(), db.begin().await?).into();
let mut uuids = vec![];
let len = match &job_payloads {
@@ -1452,7 +1472,7 @@ async fn push_next_flow_job(
json!(i),
flow_job.id
)
.execute(&mut tx)
.execute(db)
.await?;
} else {
sqlx::query!(
@@ -1468,7 +1488,7 @@ async fn push_next_flow_job(
json!(i),
flow_job.id
)
.execute(&mut tx)
.execute(db)
.await?;
};

View File

@@ -1,53 +0,0 @@
{
"type": "RANDOM",
"actions": [
{
"weight": 1,
"action": {
"type": "RANDOM",
"actions": [
{
"weight": 1,
"action": {
"type": "PREVIEW_SCRIPT",
"workspace": "demo",
"language": "deno",
"args": {},
"content": "export async function main() { return \"Hello World\"; }"
}
},
{
"weight": 1,
"action": {
"type": "PREVIEW_SCRIPT",
"workspace": "demo",
"language": "python3",
"args": {},
"content": "def main(): return \"Hello World\";"
}
},
{
"weight": 1,
"action": {
"type": "PREVIEW_SCRIPT",
"workspace": "demo",
"language": "go",
"args": {},
"content": "func main() { return \"Hello World\" }"
}
}
]
}
},
{
"weight": 0.5,
"action": {
"type": "PREVIEW_SCRIPT",
"workspace": "demo",
"language": "deno",
"args": {},
"content": "import { delay } from \"https://deno.land/std@0.131.0/async/delay.ts\"; export async function main() { await delay(1000); return \"Hello World\"; }"
}
}
]
}

7
benchmarks/identity.json Normal file
View File

@@ -0,0 +1,7 @@
{
"type": "PREVIEW_SCRIPT",
"workspace": "demo",
"language": "deno",
"args": {},
"content": "export async function main() { return \"Hello World\"; }"
}

View File

@@ -27,60 +27,60 @@ await new Command()
"The number of workers to run at once.",
{
default: 1,
}
},
)
.option(
"-s --seconds <seconds:number>",
"How long to run the benchmark for (in seconds).",
{
default: 30,
}
},
)
.option("--max <max:number>", "Maximum number of operations performed.")
.option("-e --email <email:string>", "The email to use to login.")
.option("-p --password <password:string>", "The password to use to login.")
.env(
"WM_TOKEN=<token:string>",
"The token to use when talking to the API server. Preferred over manual login."
"The token to use when talking to the API server. Preferred over manual login.",
)
.option(
"-t --token <token:string>",
"The token to use when talking to the API server. Preferred over manual login."
"The token to use when talking to the API server. Preferred over manual login.",
)
.env(
"WM_WORKSPACE=<workspace:string>",
"The workspace to spawn scripts from."
"The workspace to spawn scripts from.",
)
.option(
"-w --workspace <workspace:string>",
"The workspace to spawn scripts from.",
{ default: "starter" }
{ default: "starter" },
)
.option("-m --metrics <metrics:string>", "The url to scrape metrics from.", {
default: "http://localhost:8001/metrics",
})
.option(
"--export-json <export_json:string>",
"If set, exports will be into a JSON file."
"If set, exports will be into a JSON file.",
)
.option(
"--export-csv <export_csv:string>",
"If set, exports will be into a csv file."
"If set, exports will be into a csv file.",
)
.option(
"--export-histograms [histograms...:string]",
"Mark metrics (without label) that are reported as histograms to export."
"Mark metrics (without label) that are reported as histograms to export.",
)
.option(
"--export-simple [simple...:string]",
"Mark metrics (without label) that are reported as simple values."
"Mark metrics (without label) that are reported as simple values.",
)
.option(
"--maximum-throughput <maximum_throughput:number>",
"Maximum number of jobs/flows to start in one second.",
{
default: Infinity,
}
},
)
.option("--use-flows", "Run flows instead of jobs.")
.option("--custom <custom_path:string>", "Use custom actions during bench")
@@ -89,11 +89,11 @@ await new Command()
"The maximum time in ms to wait for jobs to complete.",
{
default: 90000,
}
},
)
.option(
"--continous",
"Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked."
"Run the benchmark forever. This effectively disables metric collection & exports. No zombie jobs will be tracked.",
)
.option(
"--histogram-buckets [buckets...:string]",
@@ -114,7 +114,7 @@ await new Command()
"0.01",
"0.005",
],
}
},
)
.action(
async ({
@@ -162,7 +162,7 @@ await new Command()
new URL("./scraper.ts", import.meta.url).href,
{
type: "module",
}
},
);
metrics_worker.postMessage({
@@ -192,8 +192,8 @@ await new Command()
zombieTimeout,
},
null,
4
)
4,
),
);
const config = {
@@ -245,19 +245,22 @@ await new Command()
const updateState = setInterval(async () => {
const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
const sum = jobsSent.reduce((a, b) => a + b, 0);
const queue_length = (
await windmill.JobService.listQueue({
workspace: config.workspace_id,
})
).length;
const queue_length = (await (await fetch(
host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
{ headers: { ["Authorization"]: "Bearer " + config.token } },
)).json()).database_length;
await Deno.stdout.write(
enc(
`elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
jobsSent
)} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
2
)}) | queue: ${queue_length} \r`
)
`elapsed: ${elapsed}/${seconds} | jobs sent: ${
JSON.stringify(
jobsSent,
)
} (sum: ${sum} thr: ${
(sum / elapsed).toFixed(
2,
)
}) | queue: ${queue_length} \r`,
),
);
}, 100);
@@ -284,7 +287,7 @@ await new Command()
const sum = jobsSent.reduce((a, b) => a + b, 0);
await Deno.stdout.write(
enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`)
enc(" ".padStart(30) + `\rduration: ${seconds} | jobs sent: ${sum}\n`),
);
const shutdown_start = Date.now();
@@ -302,7 +305,7 @@ await new Command()
};
worker.addEventListener("message", l);
worker.postMessage(
Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000
Number.isSafeInteger(zombieTimeout) ? zombieTimeout : 90000,
);
});
@@ -318,11 +321,10 @@ await new Command()
console.log("incorrect results: ", incorrect_results);
console.log(
"queue length:",
(
await windmill.JobService.listQueue({
workspace: config.workspace_id,
})
).length
(await (await fetch(
host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
{ headers: { ["Authorization"]: "Bearer " + config.token } },
)).json()).database_length,
);
metrics_worker!.postMessage("stop");
@@ -346,7 +348,7 @@ await new Command()
const value = values[i]!;
const mean = value.reduce((acc, e) => acc + e, 0) / values.length;
const stdev = Math.sqrt(
value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length
value.reduce((acc, e) => acc + (e - mean) ** 2) / values.length,
);
obj[name] = { mean, stdev };
}
@@ -374,6 +376,6 @@ await new Command()
f.close();
}
console.log("done");
}
},
)
.parse();

View File

@@ -13,6 +13,8 @@ const promise = new Promise<{
continous: boolean;
max_per_worker: number;
custom: Action | undefined;
server: string;
token: string;
}>((resolve, _reject) => {
self.onmessage = (evt) => {
const sharedConfig = evt.data;
@@ -24,6 +26,8 @@ const promise = new Promise<{
continous: sharedConfig.continous,
max_per_worker: sharedConfig.max_per_worker,
custom: sharedConfig.custom,
server: sharedConfig.server,
token: sharedConfig.token,
};
self.name = "Worker " + sharedConfig.i;
resolve(config);
@@ -46,19 +50,20 @@ const updateStatusInterval = setInterval(() => {
}, 100);
while (cont) {
const queue_length = (
await windmill.JobService.listQueue({ workspace: config.workspace_id })
).length;
if (queue_length > 500) {
const queue_length = (await (await fetch(
config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
{ headers: { ["Authorization"]: "Bearer " + config.token } },
)).json()).database_length;
if (queue_length > 2500) {
console.log(
`queue length: ${queue_length} > 500. waiting... `
`queue length: ${queue_length} > 2500. waiting... `,
);
await sleep(0.5);
continue;
}
if (
(total_spawned * 1000) / (Date.now() - start_time) >
config.per_worker_throughput
config.per_worker_throughput
) {
console.log("at maximum throughput. waiting...");
await sleep(0.1);
@@ -138,13 +143,13 @@ while (outstanding.length > 0 && Date.now() < end_time) {
await Deno.stdout.write(
enc(
`uuid: ${uuid}, queue length: ${
(
await windmill.JobService.listQueue({
workspace: config.workspace_id,
})
).length
} \r`
)
(await (await fetch(
config.server + "/api/w/" + config.workspace_id +
"/jobs/queue/count",
{ headers: { ["Authorization"]: "Bearer " + config.token } },
)).json()).database_length
} \r`,
),
);
} else if (!config.useFlows) {
r = r as api.CompletedJob;
@@ -156,7 +161,7 @@ while (outstanding.length > 0 && Date.now() < end_time) {
" != " +
uuid +
"job: \n" +
JSON.stringify(r, null, 2)
JSON.stringify(r, null, 2),
);
incorrect_results++;
}

View File

@@ -19,7 +19,7 @@ import sync from "./sync.ts";
import { tryResolveVersion } from "./context.ts";
import { GlobalOptions } from "./types.ts";
export const VERSION = "v1.86.0";
export const VERSION = "v1.87.0";
let command: any = new Command()
.name("wmill")

View File

@@ -142,6 +142,7 @@ export async function handleFile(
typed.summary === remote.summary &&
typed.is_template === remote.is_template &&
typed.kind == remote.kind &&
!remote.archived &&
remote?.lock == typed.lock?.join("\n") &&
JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
) {

86
docker-compose-dev.yml Normal file
View File

@@ -0,0 +1,86 @@
version: "3.7"
services:
db:
image: postgres:14
restart: unless-stopped
volumes:
- db_data:/var/lib/postgresql/data
expose:
- 5432
ports:
- 5432:5432
environment:
POSTGRES_PASSWORD: ${DB_PASSWORD}
POSTGRES_DB: windmill
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
windmill_server:
image: ghcr.io/windmill-labs/windmill:main
deploy:
replicas: 1
restart: unless-stopped
expose:
- 8000
ports:
- 8000:8000
environment:
- DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
- BASE_URL=http://${WM_BASE_URL}
- RUST_LOG=info
## You can set the number of workers to > 0 and not need any separate worker service
- NUM_WORKERS=0
- DISABLE_SERVER=false
- METRICS_ADDR=false
depends_on:
db:
condition: service_healthy
# volumes:
# - ./oauth.json/:/usr/src/app/oauth.json
windmill_worker:
image: ghcr.io/windmill-labs/windmill:main
deploy:
replicas: 3
restart: unless-stopped
environment:
- DATABASE_URL=postgres://postgres:${DB_PASSWORD}@db/windmill?sslmode=disable
- BASE_URL=http://${WM_BASE_URL}
- BASE_INTERNAL_URL=http://windmill_server:8000
- RUST_LOG=info
- NUM_WORKERS=1
- DISABLE_SERVER=true
- KEEP_JOB_DIR=false
- DENO_PATH=/usr/bin/deno
- PYTHON_PATH=/usr/local/bin/python3
- METRICS_ADDR=false
depends_on:
db:
condition: service_healthy
# to mount the worker folder to debug,, KEEP_JOB_DIR=true and mount /tmp/windmill
volumes:
- worker_dependency_cache:/tmp/windmill/cache
lsp:
image: ghcr.io/windmill-labs/windmill-lsp:latest
restart: unless-stopped
expose:
- 3001
caddy:
image: caddy:2.5.2-alpine
restart: unless-stopped
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile
ports:
- 80:80
- 443:443
environment:
- BASE_URL=${WM_BASE_URL}
volumes:
db_data: null
worker_dependency_cache: null

View File

@@ -4,9 +4,9 @@
:8080 {
bind {$ADDRESS}
reverse_proxy /api/* http://windmill:8000
reverse_proxy /api/* http://localhost:8000
reverse_proxy /* http://localhost:3000
reverse_proxy /ws/* http://lsp:3001 {
lb_policy header "Authorization"
}
}
}

View File

@@ -17,7 +17,7 @@ In the root folder:
```bash
docker build . -t windmill
docker compose up db windmill_server windmill_worker
docker compose -f docker-compose-dev.yml up db windmill_server windmill_worker
```
### 2. Backend is run by cargo
@@ -28,20 +28,23 @@ docker compose up db windmill_server windmill_worker
- Install llvm
**On OSX:**
```bash
brew install llvm caddy gsed
# make LLVM tools available on PATH
echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> ~/.zshrc
# now, restart your shell. You should now have the `lld` binary on your PATH.
```
- To test that you have Rust and Cargo installed run `cargo --version`
- In your terminal, go to the backend directory and run `cargo build`
- Run `cargo run`
**Known issue on M1 Mac while running `cargo build`**
- You may encounter `linking with cc failed` build time error.
- To solve this run:
```bash
@@ -49,7 +52,6 @@ docker compose up db windmill_server windmill_worker
source ~/.zshrc
```
**Do a Frontend Build**
In order to run the backend, you need to have a frontend build inside `frontend/build/`.
@@ -70,11 +72,14 @@ npm run build
```
**Known issue while running `npm run build`**
- You may encounter `FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory` error.
- To solve this run:
```bash
export NODE_OPTIONS=--max_old_space_size=8096
```
- run `npm run build` again
In the root folder:
@@ -86,7 +91,7 @@ docker-compose up db
In the backend folder:
```bash
DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5433/windmill?sslmode=disable cargo run
DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5432/windmill?sslmode=disable cargo run
```
You can now access [http://127.0.0.1:8000](http://127.0.0.1:8000).
@@ -146,13 +151,13 @@ Recommended config for VS Code:
- turn _format on save_ on
## Building
The project is built with [SvelteKit](https://kit.svelte.dev/) and uses as output static files.
There are others adapters for sveltekit, but we use the static adapter.
To build the frontend as static assets, use:
```
npm run build
```
@@ -160,10 +165,11 @@ npm run build
The output is in the `build` folder.
The default build assume you serve every non static files as the 200.html file which is catchall. If you prefer a normal layout, you can use:
```
NOTCATCHALL=true npm run build
```
which will generate an index.html and allow you to serve the frontend with any static server.
Env variables used for build are set in .env file. See [https://vitejs.dev/guide/env-and-mode.html#env-files](https://vitejs.dev/guide/env-and-mode.html#env-files) for more details.

View File

@@ -1,12 +1,12 @@
{
"name": "windmill",
"version": "1.86.0",
"version": "1.87.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "windmill",
"version": "1.86.0",
"version": "1.87.0",
"dependencies": {
"@fortawesome/free-brands-svg-icons": "^6.2.1",
"@fortawesome/free-solid-svg-icons": "^6.2.1",
@@ -29,7 +29,7 @@
"svelte-autosize": "^1.0.1",
"svelte-chartjs": "^3.1.0",
"svelte-portal": "^2.2.0",
"svelte-select": "^5.3.1",
"svelte-select": "^5.6.0",
"svelte-timezone-picker": "^2.0.3",
"tailwind-merge": "^1.9.1",
"vscode-languageclient": "^8.1.0",
@@ -47,21 +47,21 @@
"@types/d3-zoom": "^3.0.2",
"@types/lodash": "^4.14.191",
"@types/node": "^18.11.18",
"@types/vscode": "~1.76.0",
"@types/vscode": "~1.77.0",
"@typescript-eslint/eslint-plugin": "^5.49.0",
"@typescript-eslint/parser": "^5.48.0",
"@zerodevx/svelte-toast": "^0.8.1",
"autoprefixer": "^10.4.13",
"cssnano": "^6.0.0",
"d3-dag": "^0.11.5",
"eslint": "^8.28.0",
"eslint": "^8.38.0",
"eslint-config-prettier": "^8.6.0",
"eslint-plugin-svelte3": "^4.0.0",
"monaco-editor-workers": "0.36.0",
"ol": "^7.2.2",
"openapi-typescript-codegen": "^0.23.0",
"path-browserify": "^1.0.1",
"pdfjs-dist": "^3.4.120",
"pdfjs-dist": "^3.5.141",
"postcss": "^8.4.18",
"postcss-load-config": "^4.0.1",
"prettier": "^2.8.3",
@@ -80,7 +80,7 @@
"svelte-preprocess": "^5.0.1",
"svelte-range-slider-pips": "^2.1.1",
"svelte-splitpanes": "^0.7.3",
"svelte2tsx": "^0.6.10",
"svelte2tsx": "^0.6.11",
"tailwindcss": "^3.2.2",
"tslib": "^2.4.1",
"typescript": "^4.9.3",
@@ -636,9 +636,9 @@
}
},
"node_modules/@eslint/js": {
"version": "8.37.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.37.0.tgz",
"integrity": "sha512-x5vzdtOOGgFVDCUs81QRB2+liax8rFg3+7hqM+QhBG0/G3F1ZsoYl97UrqgHgQ9KKT7G6c4V+aTUCgu/n22v1A==",
"version": "8.38.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.38.0.tgz",
"integrity": "sha512-IoD2MfUnOV58ghIHCiil01PcohxjbYR/qCxsoC+xNgUwh1EY8jOOrYmu3d3a71+tJJ23uscEV4X2HJWMsPJu4g==",
"dev": true,
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -1378,9 +1378,9 @@
"dev": true
},
"node_modules/@types/vscode": {
"version": "1.76.0",
"resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.76.0.tgz",
"integrity": "sha512-CQcY3+Fe5hNewHnOEAVYj4dd1do/QHliXaknAEYSXx2KEHUzFibDZSKptCon+HPgK55xx20pR+PBJjf0MomnBA==",
"version": "1.77.0",
"resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.77.0.tgz",
"integrity": "sha512-MWFN5R7a33n8eJZJmdVlifjig3LWUNRrPeO1xemIcZ0ae0TEQuRc7G2xV0LUX78RZFECY1plYBn+dP/Acc3L0Q==",
"dev": true
},
"node_modules/@typescript-eslint/eslint-plugin": {
@@ -2823,15 +2823,15 @@
}
},
"node_modules/eslint": {
"version": "8.37.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.37.0.tgz",
"integrity": "sha512-NU3Ps9nI05GUoVMxcZx1J8CNR6xOvUT4jAUMH5+z8lpp3aEdPVCImKw6PWG4PY+Vfkpr+jvMpxs/qoE7wq0sPw==",
"version": "8.38.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.38.0.tgz",
"integrity": "sha512-pIdsD2jwlUGf/U38Jv97t8lq6HpaU/G9NKbYmpWpZGw3LdTNhZLbJePqxOXGB5+JEKfOPU/XLxYxFh03nr1KTg==",
"dev": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.4.0",
"@eslint/eslintrc": "^2.0.2",
"@eslint/js": "8.37.0",
"@eslint/js": "8.38.0",
"@humanwhocodes/config-array": "^0.11.8",
"@humanwhocodes/module-importer": "^1.0.1",
"@nodelib/fs.walk": "^1.2.8",
@@ -4735,14 +4735,17 @@
}
},
"node_modules/pdfjs-dist": {
"version": "3.4.120",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.4.120.tgz",
"integrity": "sha512-B1hw9ilLG4m/jNeFA0C2A0PZydjxslP8ylU+I4XM7Bzh/xWETo9EiBV848lh0O0hLut7T6lK1V7cpAXv5BhxWw==",
"version": "3.5.141",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.5.141.tgz",
"integrity": "sha512-lYIvyi5grtYOIatsfCifIKwxHeAJ8eHyP22DTdvY4pm0yWVSFQnMafpgCPSw8gaNRDDdcHnBVOkqMsyK8SRxZg==",
"dev": true,
"dependencies": {
"path2d-polyfill": "^2.0.1",
"web-streams-polyfill": "^3.2.1"
},
"engines": {
"node": ">=16"
},
"optionalDependencies": {
"canvas": "^2.11.0"
}
@@ -6559,9 +6562,9 @@
"dev": true
},
"node_modules/svelte-select": {
"version": "5.5.2",
"resolved": "https://registry.npmjs.org/svelte-select/-/svelte-select-5.5.2.tgz",
"integrity": "sha512-uvTOJyrrD5OnSCAv9A+gLtIeeYHL5PUw3YpnuBkmfp7a3IHUyjgNNM4GIf462Lh9QEEEM3nD0R0E+G/Jp1iq1w==",
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/svelte-select/-/svelte-select-5.6.0.tgz",
"integrity": "sha512-+EDXupp3Lr/hKlXNra1vR1VEGxEEsFxYPs5yDDqQE+xagyAOd8P54IJtU84Z5QN3HYc8l2Hp+tNCCYZoTWWxyQ==",
"dependencies": {
"@floating-ui/dom": "^1.2.1",
"svelte-floating-ui": "1.2.8"
@@ -6595,9 +6598,9 @@
}
},
"node_modules/svelte2tsx": {
"version": "0.6.10",
"resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.6.10.tgz",
"integrity": "sha512-7CtUexhSHppSpLwVX7yI9OxJp7+esCNTuOF/CaVxWTgWp88QF/2/dCbpND/cTZmBxht5cFFQBwoLXXqFLdYe3Q==",
"version": "0.6.11",
"resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.6.11.tgz",
"integrity": "sha512-rRW/3V/6mcejYWmSqcHpmILOSPsOhLgkbKbrTOz82s2n8TywmIsqj2jYPsiL6HeGoUM/atiTD0YKguW4b7ECog==",
"dev": true,
"dependencies": {
"dedent-js": "^1.0.1",
@@ -7672,9 +7675,9 @@
}
},
"@eslint/js": {
"version": "8.37.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.37.0.tgz",
"integrity": "sha512-x5vzdtOOGgFVDCUs81QRB2+liax8rFg3+7hqM+QhBG0/G3F1ZsoYl97UrqgHgQ9KKT7G6c4V+aTUCgu/n22v1A==",
"version": "8.38.0",
"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.38.0.tgz",
"integrity": "sha512-IoD2MfUnOV58ghIHCiil01PcohxjbYR/qCxsoC+xNgUwh1EY8jOOrYmu3d3a71+tJJ23uscEV4X2HJWMsPJu4g==",
"dev": true
},
"@floating-ui/core": {
@@ -8296,9 +8299,9 @@
"dev": true
},
"@types/vscode": {
"version": "1.76.0",
"resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.76.0.tgz",
"integrity": "sha512-CQcY3+Fe5hNewHnOEAVYj4dd1do/QHliXaknAEYSXx2KEHUzFibDZSKptCon+HPgK55xx20pR+PBJjf0MomnBA==",
"version": "1.77.0",
"resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.77.0.tgz",
"integrity": "sha512-MWFN5R7a33n8eJZJmdVlifjig3LWUNRrPeO1xemIcZ0ae0TEQuRc7G2xV0LUX78RZFECY1plYBn+dP/Acc3L0Q==",
"dev": true
},
"@typescript-eslint/eslint-plugin": {
@@ -9310,15 +9313,15 @@
"dev": true
},
"eslint": {
"version": "8.37.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.37.0.tgz",
"integrity": "sha512-NU3Ps9nI05GUoVMxcZx1J8CNR6xOvUT4jAUMH5+z8lpp3aEdPVCImKw6PWG4PY+Vfkpr+jvMpxs/qoE7wq0sPw==",
"version": "8.38.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.38.0.tgz",
"integrity": "sha512-pIdsD2jwlUGf/U38Jv97t8lq6HpaU/G9NKbYmpWpZGw3LdTNhZLbJePqxOXGB5+JEKfOPU/XLxYxFh03nr1KTg==",
"dev": true,
"requires": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.4.0",
"@eslint/eslintrc": "^2.0.2",
"@eslint/js": "8.37.0",
"@eslint/js": "8.38.0",
"@humanwhocodes/config-array": "^0.11.8",
"@humanwhocodes/module-importer": "^1.0.1",
"@nodelib/fs.walk": "^1.2.8",
@@ -10772,9 +10775,9 @@
}
},
"pdfjs-dist": {
"version": "3.4.120",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.4.120.tgz",
"integrity": "sha512-B1hw9ilLG4m/jNeFA0C2A0PZydjxslP8ylU+I4XM7Bzh/xWETo9EiBV848lh0O0hLut7T6lK1V7cpAXv5BhxWw==",
"version": "3.5.141",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-3.5.141.tgz",
"integrity": "sha512-lYIvyi5grtYOIatsfCifIKwxHeAJ8eHyP22DTdvY4pm0yWVSFQnMafpgCPSw8gaNRDDdcHnBVOkqMsyK8SRxZg==",
"dev": true,
"requires": {
"canvas": "^2.11.0",
@@ -12017,9 +12020,9 @@
"dev": true
},
"svelte-select": {
"version": "5.5.2",
"resolved": "https://registry.npmjs.org/svelte-select/-/svelte-select-5.5.2.tgz",
"integrity": "sha512-uvTOJyrrD5OnSCAv9A+gLtIeeYHL5PUw3YpnuBkmfp7a3IHUyjgNNM4GIf462Lh9QEEEM3nD0R0E+G/Jp1iq1w==",
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/svelte-select/-/svelte-select-5.6.0.tgz",
"integrity": "sha512-+EDXupp3Lr/hKlXNra1vR1VEGxEEsFxYPs5yDDqQE+xagyAOd8P54IJtU84Z5QN3HYc8l2Hp+tNCCYZoTWWxyQ==",
"requires": {
"@floating-ui/dom": "^1.2.1",
"svelte-floating-ui": "1.2.8"
@@ -12043,9 +12046,9 @@
}
},
"svelte2tsx": {
"version": "0.6.10",
"resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.6.10.tgz",
"integrity": "sha512-7CtUexhSHppSpLwVX7yI9OxJp7+esCNTuOF/CaVxWTgWp88QF/2/dCbpND/cTZmBxht5cFFQBwoLXXqFLdYe3Q==",
"version": "0.6.11",
"resolved": "https://registry.npmjs.org/svelte2tsx/-/svelte2tsx-0.6.11.tgz",
"integrity": "sha512-rRW/3V/6mcejYWmSqcHpmILOSPsOhLgkbKbrTOz82s2n8TywmIsqj2jYPsiL6HeGoUM/atiTD0YKguW4b7ECog==",
"dev": true,
"requires": {
"dedent-js": "^1.0.1",

View File

@@ -1,6 +1,6 @@
{
"name": "windmill",
"version": "1.86.0",
"version": "1.87.0",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -26,21 +26,21 @@
"@types/d3-zoom": "^3.0.2",
"@types/lodash": "^4.14.191",
"@types/node": "^18.11.18",
"@types/vscode": "~1.76.0",
"@types/vscode": "~1.77.0",
"@typescript-eslint/eslint-plugin": "^5.49.0",
"@typescript-eslint/parser": "^5.48.0",
"@zerodevx/svelte-toast": "^0.8.1",
"autoprefixer": "^10.4.13",
"cssnano": "^6.0.0",
"d3-dag": "^0.11.5",
"eslint": "^8.28.0",
"eslint": "^8.38.0",
"eslint-config-prettier": "^8.6.0",
"eslint-plugin-svelte3": "^4.0.0",
"monaco-editor-workers": "0.36.0",
"ol": "^7.2.2",
"openapi-typescript-codegen": "^0.23.0",
"path-browserify": "^1.0.1",
"pdfjs-dist": "^3.4.120",
"pdfjs-dist": "^3.5.141",
"postcss": "^8.4.18",
"postcss-load-config": "^4.0.1",
"prettier": "^2.8.3",
@@ -59,7 +59,7 @@
"svelte-preprocess": "^5.0.1",
"svelte-range-slider-pips": "^2.1.1",
"svelte-splitpanes": "^0.7.3",
"svelte2tsx": "^0.6.10",
"svelte2tsx": "^0.6.11",
"tailwindcss": "^3.2.2",
"tslib": "^2.4.1",
"typescript": "^4.9.3",
@@ -89,7 +89,7 @@
"svelte-autosize": "^1.0.1",
"svelte-chartjs": "^3.1.0",
"svelte-portal": "^2.2.0",
"svelte-select": "^5.3.1",
"svelte-select": "^5.6.0",
"svelte-timezone-picker": "^2.0.3",
"tailwind-merge": "^1.9.1",
"vscode-languageclient": "^8.1.0",

View File

@@ -101,8 +101,8 @@
>{/if}{#if typeof result == 'object' && Object.keys(result).length > 0}<div
class="mb-2 w-full text-sm text-gray-700 relative"
>The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
<div class="text-gray-500 text-sm absolute top-6 right-0">
<button on:click={jsonViewer.openDrawer}>Expand JSON</button>
<div class="text-gray-500 text-xs absolute top-5 right-0">
<button on:click={jsonViewer.openDrawer}>Expand</button>
</div></div
>{/if}{#if !forceJson && resultKind == 'table-col'}<div
class="grid grid-flow-col-dense border border-gray-200 rounded-md"

View File

@@ -4,7 +4,7 @@
<script lang="ts">
import { ResourceService, VariableService } from '$lib/gen'
import { getScriptByPath, sendUserToast } from '$lib/utils'
import { getModifierKey, getScriptByPath, sendUserToast } from '$lib/utils'
import {
faBroom,
@@ -365,10 +365,12 @@
{iconOnly}
startIcon={{ icon: faBroom }}
>
Format (Ctrl+S)
Format ({getModifierKey()}+S)
</Button>
<svelte:fragment slot="text">
Format <Kbd class="!text-gray-800">Ctrl</Kbd> + <Kbd class="!text-gray-800">S</Kbd>
Format <Kbd class="!text-gray-800">{getModifierKey()}</Kbd> + <Kbd class="!text-gray-800">
S
</Kbd>
</svelte:fragment>
</Popover>
</div>

View File

@@ -217,20 +217,20 @@
switch (event.key) {
case 'Z':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
$flowStore = redo(history)
event.preventDefault()
}
break
case 'z':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
$flowStore = undo(history, $flowStore)
$selectedIdStore = 'Input'
event.preventDefault()
}
break
case 's':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
saveFlow(false)
event.preventDefault()
}

View File

@@ -13,6 +13,7 @@
import FlowProgressBar from './flows/FlowProgressBar.svelte'
import CapturePayload from './flows/content/CapturePayload.svelte'
import { Loader2 } from 'lucide-svelte'
import { getModifierKey } from '$lib/utils'
let capturePayload: CapturePayload
export let previewMode: 'upTo' | 'whole'
@@ -75,7 +76,7 @@
if (open) {
switch (event.key) {
case 'Enter':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
event.preventDefault()
runPreview($previewArgs)
}
@@ -134,7 +135,7 @@
btnClasses="w-full max-w-lg"
on:click={() => runPreview($previewArgs)}
>
Test flow <Kbd class="ml-2">Ctrl+Enter</Kbd>
Test flow <Kbd class="ml-2">{getModifierKey()}+Enter</Kbd>
</Button>
{/if}
<Button

View File

@@ -308,6 +308,9 @@
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
placeholder={defaultValue ?? ''}
bind:value
on:pointerdown|stopPropagation={(e) => {
dispatch('inputClicked', e)
}}
/>
</div>
</div>

View File

@@ -28,6 +28,7 @@
properties={schema.properties[argName].properties}
itemsType={schema.properties[argName].items}
extra={schema.properties[argName]}
on:inputClicked
/>
{/if}
</div>

View File

@@ -2,7 +2,7 @@
import type { Schema } from '$lib/common'
import { ScriptService, type FlowModule, type InputTransform, type Job } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { getScriptByPath } from '$lib/utils'
import { getModifierKey, getScriptByPath } from '$lib/utils'
import { Loader2 } from 'lucide-svelte'
import { getContext } from 'svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
@@ -86,7 +86,7 @@
runnable={{ summary: mod.summary ?? '', schema, description: '' }}
runAction={(_, args) => runTest(args)}
schedulable={false}
buttonText="Test (Ctrl+Enter)"
buttonText={`Test (${getModifierKey()}+Enter)`}
detailed={false}
topButton
bind:args={stepArgs}

View File

@@ -317,16 +317,28 @@
></span
>
<div class="flex flex-row gap-1 w-full">
<div class="flex flex-row items-center gap-1 w-full">
<select class="grow w-full" {disabled} bind:value={meta.owner}>
{#each folders as { name, write }}
<option disabled={!write}>{name}{write ? '' : ' (read-only)'}</option>
{/each}
</select>
<Button variant="border" size="xs" on:click={viewFolder.openDrawer}>
<Button
title="View folder"
btnClasses="!p-1.5"
variant="border"
size="xs"
on:click={viewFolder.openDrawer}
>
<Icon scale={0.8} data={faEye} /></Button
>
<Button variant="border" size="xs" on:click={newFolder.openDrawer}>
<Button
title="New folder"
btnClasses="!p-1.5"
variant="border"
size="xs"
on:click={newFolder.openDrawer}
>
<Icon scale={0.8} data={faPlus} /></Button
></div
>

View File

@@ -0,0 +1,326 @@
<script lang="ts">
import { Button } from '$lib/components/common'
import { InputService, type Input, RunnableType, type CreateInput } from '$lib/gen/index.js'
import { userStore, workspaceStore } from '$lib/stores.js'
import { classNames, displayDate, displayDaysAgo, sendUserToast } from '$lib/utils.js'
import { faSave } from '@fortawesome/free-solid-svg-icons'
import { createEventDispatcher } from 'svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
import { ArrowLeftIcon, Edit, X } from 'lucide-svelte'
import Toggle from './Toggle.svelte'
import Tooltip from './Tooltip.svelte'
export let scriptHash: string | null = null
export let scriptPath: string | null = null
export let flowPath: string | null = null
let runnableId: string | undefined = scriptHash || scriptPath || flowPath || undefined
let runnableType: RunnableType | undefined = scriptHash
? RunnableType.SCRIPT_HASH
: scriptPath
? RunnableType.SCRIPT_PATH
: flowPath
? RunnableType.FLOW_PATH
: undefined
// Are the current Inputs valid and able to be saved?
export let isValid: boolean
export let args: object
let previousInputs: Input[] = []
interface EditableInput extends Input {
isEditing?: boolean
isSaving?: boolean
}
let savedInputs: EditableInput[] = []
let selectedInput: Input | null
async function loadInputHistory() {
previousInputs = await InputService.getInputHistory({
workspace: $workspaceStore!,
runnableId,
runnableType,
perPage: 10
})
}
async function loadSavedInputs() {
savedInputs = await InputService.listInputs({
workspace: $workspaceStore!,
runnableId,
runnableType,
perPage: 10
})
}
let savingInputs = false
async function saveInput(args: object) {
savingInputs = true
const requestBody: CreateInput = {
name: 'Saved ' + displayDate(new Date()),
args
}
try {
let id = await InputService.createInput({
workspace: $workspaceStore!,
runnableId,
runnableType,
requestBody
})
const input = {
id,
created_by: '',
created_at: new Date().toISOString(),
is_public: false,
...requestBody
}
savedInputs = [input, ...savedInputs]
} catch (err) {
console.error(err)
sendUserToast(`Failed to save Input: ${err}`, true)
}
savingInputs = false
}
async function updateInput(input: EditableInput) {
input.isSaving = true
try {
await InputService.updateInput({
workspace: $workspaceStore!,
requestBody: {
id: input.id,
name: input.name,
is_public: input.is_public
}
})
} catch (err) {
console.error(err)
sendUserToast(`Failed to update Input: ${err}`, true)
}
input.isSaving = false
}
async function deleteInput(input: Input) {
try {
await InputService.deleteInput({
workspace: $workspaceStore!,
input: input.id
})
savedInputs = savedInputs.filter((i) => i.id !== input.id)
if (selectedInput === input) {
selectedInput = null
}
} catch (err) {
console.error(err)
sendUserToast(`Failed to delete Input: ${err}`, true)
}
}
$: {
if ($workspaceStore && (scriptHash || scriptPath || flowPath)) {
loadInputHistory()
loadSavedInputs()
}
}
const dispatch = createEventDispatcher()
const selectArgs = (selected_args: object) => {
dispatch('selected_args', selected_args)
}
</script>
<div class="min-w-[300px] h-full">
<Splitpanes horizontal={true}>
<Pane>
<div class="w-full flex flex-col gap-4 p-4">
<div class="w-full flex justify-between items-center gap-4 flex-wrap">
<span class="text-sm font-extrabold flex-shrink-0"
>Saved Inputs <Tooltip
>Shared tooltips are available to anyone with access to the script</Tooltip
></span
>
<Button
on:click={() => saveInput(args)}
disabled={!isValid}
loading={savingInputs}
startIcon={{ icon: faSave }}
color="blue"
size="xs"
>
<span>Save Current Input</span>
</Button>
</div>
<div class="w-full flex flex-col gap-2 h-full overflow-y-auto p">
{#if savedInputs.length > 0}
{#each savedInputs as i}
<button
class={classNames(
`w-full flex items-center group justify-between gap-4 py-2 px-4 text-left border rounded-md hover:bg-gray-100 transition-all`,
selectedInput === i ? 'border-blue-500 bg-blue-50' : ''
)}
on:click={() => {
if (!i.isEditing) {
if (selectedInput === i) {
selectedInput = null
} else {
selectedInput = i
}
}
}}
>
<div class="w-full h-full items-center justify-between flex gap-1 min-w-0">
{#if i.isEditing}
<form
on:submit={() => {
updateInput(i)
i.isEditing = false
i.isSaving = false
}}
class="w-full"
>
<input type="text" bind:value={i.name} class="text-gray-700" />
</form>
{:else}
<small
class="whitespace-nowrap overflow-hidden text-ellipsis flex-shrink text-left"
>
{i.name}
</small>
{/if}
{#if i.created_by == $userStore?.username || $userStore?.is_admin || $userStore?.is_super_admin}
<div class="items-center flex gap-2">
{#if !i.isEditing}
<div class="group-hover:block hidden -my-2">
<Toggle
size="xs"
options={{ right: 'shared' }}
bind:checked={i.is_public}
on:change={() => {
updateInput(i)
}}
/>
</div>
{/if}
<Button
loading={i.isSaving}
color="gray"
size="xs"
variant="border"
spacingSize="xs2"
btnClasses={'group-hover:block hidden'}
on:click={(e) => {
e.stopPropagation()
i.isEditing = !i.isEditing
if (!i.isEditing) {
updateInput(i)
i.isSaving = false
}
}}
>
<Edit class="w-4 h-4" />
</Button>
<Button
color="red"
size="xs"
spacingSize="xs2"
variant="border"
btnClasses={i.isEditing ? 'block' : 'group-hover:block hidden'}
on:click={() => deleteInput(i)}
>
<X class="w-4 h-4" />
</Button>
</div>
{:else}
<span class="text-xs text-gray-600">By {i.created_by}</span>
{/if}
</div>
</button>
{/each}
{:else}
<div class="text-center text-gray-500">No saved Inputs</div>
{/if}
</div>
</div>
</Pane>
<Pane>
<div class="w-full flex flex-col gap-4 p-4">
<span class="text-sm font-extrabold">Previous Inputs</span>
<div class="w-full flex flex-col gap-1 p-0 h-full overflow-y-auto">
{#if previousInputs.length > 0}
{#each previousInputs as i}
<button
class={classNames(
`w-full flex items-center justify-between gap-4 py-2 px-4 text-left border rounded-sm hover:bg-gray-100 transition-a`,
selectedInput === i ? 'border-blue-500 bg-blue-50' : ''
)}
on:click={() => {
if (selectedInput === i) {
selectedInput = null
} else {
selectedInput = i
}
}}
>
<div class="w-full h-full items-center flex gap-4 min-w-0">
<small
class="whitespace-nowrap overflow-hidden text-ellipsis flex-shrink text-left"
>
{displayDaysAgo(i.created_at)} by {i.created_by}
</small>
</div>
</button>
{/each}
{:else}
<div class="text-center text-gray-500">No previous Inputs</div>
{/if}
</div>
</div>
</Pane>
<Pane class="flex flex-col justify-between">
<div class="w-full flex flex-col gap-4 p-4 h-full">
<span class="text-sm font-extrabold">Preview</span>
<div class="w-full h-full overflow-auto">
{#if Object.keys(selectedInput?.args || {}).length > 0}
<div class="border h-full p-2">
<ObjectViewer json={selectedInput?.args} />
</div>
{:else}
<div class="text-center text-gray-500">
Select an Input to preview scripts arguments
</div>
{/if}
</div>
</div>
<div class="w-full flex flex-col p-4">
<Button
color="blue"
btnClasses="w-full"
size="sm"
spacingSize="xl"
on:click={() => selectArgs(selectedInput?.args)}
disabled={Object.keys(selectedInput?.args || {}).length === 0}
>
<ArrowLeftIcon class="w-4 h-4 mr-2" />
Use Input
</Button>
</div>
</Pane>
</Splitpanes>
</div>

View File

@@ -22,18 +22,17 @@
<div
class="pointer-events-auto w-full max-w-sm overflow-hidden bg-white shadow-lg ring-1 ring-black ring-opacity-5 border"
>
<div class="p-4">
<div class="p-2 min-h-[60px]">
<div class="flex items-start">
<div class="flex-shrink-0">
<div class="flex-shrink-0 mt-0.5">
{#if error}
<XCircleIcon class="h-6 w-6 text-red-400" />
<XCircleIcon class="h-4 w-4 text-red-400" />
{:else}
<CheckCircle2 class="h-6 w-6 text-green-400" />
<CheckCircle2 class="h-4 w-4 text-green-400" />
{/if}
</div>
<div class="ml-3 w-0 flex-1 pt-0.5">
<p class="text-sm font-medium text-gray-900">{error ? 'Error' : 'Success'}</p>
<p class="mt-1 text-sm text-gray-500">{message}</p>
<div class="ml-3 w-0 flex-1">
<p class="text-sm text-gray-500">{message}</p>
</div>
<div class="ml-4 flex flex-shrink-0">
<button

View File

@@ -53,9 +53,9 @@
}}
/>
<div
class="w-11 h-6 bg-gray-200 rounded-full peer peer-focus:ring-4 peer-focus:ring-blue-300
peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
after:absolute after:top-0.5 after:left-[2px] after:bg-white after:border-gray-300
class="w-11 h-6 bg-gray-200 rounded-full peer peer-focus:ring-4 peer-focus:ring-blue-300
peer-checked:after:translate-x-full peer-checked:after:border-white after:content-['']
after:absolute after:top-0.5 after:left-[2px] after:bg-white after:border-gray-300
after:border after:rounded-full after:h-5 after:w-5 after:transition-all {color == 'red'
? 'peer-checked:bg-red-600'
: 'peer-checked:bg-blue-600'}"

View File

@@ -67,8 +67,6 @@
}
}
$: resolvedConfig?.triggerOnAppLoad && runnableComponent?.runComponent()
let errors: Record<string, string> = {}
$: errorsMessage = Object.values(errors)
.filter((x) => x != '')
@@ -119,6 +117,7 @@
{render}
{outputs}
{extraKey}
refreshOnStart={resolvedConfig.triggerOnAppLoad}
>
<AlignWrapper {noWFull} {horizontalAlignment} {verticalAlignment}>
{#if errorsMessage}

View File

@@ -59,7 +59,7 @@
title="sandbox"
srcdoc={result
? '<scr' + `ipt type="application/javascript" src="/tailwind.js"></script>` + result
: 'No html'}
: ''}
/>
{/key}
<!-- svelte-ignore a11y-click-events-have-key-events -->

View File

@@ -5,6 +5,7 @@
import { concatCustomCss } from '../../utils'
import { AlignWrapper, InputValue } from '../helpers'
import { loadIcon } from '../icon'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let horizontalAlignment: 'left' | 'center' | 'right' | undefined = 'left'
@@ -38,6 +39,8 @@
<InputValue {id} input={configuration.color} bind:value={color} />
<InputValue {id} input={configuration.strokeWidth} bind:value={strokeWidth} />
<InitializeComponent {id} />
<AlignWrapper
{render}
{horizontalAlignment}

View File

@@ -7,6 +7,7 @@
import { concatCustomCss } from '../../utils'
import Loader from '../helpers/Loader.svelte'
import ResolveConfig from '../helpers/ResolveConfig.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -31,6 +32,8 @@
$: css = concatCustomCss($app.css?.imagecomponent, customCss)
</script>
<InitializeComponent {id} />
{#each Object.keys(components['imagecomponent'].initialData.configuration) as key (key)}
<ResolveConfig
{id}

View File

@@ -11,6 +11,7 @@
import { Point } from 'ol/geom'
import { defaults as defaultControls } from 'ol/control'
import { findGridItem, initOutput } from '../../editor/appUtils'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
interface Marker {
lon: number
@@ -193,6 +194,8 @@
<InputValue {id} input={configuration.zoom} bind:value={zoom} />
<InputValue {id} input={configuration.markers} bind:value={markers} />
<InitializeComponent {id} />
{#if render}
<div class="relative h-full w-full">
<div

View File

@@ -11,6 +11,7 @@
import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
import { concatCustomCss } from '../../utils'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -192,6 +193,8 @@
<InputValue {id} input={configuration.source} bind:value={source} />
<InputValue {id} input={configuration.zoom} bind:value={zoom} />
<InitializeComponent {id} />
{#if render}
<div class="relative flex flex-col w-full h-full bg-gray-100">
{#if source && zoom}

View File

@@ -226,7 +226,7 @@
)}
>
{String(result)}
{#if resolvedConfig.tooltip != ''}
{#if resolvedConfig.tooltip && resolvedConfig.tooltip != ''}
<Tooltip>{resolvedConfig.tooltip}</Tooltip>
{/if}
{#if resolvedConfig.copyButton && result}

View File

@@ -25,6 +25,7 @@
{placeholder}
on:pointerdown|stopPropagation
on:keyup={debounce}
on:keydown|stopPropagation
class={parentClass}
{...$$restProps}
/>

View File

@@ -1,5 +1,5 @@
<script lang="ts">
import { getContext } from 'svelte'
import { getContext, onMount } from 'svelte'
import { initOutput } from '../../editor/appUtils'
import type {
ConnectedAppInput,
@@ -16,13 +16,20 @@
export let fields: Record<string, StaticAppInput | ConnectedAppInput | RowAppInput | UserAppInput>
export let recomputeOnInputChanged: boolean
export let recomputableByRefreshButton: boolean
export let noBackendValue: any = undefined
let result: any = undefined
const { worldStore, staticExporter, noBackend } = getContext<AppViewerContext>('AppViewerContext')
const { worldStore } = getContext<AppViewerContext>('AppViewerContext')
let result: any = noBackend ? noBackendValue : undefined
onMount(() => {
$staticExporter[id] = () => {
return result
}
})
let outputs = initOutput($worldStore, id, {
result: undefined,
result: result,
loading: false
})
</script>

View File

@@ -0,0 +1,17 @@
<script lang="ts">
import { getContext, onDestroy, onMount } from 'svelte'
import type { AppViewerContext } from '../../types'
export let id: string
const { initialized } = getContext<AppViewerContext>('AppViewerContext')
onMount(() => {
if (!$initialized.initializedComponents.includes(id)) {
$initialized.initializedComponents = [...$initialized.initializedComponents, id]
}
})
onDestroy(() => {
$initialized.initializedComponents = $initialized.initializedComponents.filter((c) => c !== id)
})
</script>

View File

@@ -41,8 +41,15 @@
$: stateId = $worldStore?.stateId
let timeout: NodeJS.Timeout | undefined = undefined
let firstDebounce = true
const debounce_ms = 50
function debounce(cb: () => Promise<void>) {
if (firstDebounce) {
firstDebounce = false
cb()
return
}
if (timeout) {
clearTimeout(timeout)
}

View File

@@ -4,6 +4,7 @@
import type { Output } from '../../rx'
import type { AppViewerContext } from '../../types'
import InputValue from './InputValue.svelte'
import InitializeComponent from './InitializeComponent.svelte'
export let componentInput: AppInput
export let id: string
@@ -25,6 +26,8 @@
$: result && outputs && setOutput(result)
</script>
<InitializeComponent {id} />
{#if componentInput.type !== 'runnable'}
<InputValue {id} input={componentInput} bind:value={result} />
{/if}

View File

@@ -8,13 +8,14 @@
import { classNames, defaultIfEmptyString, emptySchema, sendUserToast } from '$lib/utils'
import { deepEqual } from 'fast-equals'
import { Bug } from 'lucide-svelte'
import { createEventDispatcher, getContext } from 'svelte'
import { createEventDispatcher, getContext, onDestroy } from 'svelte'
import type { AppInputs, Runnable } from '../../inputType'
import type { Output } from '../../rx'
import type { AppViewerContext, InlineScript } from '../../types'
import { computeGlobalContext, eval_like } from './eval'
import InputValue from './InputValue.svelte'
import RefreshButton from './RefreshButton.svelte'
import { selectId } from '../../editor/appUtils'
// Component props
export let id: string
@@ -31,9 +32,10 @@
export let render: boolean
export let outputs: { result: Output<any>; loading: Output<boolean> }
export let extraKey = ''
export let recomputeOnInputChanged: boolean = false
export let recomputeOnInputChanged: boolean = true
export let loading = false
export let recomputableByRefreshButton: boolean = true
export let refreshOnStart: boolean = false
const {
worldStore,
@@ -47,17 +49,31 @@
mode,
stateId,
state,
componentControl
componentControl,
initialized,
selectedComponent,
app,
connectingInput
} = getContext<AppViewerContext>('AppViewerContext')
const dispatch = createEventDispatcher()
$runnableComponents[id] = {
autoRefresh: autoRefresh && recomputableByRefreshButton,
refreshOnStart,
cb: async (inlineScript?: InlineScript) => {
await executeComponent(true, inlineScript)
}
}
if (!$initialized.initializedComponents.includes(id)) {
$initialized.initializedComponents = [...$initialized.initializedComponents, id]
}
onDestroy(() => {
$initialized.initializedComponents = $initialized.initializedComponents.filter((c) => c !== id)
})
$runnableComponents = $runnableComponents
let args: Record<string, any> | undefined = undefined
@@ -101,7 +117,7 @@
function refreshIfAutoRefresh(_src: string) {
const refreshEnabled =
autoRefresh && ((recomputeOnInputChanged ?? true) || refreshOn?.length > 0)
if (refreshEnabled && $worldStore.initialized) {
if (refreshEnabled && $initialized.initialized) {
setDebouncedExecute()
}
}
@@ -174,7 +190,7 @@
$state = $state
} catch (e) {
sendUserToast('Error running frontend script: ' + e.message, true)
sendUserToast(`Error running frontend script ${id}: ` + e.message, true)
// Manually add a fake job to the job list to show the error
@@ -190,8 +206,7 @@
}
loading = false
return
}
if (noBackend) {
} else if (noBackend) {
if (!noToast) {
sendUserToast('This app is not connected to a windmill backend, it is a static preview')
}
@@ -319,6 +334,11 @@
$errorByComponent = $errorByComponent
}
}
function handleInputClick(e: CustomEvent) {
const event = e as unknown as PointerEvent
!$connectingInput.opened && selectId(event, id, selectedComponent, $app)
}
</script>
{#each Object.entries(fields ?? {}) as [key, v] (key)}
@@ -365,7 +385,11 @@
<div class="h-full flex relative flex-row flex-wrap {wrapperClass}" style={wrapperStyle}>
{#if (autoRefresh || forceSchemaDisplay) && schemaStripped && Object.keys(schemaStripped?.properties ?? {}).length > 0}
<div class="px-2 h-fit min-h-0">
<LightweightSchemaForm schema={schemaStripped} bind:args />
<LightweightSchemaForm
schema={schemaStripped}
bind:args
on:inputClicked={handleInputClick}
/>
</div>
{/if}

View File

@@ -8,6 +8,7 @@
import RunnableComponent from './RunnableComponent.svelte'
import { goto } from '$app/navigation'
import { sendUserToast } from '$lib/utils'
import InitializeComponent from './InitializeComponent.svelte'
export let componentInput: AppInput | undefined
export let id: string
@@ -39,6 +40,7 @@
export let recomputeIds: string[] = []
export let outputs: { result: Output<any>; loading: Output<boolean> }
export let extraKey: string | undefined = undefined
export let refreshOnStart: boolean = false
const { staticExporter, noBackend, componentControl, runnableComponents } =
getContext<AppViewerContext>('AppViewerContext')
@@ -51,7 +53,9 @@
result = componentInput?.['value']
}
onMount(() => {
$staticExporter[id] = () => result
$staticExporter[id] = () => {
return result
}
})
function isRunnableDefined(componentInput) {
@@ -96,9 +100,11 @@
</script>
{#if componentInput === undefined}
<InitializeComponent {id} />
<slot />
{:else if componentInput.type === 'runnable' && isRunnableDefined(componentInput)}
<RunnableComponent
{refreshOnStart}
{extraKey}
bind:loading
bind:this={runnableComponent}

View File

@@ -6,6 +6,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -34,6 +35,8 @@
<InputValue {id} input={configuration.label} bind:value={labelValue} />
<InputValue {id} input={configuration.defaultValue} bind:value={defaultValue} />
<InitializeComponent {id} />
<AlignWrapper {render} {horizontalAlignment} {verticalAlignment}>
<Toggle
checked={defaultValue}

View File

@@ -6,6 +6,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -41,6 +42,8 @@
<InputValue {id} input={configuration.maxDate} bind:value={maxValue} />
<InputValue {id} input={configuration.defaultValue} bind:value={defaultValue} />
<InitializeComponent {id} />
<AlignWrapper {render} {verticalAlignment}>
{#if inputType === 'date'}
<input

View File

@@ -6,6 +6,7 @@
import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
import { concatCustomCss } from '../../utils'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -19,11 +20,11 @@
let text: string | undefined = undefined
let outputs = initOutput($worldStore, id, {
result: [] as string[] | undefined
result: [] as { name: string; data: string }[] | undefined
})
// Receives Base64 encoded strings from the input component
async function handleChange(files: string[] | undefined) {
async function handleChange(files: { name: string; data: string }[] | undefined) {
outputs?.result.set(files)
}
@@ -34,12 +35,15 @@
<InputValue {id} input={configuration.allowMultiple} bind:value={allowMultiple} />
<InputValue {id} input={configuration.text} bind:value={text} />
<InitializeComponent {id} />
{#if render}
<div class="w-full h-full p-1">
<FileInput
accept={acceptedFileTypes?.length ? acceptedFileTypes?.join(', ') : undefined}
multiple={allowMultiple}
convertTo="base64"
returnFileNames
on:change={({ detail }) => {
handleChange(detail)
}}

View File

@@ -7,6 +7,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -55,6 +56,8 @@
<InputValue {id} input={configuration.items} bind:value={labels} />
<InputValue {id} input={configuration.placeholder} bind:value={placeholder} />
<InitializeComponent {id} />
<AlignWrapper {render} {horizontalAlignment} {verticalAlignment}>
<div
class="app-select w-full"

View File

@@ -6,6 +6,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -44,6 +45,8 @@
<InputValue {id} input={configuration.placeholder} bind:value={placeholder} />
<InputValue {id} input={configuration.defaultValue} bind:value={defaultValue} />
<InitializeComponent {id} />
<AlignWrapper {render} {verticalAlignment}>
<input
on:pointerdown|stopPropagation={() => ($selectedComponent = [id])}

View File

@@ -7,6 +7,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -54,6 +55,8 @@
bind:value={values[1]}
/>
<InitializeComponent {id} />
<AlignWrapper {render} {verticalAlignment}>
<div class="flex flex-col w-full">
<div class="flex items-center w-full gap-1 px-1">

View File

@@ -8,6 +8,7 @@
import InputValue from '../helpers/InputValue.svelte'
import { SELECT_INPUT_DEFAULT_STYLE } from '../../../../defaults'
import { initOutput } from '../../editor/appUtils'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -103,6 +104,8 @@
<InputValue {id} input={configuration.defaultValue} bind:value={defaultValue} />
<InputValue {id} input={configuration.create} bind:value={create} />
<InitializeComponent {id} />
<AlignWrapper {render} {horizontalAlignment} {verticalAlignment}>
<div
class="app-select w-full"

View File

@@ -7,6 +7,7 @@
import { twMerge } from 'tailwind-merge'
import { concatCustomCss } from '../../utils'
import { initOutput } from '../../editor/appUtils'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -59,6 +60,7 @@
input={configuration.defaultValue}
bind:value={values[0]}
/>
<InitializeComponent {id} />
<AlignWrapper {render} {verticalAlignment}>
<div class="flex items-center w-full gap-1 px-1">

View File

@@ -6,6 +6,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -44,6 +45,8 @@
<InputValue {id} input={configuration.placeholder} bind:value={placeholder} />
<InputValue {id} input={configuration.defaultValue} bind:value={defaultValue} />
<InitializeComponent {id} />
{#if inputType === 'textarea'}
<textarea
class={twMerge(

View File

@@ -7,6 +7,7 @@
import AlignWrapper from '../../helpers/AlignWrapper.svelte'
import InputValue from '../../helpers/InputValue.svelte'
import CurrencyInput from './CurrencyInput.svelte'
import InitializeComponent from '../../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -48,6 +49,8 @@
<InputValue {id} input={configuration.currency} bind:value={currency} />
<InputValue {id} input={configuration.locale} bind:value={locale} />
<InitializeComponent {id} />
<AlignWrapper {render} {verticalAlignment}>
{#key isNegativeAllowed}
{#key locale}

View File

@@ -4,6 +4,7 @@
import SubGridEditor from '../../editor/SubGridEditor.svelte'
import type { AppViewerContext, ComponentCustomCSS } from '../../types'
import { concatCustomCss } from '../../utils'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let componentContainerHeight: number
@@ -26,6 +27,8 @@
$: css = concatCustomCss($app.css?.containercomponent, customCss)
</script>
<InitializeComponent {id} />
<div class="w-full h-full">
{#if $app.subgrids?.[`${id}-0`]}
<SubGridEditor

View File

@@ -12,6 +12,7 @@
import { concatCustomCss } from '../../utils'
import AlignWrapper from '../helpers/AlignWrapper.svelte'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -36,6 +37,8 @@
<InputValue {id} input={configuration.size} bind:value={size} />
<InputValue {id} input={configuration.color} bind:value={color} />
<InitializeComponent {id} />
<AlignWrapper
{horizontalAlignment}
{verticalAlignment}

View File

@@ -9,6 +9,7 @@
import { twMerge } from 'tailwind-merge'
import { AlignWrapper } from '../helpers'
import { initOutput } from '../../editor/appUtils'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let customCss: ComponentCustomCSS<'drawercomponent'> | undefined = undefined
export let id: string
@@ -70,6 +71,8 @@
<InputValue {id} input={configuration.size} bind:value={size} />
<InputValue {id} input={configuration.fillContainer} bind:value={fillContainer} />
<InitializeComponent {id} />
<Portal target="#app-editor-top-level-drawer">
<Drawer let:open bind:this={appDrawer} size="800px" alwaysOpen positionClass="!absolute">
<DrawerContent

View File

@@ -6,6 +6,7 @@
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { deepEqual } from 'fast-equals'
import { initOutput } from '../../editor/appUtils'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let componentContainerHeight: number
@@ -65,6 +66,8 @@
}
</script>
<InitializeComponent {id} />
<div class="h-full w-full border" on:pointerdown={onFocus}>
<Splitpanes {horizontal}>
{#each sumedup as paneSize, index (index)}

View File

@@ -7,6 +7,7 @@
import type { AppViewerContext, ComponentCustomCSS, RichConfigurations } from '../../types'
import { concatCustomCss } from '../../utils'
import InputValue from '../helpers/InputValue.svelte'
import InitializeComponent from '../helpers/InitializeComponent.svelte'
export let id: string
export let configuration: RichConfigurations
@@ -78,6 +79,8 @@
<InputValue {id} input={configuration.tabsKind} bind:value={resolvedConfig.tabsKind} />
<InitializeComponent {id} />
<div class={resolvedConfig.tabsKind == 'sidebar' ? 'flex gap-4 w-full' : 'w-full'}>
{#if !resolvedConfig.tabsKind || resolvedConfig.tabsKind == 'tabs' || (resolvedConfig.tabsKind == 'invisibleOnView' && $mode == 'dnd')}
<div bind:clientHeight={tabHeight}>

View File

@@ -80,6 +80,7 @@
worldStore,
app: appStore,
summary: summaryStore,
initialized: writable({ initialized: false, initializedComponents: [] }),
selectedComponent,
mode,
connectingInput,

View File

@@ -25,7 +25,6 @@
faClipboard,
faExternalLink,
faFileExport,
faGlobe,
faSave
} from '@fortawesome/free-solid-svg-icons'
import {
@@ -42,7 +41,7 @@
import { getContext } from 'svelte'
import { Icon } from 'svelte-awesome'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { appToHubUrl, classNames, copyToClipboard, sendUserToast } from '../../../utils'
import { classNames, copyToClipboard, sendUserToast } from '../../../utils'
import type {
AppInput,
ConnectedAppInput,
@@ -257,20 +256,20 @@
switch (event.key) {
case 'Z':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
$app = redo(history)
event.preventDefault()
}
break
case 'z':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
$app = undo(history, $app)
event.preventDefault()
}
break
case 's':
if (event.ctrlKey) {
if (event.ctrlKey || event.metaKey) {
save()
event.preventDefault()
}
@@ -607,14 +606,14 @@
appExport.open($app)
}
},
{
displayName: 'Publish to Hub',
icon: faGlobe,
action: () => {
const url = appToHubUrl(toStatic($app, $staticExporter, $summary))
window.open(url.toString(), '_blank')
}
},
// {
// displayName: 'Publish to Hub',
// icon: faGlobe,
// action: () => {
// const url = appToHubUrl(toStatic($app, $staticExporter, $summary))
// window.open(url.toString(), '_blank')
// }
// },
{
displayName: 'Hub compatible JSON',
icon: faFileExport,

View File

@@ -48,6 +48,7 @@
const parentWidth = writable(0)
setContext<AppViewerContext>('AppViewerContext', {
worldStore: buildWorld(context),
initialized: writable({ initialized: false, initializedComponents: [] }),
app: appStore,
summary: writable(summary),
selectedComponent,
@@ -171,6 +172,7 @@
fields={script.fields}
recomputeOnInputChanged={script.recomputeOnInputChanged ?? true}
recomputableByRefreshButton={script.autoRefresh ?? false}
noBackendValue={script.noBackendValue}
/>
{/if}
{/each}

View File

@@ -22,7 +22,6 @@
app,
mode,
connectingInput,
runnableComponents,
summary,
focusedGrid,
parentWidth,
@@ -39,40 +38,6 @@
.flatMap((id) => dfs($app.grid, id, $app.subgrids ?? {}))
.filter((x) => x != undefined) as string[]
}
function removeGridElement(component) {
if (component) {
$app.grid = $app.grid.filter((gridComponent) => {
if (gridComponent.data.id === component.id) {
if (
gridComponent.data.componentInput?.type === 'runnable' &&
gridComponent.data.componentInput?.runnable?.type === 'runnableByName' &&
gridComponent.data.componentInput?.runnable.inlineScript
) {
const { name, inlineScript } = gridComponent.data.componentInput?.runnable
if (!$app.unusedInlineScripts) {
$app.unusedInlineScripts = []
}
$app.unusedInlineScripts.push({
name,
inlineScript
})
$app = $app
}
}
return gridComponent.data.id !== component?.id
})
delete $runnableComponents[component.id]
$runnableComponents = $runnableComponents
$selectedComponent = undefined
}
}
</script>
<div class="relative w-full z-20 overflow-visible">
@@ -135,7 +100,6 @@
component={dataItem.data}
selected={Boolean($selectedComponent?.includes(dataItem.id))}
locked={isFixed(dataItem)}
on:delete={() => removeGridElement(dataItem.data)}
on:lock={() => {
const gridItem = findGridItem($app, dataItem.id)
if (gridItem) {

View File

@@ -6,7 +6,7 @@
import type { AppViewerContext } from '../types'
import { allItems } from '../utils'
const { runnableComponents, app, worldStore } = getContext<AppViewerContext>('AppViewerContext')
const { runnableComponents, app, initialized } = getContext<AppViewerContext>('AppViewerContext')
let loading: boolean = false
let timeout: NodeJS.Timer | undefined = undefined
let interval: number | undefined = undefined
@@ -14,7 +14,7 @@
let firstLoad = false
$: !firstLoad &&
$worldStore.initializedOutputs ==
$initialized.initializedComponents?.length ==
allItems($app.grid, $app.subgrids).length + $app.hiddenInlineScripts.length &&
refresh()
$: componentNumber = Object.values($runnableComponents).filter((x) => x.autoRefresh).length
@@ -39,14 +39,19 @@
}
function refresh() {
let isFirstLoad = false
if (!firstLoad) {
$worldStore.initialized = true
$initialized.initialized = true
firstLoad = true
isFirstLoad = true
}
loading = true
Promise.all(
Object.keys($runnableComponents).map((id) => {
if (!$runnableComponents?.[id]?.autoRefresh) {
if (
!$runnableComponents?.[id]?.autoRefresh &&
(!isFirstLoad || !$runnableComponents?.[id]?.refreshOnStart)
) {
return
}
return $runnableComponents?.[id]?.cb?.()

View File

@@ -103,11 +103,11 @@
autoRefresh={hiddenInlineScript.script.autoRefresh}
id={`bg_${hiddenInlineScript.index}`}
bind:recomputeOnInputChanged={hiddenInlineScript.script.recomputeOnInputChanged}
doNotRecomputeOnInputChanged={hiddenInlineScript.script.doNotRecomputeOnInputChanged}
bind:doNotRecomputeOnInputChanged={hiddenInlineScript.script.doNotRecomputeOnInputChanged}
bind:inlineScript={hiddenInlineScript.script.inlineScript}
/>
{:else}
<span class="text-gray-600 text-xs">No hiddenInlineScript.script defined</span>
<span class="text-gray-600 text-xs">No script defined</span>
{/if}
</div>
{#if Object.keys(hiddenInlineScript.script.fields).length > 0}

View File

@@ -32,7 +32,10 @@ export function dfs(
for (const item of grid) {
if (item.id === id) {
return [id]
} else if (item.data.type == 'tablecomponent' && item.data.actionButtons.find((x) => x.id)) {
} else if (
item.data.type == 'tablecomponent' &&
item.data.actionButtons.find((x) => id == x.id)
) {
return [item.id, id]
} else {
for (let i = 0; i < (item.data.numberOfSubgrids ?? 0); i++) {
@@ -53,13 +56,14 @@ export function selectId(
app: App
) {
// this ensure handleClickOutside are triggered
window.dispatchEvent(
new MouseEvent('click', {
view: window,
bubbles: true,
cancelable: true
})
)
let event = new MouseEvent('click', {
view: window,
bubbles: true,
cancelable: true,
relatedTarget: e.target
})
window.dispatchEvent(event)
if (e.shiftKey) {
selectedComponent.update((old) => {
if (old && old?.[0]) {
@@ -132,11 +136,14 @@ export function findGridItem(app: App, id: string): GridItem | undefined {
}
export function getNextGridItemId(app: App): string {
const subgridsKeys = allItems(app.grid, app.subgrids).map((x) => x.id)
const withoutDash = subgridsKeys.map((element) => element.split('-')[0])
const id = getNextId([...new Set(withoutDash)])
return id
const allIds = allItems(app.grid, app.subgrids).flatMap((x) => {
if (x.data.type === 'tablecomponent') {
return [x.id, ...x.data.actionButtons.map((x) => x.id)]
} else {
return [x.id]
}
})
return getNextId(allIds)
}
export function getAllRecomputeIdsForComponent(app: App, id: string | undefined) {
@@ -453,7 +460,6 @@ export function initOutput<I extends Record<string, any>>(
id: string,
init: I
): Outputtable<I> {
world.initializedOutputs += 1
if (!world) {
return {} as any
}

View File

@@ -79,7 +79,6 @@
{component}
{selected}
shouldHideActions={$connectingInput.opened}
on:delete
on:lock
on:expand
{locked}

View File

@@ -12,7 +12,7 @@
export let first: boolean = false
</script>
<OutputHeader selectable={false} {id} {name} color="blue" {first}>
<OutputHeader renamable={false} selectable={true} {id} {name} color="blue" {first}>
<ComponentOutputViewer
componentId={id}
on:select={({ detail }) => {

View File

@@ -6,6 +6,8 @@
import { getContext } from 'svelte'
import { allsubIds, findGridItem } from '../../appUtils'
import IdEditor from './IdEditor.svelte'
import type { AppComponent } from '../../component'
import type { Runnable } from '$lib/components/apps/inputType'
export let id: string
export let name: string
@@ -13,6 +15,7 @@
export let nested: boolean = false
export let color: 'blue' | 'indigo' = 'indigo'
export let selectable: boolean = true
export let renamable: boolean = true
const { manuallyOpened, search, hasResult } = getContext<ContextPanelContext>('ContextPanel')
@@ -47,66 +50,102 @@
}
function renameId(newId: string): void {
{
const item = findGridItem($app, id)
if (item) {
item.data.id = newId
item.id = newId
}
const oldSubgrids = Object.keys($app.subgrids ?? {}).filter((subgrid) =>
subgrid.startsWith(id + '-')
)
oldSubgrids.forEach((subgrid) => {
if ($app.subgrids) {
$app.subgrids[subgrid.replace(id, newId)] = $app.subgrids[subgrid]
delete $app.subgrids[subgrid]
}
})
allItems($app.grid, $app.subgrids).forEach((item) => {
if (item.data.componentInput?.type == 'connected') {
if (item.data.componentInput.connection?.componentId === id) {
item.data.componentInput.connection.componentId = newId
}
} else if (item.data.componentInput?.type == 'runnable') {
if (
item.data.componentInput?.runnable?.type === 'runnableByName' &&
item.data.componentInput?.runnable?.inlineScript?.refreshOn
?.map((x) => x.id)
?.includes(id)
) {
item.data.componentInput.runnable.inlineScript.refreshOn =
item.data.componentInput.runnable.inlineScript.refreshOn.map((x) => {
if (x.id === id) {
return {
id: newId,
key: x.key
}
}
return x
})
}
}
const item = findGridItem($app, id)
Object.values(item.data.configuration ?? {}).forEach((config) => {
if (config.type === 'connected') {
if (config.connection?.componentId === id) {
config.connection.componentId = newId
}
} else if (config.type == 'oneOf') {
Object.values(config.configuration ?? {}).forEach((choices) => {
Object.values(choices).forEach((c) => {
if (c.type === 'connected') {
if (c.connection?.componentId === id) {
c.connection.componentId = newId
}
}
})
})
}
if (!item) {
return
}
item.data.id = newId
item.id = newId
const oldSubgrids = Object.keys($app.subgrids ?? {}).filter((subgrid) =>
subgrid.startsWith(id + '-')
)
oldSubgrids.forEach((subgrid) => {
if ($app.subgrids) {
$app.subgrids[subgrid.replace(id, newId)] = $app.subgrids[subgrid]
delete $app.subgrids[subgrid]
}
})
function propagateRename(from: string, to: string) {
allItems($app.grid, $app.subgrids).forEach((item) => {
renameComponent(from, to, item.data)
})
$app.hiddenInlineScripts.forEach((x) => {
console.log('process', x.name, id)
processRunnable(from, to, {
name: x.name,
inlineScript: x.inlineScript,
type: 'runnableByName'
})
})
$app = $app
$selectedComponent = [newId]
}
propagateRename(id, newId)
if (item?.data.type == 'tablecomponent') {
for (let c of item.data.actionButtons) {
let old = c.id
c.id = c.id.replace(id + '_', newId + '_')
propagateRename(old, c.id)
}
}
$app = $app
$selectedComponent = [newId]
}
function renameComponent(from: string, to: string, data: AppComponent) {
if (data.type == 'tablecomponent') {
for (let c of data.actionButtons) {
renameComponent(from, to, c)
}
}
let componentInput = data.componentInput
if (componentInput?.type == 'connected') {
if (componentInput.connection?.componentId === from) {
componentInput.connection.componentId = to
}
} else if (componentInput?.type == 'runnable') {
processRunnable(from, to, componentInput.runnable)
}
Object.values(data.configuration ?? {}).forEach((config) => {
if (config.type === 'connected') {
if (config.connection?.componentId === from) {
config.connection.componentId = to
}
} else if (config.type == 'oneOf') {
Object.values(config.configuration ?? {}).forEach((choices) => {
Object.values(choices).forEach((c) => {
if (c.type === 'connected') {
if (c.connection?.componentId === id) {
c.connection.componentId = to
}
}
})
})
}
})
}
function processRunnable(from: string, to: string, runnable: Runnable) {
if (
runnable?.type === 'runnableByName' &&
runnable?.inlineScript?.refreshOn?.find((x) => x.id === from)
) {
console.log('processss')
runnable.inlineScript.refreshOn = runnable.inlineScript.refreshOn.map((x) => {
console.log('renaming', x)
if (x.id === from) {
return {
id: to,
key: x.key
}
}
return x
})
}
}
</script>
@@ -161,7 +200,7 @@
</div>
{/if}
</button>
{#if selectable && ($selectedComponent?.includes(id) || $hoverStore === id)}
{#if selectable && renamable && ($selectedComponent?.includes(id) || $hoverStore === id)}
<IdEditor
{id}
on:selected={() => ($selectedComponent = [id])}

View File

@@ -11,7 +11,7 @@
export let first: boolean = false
</script>
<OutputHeader {id} name={'Table action'} {first}>
<OutputHeader renamable={false} {id} name={'Table action'} {first}>
<ComponentOutputViewer
componentId={id}
on:select={({ detail }) => {

View File

@@ -150,7 +150,7 @@
startIcon={{ icon: faCodeBranch }}
btnClasses="truncate"
>
fork a detached script
Copy a script
</Button>
<Button
on:click={() => dispatch('delete')}

Some files were not shown because too many files have changed in this diff Show More