Compare commits
1 Commits
v1.173.0
...
fix-chartj
| Author | SHA1 | Date | |
|---|---|---|---|
|
b3fbf69b35 |
3
.github/DockerfileBackendTests
vendored
3
.github/DockerfileBackendTests
vendored
@@ -37,10 +37,9 @@ RUN apt-get update \
|
||||
make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
|
||||
libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
|
||||
libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
|
||||
libv8-dev tesseract-ocr nodejs npm clang libclang-dev\
|
||||
libv8-dev tesseract-ocr nodejs npm\
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
|
||||
ENV PATH="${PATH}:/usr/local/go/bin"
|
||||
ENV GO_PATH=/usr/local/go/bin/go
|
||||
|
||||
2
.github/workflows/backend-test.yml
vendored
2
.github/workflows/backend-test.yml
vendored
@@ -42,4 +42,4 @@ jobs:
|
||||
mkdir frontend/build && cd backend && touch
|
||||
windmill-api/openapi-deref.yaml &&
|
||||
DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
|
||||
--features enterprise test --all -- --nocapture
|
||||
test --all -- --nocapture
|
||||
|
||||
1
.github/workflows/benchmark.yml
vendored
1
.github/workflows/benchmark.yml
vendored
@@ -40,7 +40,6 @@ jobs:
|
||||
run: deno run --unstable -A
|
||||
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
|
||||
--host http://localhost:8000
|
||||
-r
|
||||
-e admin@windmill.dev
|
||||
-p changeme
|
||||
-c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
|
||||
|
||||
79
CHANGELOG.md
79
CHANGELOG.md
@@ -1,85 +1,6 @@
|
||||
# Changelog
|
||||
|
||||
|
||||
## [1.173.0](https://github.com/windmill-labs/windmill/compare/v1.172.1...v1.173.0) (2023-09-14)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* cli sync on windows ([#2283](https://github.com/windmill-labs/windmill/issues/2283)) ([c371cb3](https://github.com/windmill-labs/windmill/commit/c371cb397ab3d0c534e2c553d1dfb1ad5176d2a6))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* accept jobs whose duration > 24 days ([2c00894](https://github.com/windmill-labs/windmill/commit/2c00894122aa8caee59b20625935284de6902950))
|
||||
|
||||
## [1.172.1](https://github.com/windmill-labs/windmill/compare/v1.172.0...v1.172.1) (2023-09-14)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* improve splitpane + improve deleting conditional tab ([1629008](https://github.com/windmill-labs/windmill/commit/1629008eb2eb48ff9cc2cf6b3a351efcf682244d))
|
||||
* update to svelte 4 ([#2280](https://github.com/windmill-labs/windmill/issues/2280)) ([90c10d8](https://github.com/windmill-labs/windmill/commit/90c10d803b4c47a9e1ac5b9e49e2a614344299a9))
|
||||
|
||||
## [1.172.0](https://github.com/windmill-labs/windmill/compare/v1.171.0...v1.172.0) (2023-09-13)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* improve ai flow ([#2270](https://github.com/windmill-labs/windmill/issues/2270)) ([b23417a](https://github.com/windmill-labs/windmill/commit/b23417ab5b9938bbdf9db6449102760ff8c80152))
|
||||
* worker groups admin panel ([#2277](https://github.com/windmill-labs/windmill/issues/2277)) ([070b162](https://github.com/windmill-labs/windmill/commit/070b16222bc666866284180b3878f4d4f27bfa85))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* ai flow nits ([#2272](https://github.com/windmill-labs/windmill/issues/2272)) ([8f6f46d](https://github.com/windmill-labs/windmill/commit/8f6f46de199d58133b9faa77cdbcbcfd6cb962f7))
|
||||
|
||||
## [1.171.0](https://github.com/windmill-labs/windmill/compare/v1.170.0...v1.171.0) (2023-09-12)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* attempt to SIGTERM before SIGKILL for bash ([f40bbba](https://github.com/windmill-labs/windmill/commit/f40bbba519a97cbb1ec142c335f038dbebcd4e7c))
|
||||
* zero copy result for job result ([#2263](https://github.com/windmill-labs/windmill/issues/2263)) ([22a7da5](https://github.com/windmill-labs/windmill/commit/22a7da58b1d20721892906cba2dee6fbeb1cc1fd))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* 2257 TIME convertion in pg_executor.rs ([#2267](https://github.com/windmill-labs/windmill/issues/2267)) ([3d71253](https://github.com/windmill-labs/windmill/commit/3d71253abdb0dff1670a796d07a53ecd0a98414e))
|
||||
* fix field duplicate in app background settings ([164cdaf](https://github.com/windmill-labs/windmill/commit/164cdaf09464646dee4e70a699222a454eb0d898))
|
||||
* improve bun lockfile resolution ([9103ec4](https://github.com/windmill-labs/windmill/commit/9103ec445db81395a5851202eecb87301d0b4987))
|
||||
* remove result and args from list completed and list queue jobs ([e7e63e1](https://github.com/windmill-labs/windmill/commit/e7e63e111a73e0986050a8fe7fdc18784ba902b0))
|
||||
|
||||
## [1.170.0](https://github.com/windmill-labs/windmill/compare/v1.169.0...v1.170.0) (2023-09-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* display jobs currently waiting for a worker ([3c950c0](https://github.com/windmill-labs/windmill/commit/3c950c03de0bc71974eb29985381adba8c098660))
|
||||
* snowflake schema explorer + refactoring ([#2260](https://github.com/windmill-labs/windmill/issues/2260)) ([5cca583](https://github.com/windmill-labs/windmill/commit/5cca5833e94fc4c8a80e210164da09f2a1ceb677))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* fix get_result for python-client ([fe41f4f](https://github.com/windmill-labs/windmill/commit/fe41f4ff4ce596cf394bd69a0ba48e88db8d2328))
|
||||
|
||||
## [1.169.0](https://github.com/windmill-labs/windmill/compare/v1.168.3...v1.169.0) (2023-09-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* benchmarks graph ([#2244](https://github.com/windmill-labs/windmill/issues/2244)) ([c496602](https://github.com/windmill-labs/windmill/commit/c496602e9e2e0dfecaaffe731e58e551d039d02f))
|
||||
* big query schema explorer ([#2247](https://github.com/windmill-labs/windmill/issues/2247)) ([ec7d923](https://github.com/windmill-labs/windmill/commit/ec7d923cca0f6050855473ababd1bb27d668711b))
|
||||
* flow copilot ([#2219](https://github.com/windmill-labs/windmill/issues/2219)) ([2f3138c](https://github.com/windmill-labs/windmill/commit/2f3138c65d9d3f0161bf3e069c6eec0c32ac3b86))
|
||||
* **frontend:** fix runs page when the row has a parent ([#2255](https://github.com/windmill-labs/windmill/issues/2255)) ([2271263](https://github.com/windmill-labs/windmill/commit/22712632f683fb63ad6d4b475a01c63800a9559d))
|
||||
* introduce container groups ([49c5553](https://github.com/windmill-labs/windmill/commit/49c5553f3b496c2aaf03376689ee0fd42ecbd2bf))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* benchmark svg ([#2249](https://github.com/windmill-labs/windmill/issues/2249)) ([24c5802](https://github.com/windmill-labs/windmill/commit/24c580211572d6447ca502db141e90c5e084d790))
|
||||
* pass TZ from env to runtimes ([75a1490](https://github.com/windmill-labs/windmill/commit/75a149009a5a13230b4d6de6eac8bba0618629d6))
|
||||
|
||||
## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)
|
||||
|
||||
|
||||
|
||||
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
|
||||
|
||||
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
|
||||
|
||||
COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
|
||||
COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# add the docker client to call docker from a worker if enabled
|
||||
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
|
||||
|
||||
5
LICENSE
5
LICENSE
@@ -6,9 +6,8 @@ Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise speci
|
||||
Every file is under License AGPL unless otherwise specified
|
||||
or belonging to one of the below cases:
|
||||
|
||||
The files under backend/ are AGPLv3 Licensed, except any snippets of code under the compile flag "enterprise". Those snippets and files are under a proprietary and commerccial license.
|
||||
The files under frontend/ are AGPLv3 Licensed, except any snippers of code that require a positive license check to be activated. Those snippets and files are under a proprietary and commercial license.
|
||||
Private and public forks MUST not include any of the above proprietary and commercial code. Windmill Labs, Inc. provide tools to clean the codebase from those snippets upon demand.
|
||||
The files under backend/ are AGPLv3 Licensed.
|
||||
The files under frontend/ are AGPLv3 Licensed.
|
||||
The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
|
||||
|
||||
The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6"
|
||||
}
|
||||
@@ -1,16 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"TextArray",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f"
|
||||
"hash": "07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT script_path FROM completed_job WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "script_path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM worker_group_config WHERE name = $1 RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "210b7fa50246d9b8fd1a24ade787bad6882c86e9422b715844aa92b67ed05174"
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Int8"
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -37,11 +37,6 @@
|
||||
"ordinal": 6,
|
||||
"name": "custom_tags",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "worker_group",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -57,9 +52,8 @@
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0"
|
||||
"hash": "4f6b3b472b4b78c0325cf3755f9ef1806d2e82328ceccbeade8cc2333c6dfe47"
|
||||
}
|
||||
@@ -11,7 +11,7 @@
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8"
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
|
||||
17
backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json
generated
Normal file
17
backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json
generated
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags) VALUES ($1, $2, $3, $4) ON CONFLICT (worker) DO NOTHING",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT config FROM worker_group_config WHERE name = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "config",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "6c0136f7965f1e01620a7d5efd7edbc78f0bf3f815676d8b343152bfce2dfc4c"
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT * FROM worker_group_config",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "config",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "7998b23eb72f5967a0fd376fa1015bf29ba5150cd59732288ceb72ce9cecf987"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COUNT(*) FROM password WHERE login_type != 'password'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483"
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_group_config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Jsonb"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "903f2f62f3829274f5dfa0cb1ebbb9e132d310d19ca744531a42ca2c7ac56f56"
|
||||
}
|
||||
22
backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json
generated
Normal file
22
backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json
generated
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM\n (SELECT duration_ms FROM completed_job WHERE script_path = $1\n ORDER BY started_at\n DESC LIMIT 10) AS t",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "avg_duration_s",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1"
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "exists",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM\n (SELECT duration_ms FROM completed_job WHERE script_path = $1\n ORDER BY started_at\n DESC LIMIT 10) AS t",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "avg_duration_s",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468"
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "duration",
|
||||
"type_info": "Numeric"
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "database_length!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca"
|
||||
}
|
||||
324
backend/Cargo.lock
generated
324
backend/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.173.0"
|
||||
version = "1.168.3"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -22,7 +22,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.173.0"
|
||||
version = "1.168.3"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -96,7 +96,7 @@ tower = "^0"
|
||||
tower-http = { version = "^0", features = ["trace", "cors"] }
|
||||
tower-cookies = "^0"
|
||||
serde = "^1"
|
||||
serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
|
||||
serde_json = { version = "^1", features = ["preserve_order"] }
|
||||
uuid = { version = "^1", features = ["serde", "v4"] }
|
||||
thiserror = "^1"
|
||||
anyhow = "^1"
|
||||
@@ -155,9 +155,7 @@ sqlx = { version = "^0", features = [
|
||||
"chrono",
|
||||
"postgres",
|
||||
"runtime-tokio-rustls",
|
||||
"bigdecimal"
|
||||
] }
|
||||
bigdecimal = "^0"
|
||||
dotenv = "^0"
|
||||
ulid = { version = "^1", features = ["uuid"] }
|
||||
futures = "^0"
|
||||
@@ -189,7 +187,6 @@ postgres-native-tls = "^0"
|
||||
native-tls = "^0"
|
||||
samael = { version = "0.0.12", features = ["xmlsec"] }
|
||||
gcp_auth = "0.9.0"
|
||||
rust_decimal = { version = "1.31.0", features = ["db-postgres"]}
|
||||
rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
|
||||
jsonwebtoken = "8.3.0"
|
||||
pem = "3.0.1"
|
||||
nix = { version = "0.27.1", features = ["process", "signal"] }
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add down migration script here
|
||||
DROP TABLE worker_group_config;
|
||||
|
||||
ALTER TABLE worker_ping DROP COLUMN worker_group;
|
||||
@@ -1,8 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE TABLE worker_group_config (
|
||||
name VARCHAR(255) PRIMARY KEY,
|
||||
config JSONB DEFAULT '{}'::jsonb
|
||||
);
|
||||
|
||||
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS worker_group VARCHAR(255) NOT NULL DEFAULT 'default';
|
||||
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS dedicated_worker VARCHAR(255);
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1 +0,0 @@
|
||||
-- Add up migration script here
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,5 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE completed_job ALTER COLUMN duration_ms TYPE bigint;
|
||||
ALTER TABLE completed_job ALTER COLUMN email TYPE VARCHAR(255);
|
||||
ALTER TABLE queue ALTER COLUMN email TYPE VARCHAR(255);
|
||||
ALTER TABLE queue ALTER COLUMN canceled_by TYPE VARCHAR(255);
|
||||
@@ -35,7 +35,6 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
|
||||
"dateutil" => "python-dateutil",
|
||||
"mailparser" => "mail-parser",
|
||||
"mailparser-reply" => "mail-parser-reply",
|
||||
"gitlab" => "python-gitlab"
|
||||
};
|
||||
|
||||
fn replace_import(x: String) -> String {
|
||||
|
||||
@@ -8,14 +8,15 @@
|
||||
|
||||
use gethostname::gethostname;
|
||||
use git_version::git_version;
|
||||
use monitor::handle_zombie_jobs_periodically;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
fs::{metadata, DirBuilder},
|
||||
join,
|
||||
sync::RwLock,
|
||||
};
|
||||
use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
|
||||
@@ -27,8 +28,6 @@ use windmill_worker::{
|
||||
PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
|
||||
};
|
||||
|
||||
use crate::monitor::monitor_db;
|
||||
|
||||
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
|
||||
const DEFAULT_NUM_WORKERS: usize = 1;
|
||||
const DEFAULT_PORT: u16 = 8000;
|
||||
@@ -136,10 +135,7 @@ Windmill Community Edition {GIT_VERSION}
|
||||
tracing::info!("Smtp client connected.");
|
||||
}
|
||||
}
|
||||
|
||||
let worker_mode = num_workers > 0;
|
||||
|
||||
if server_mode || worker_mode {
|
||||
if server_mode || num_workers > 0 {
|
||||
let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
|
||||
|
||||
let port = if server_mode {
|
||||
@@ -148,19 +144,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
port_var.unwrap_or(0)
|
||||
};
|
||||
|
||||
// since it's only on server mode, the port is statically defined
|
||||
let base_internal_url: String = format!("http://localhost:{}", port.to_string());
|
||||
|
||||
monitor_db(
|
||||
&db,
|
||||
tx.clone(),
|
||||
&base_internal_url,
|
||||
rsmq.clone(),
|
||||
worker_mode,
|
||||
server_mode,
|
||||
)
|
||||
.await;
|
||||
|
||||
if std::env::var("BASE_INTERNAL_URL").is_ok() {
|
||||
tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
|
||||
}
|
||||
@@ -178,11 +161,10 @@ Windmill Community Edition {GIT_VERSION}
|
||||
let workers_f = async {
|
||||
let port = port_rx.await?;
|
||||
let base_internal_url: String = format!("http://localhost:{}", port.to_string());
|
||||
if worker_mode {
|
||||
if num_workers > 0 {
|
||||
run_workers(
|
||||
db.clone(),
|
||||
rx.resubscribe(),
|
||||
tx.clone(),
|
||||
num_workers,
|
||||
base_internal_url.clone(),
|
||||
rsmq.clone(),
|
||||
@@ -194,48 +176,29 @@ Windmill Community Edition {GIT_VERSION}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let rsmq2 = rsmq.clone();
|
||||
let monitor_f = async {
|
||||
let db = db.clone();
|
||||
let tx = tx.clone();
|
||||
let rsmq = rsmq.clone();
|
||||
|
||||
let mut rx = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
tokio::spawn(async move {
|
||||
//monitor_db is applied at start, no need to apply it twice
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
loop {
|
||||
monitor_db(
|
||||
&db,
|
||||
tx.clone(),
|
||||
&base_internal_url,
|
||||
rsmq.clone(),
|
||||
worker_mode,
|
||||
server_mode,
|
||||
)
|
||||
.await;
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(30)) => (),
|
||||
_ = rx.recv() => {
|
||||
println!("received killpill for monitor job");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if server_mode {
|
||||
// since it's only on server mode, the port is statically defined
|
||||
let base_internal_url: String = format!("http://localhost:{}", port.to_string());
|
||||
monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2).await;
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let metrics_f = async {
|
||||
if let Some(_addr) = metrics_addr {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
panic!("Metrics are only available in the Enterprise Edition");
|
||||
match metrics_addr {
|
||||
Some(_addr) => {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
panic!("Metrics are only available in the Enterprise Edition");
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0).await;
|
||||
#[cfg(feature = "enterprise")]
|
||||
windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
None => Ok(()),
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
|
||||
@@ -262,10 +225,28 @@ fn display_config(envs: &[&str]) {
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
db: &Pool<Postgres>,
|
||||
rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let db1 = db.clone();
|
||||
let db2 = db.clone();
|
||||
|
||||
let rx2 = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
tokio::spawn(async move {
|
||||
join!(
|
||||
handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
|
||||
windmill_api::delete_expired_items_perdiodically(&db2, rx2)
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
db: Pool<Postgres>,
|
||||
rx: tokio::sync::broadcast::Receiver<()>,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
num_workers: i32,
|
||||
base_internal_url: String,
|
||||
rsmq: Option<R>,
|
||||
@@ -339,7 +320,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
let worker_name = format!("wk-{}-{}", &instance_name, rd_string(5));
|
||||
let ip = ip.clone();
|
||||
let rx = rx.resubscribe();
|
||||
let tx = tx.clone();
|
||||
let base_internal_url = base_internal_url.clone();
|
||||
let rsmq2 = rsmq.clone();
|
||||
let sync_barrier = sync_barrier.clone();
|
||||
@@ -353,7 +333,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
num_workers as u32,
|
||||
&ip,
|
||||
rx,
|
||||
tx,
|
||||
&base_internal_url,
|
||||
rsmq2,
|
||||
sync_barrier,
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{join, sync::mpsc};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error,
|
||||
jobs::{JobKind, QueuedJob},
|
||||
worker::{load_worker_config, reload_custom_tags_setting, WORKER_CONFIG},
|
||||
METRICS_ENABLED,
|
||||
};
|
||||
use windmill_worker::{
|
||||
@@ -36,61 +37,23 @@ lazy_static::lazy_static! {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
pub async fn handle_zombie_jobs_periodically<
|
||||
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
worker_mode: bool,
|
||||
server_mode: bool,
|
||||
) {
|
||||
let zombie_jobs_f = async {
|
||||
if server_mode {
|
||||
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
|
||||
}
|
||||
};
|
||||
let expired_items_f = async {
|
||||
if server_mode {
|
||||
windmill_api::delete_expired_items(&db).await;
|
||||
}
|
||||
};
|
||||
let reload_worker_config_f = async {
|
||||
if worker_mode {
|
||||
reload_worker_config(&db, tx).await;
|
||||
}
|
||||
};
|
||||
let reload_custom_tags_f = async {
|
||||
if server_mode {
|
||||
if let Err(e) = reload_custom_tags_setting(db).await {
|
||||
tracing::error!("Error reloading custom tags: {:?}", e)
|
||||
}
|
||||
}
|
||||
};
|
||||
join!(
|
||||
expired_items_f,
|
||||
zombie_jobs_f,
|
||||
reload_worker_config_f,
|
||||
reload_custom_tags_f
|
||||
);
|
||||
}
|
||||
loop {
|
||||
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
|
||||
|
||||
pub async fn reload_worker_config(db: &Pool<Postgres>, tx: tokio::sync::broadcast::Sender<()>) {
|
||||
let config = load_worker_config(&db).await;
|
||||
if let Err(e) = config {
|
||||
tracing::error!("Error reloading worker config: {:?}", e)
|
||||
} else {
|
||||
let wc = WORKER_CONFIG.read().await;
|
||||
let config = config.unwrap();
|
||||
if *wc != config {
|
||||
if (*wc).dedicated_worker != config.dedicated_worker {
|
||||
tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
|
||||
let _ = tx.send(());
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(30)) => (),
|
||||
_ = rx.recv() => {
|
||||
println!("received killpill for monitor job");
|
||||
break;
|
||||
}
|
||||
drop(wc);
|
||||
|
||||
let mut wc = WORKER_CONFIG.write().await;
|
||||
tracing::info!("Reloading worker config...");
|
||||
*wc = config
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,64 +5,22 @@ use futures::StreamExt;
|
||||
use futures::{stream, Stream};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, query};
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
|
||||
use tokio::{
|
||||
sync::RwLock,
|
||||
time::{timeout, Duration},
|
||||
};
|
||||
use windmill_api::jobs::{CompletedJob, Job};
|
||||
use windmill_api_client::types::{
|
||||
CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
|
||||
};
|
||||
use windmill_common::worker::{WORKER_CONFIG, load_worker_config};
|
||||
use windmill_common::{
|
||||
flow_status::{FlowStatus, FlowStatusModule},
|
||||
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
|
||||
jobs::{JobPayload, RawCode, JobKind},
|
||||
scripts::{ScriptLang, ScriptHash}
|
||||
jobs::{JobPayload, RawCode},
|
||||
scripts::ScriptLang,
|
||||
};
|
||||
use windmill_queue::PushIsolationLevel;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize)]
|
||||
pub struct CompletedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub started_at: chrono::DateTime<chrono::Utc>,
|
||||
pub duration_ms: i32,
|
||||
pub success: bool,
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
pub result: Option<serde_json::Value>,
|
||||
pub logs: Option<String>,
|
||||
pub deleted: bool,
|
||||
pub raw_code: Option<String>,
|
||||
pub canceled: bool,
|
||||
pub canceled_by: Option<String>,
|
||||
pub canceled_reason: Option<String>,
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
pub is_skipped: bool,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
pub mem_peak: Option<i32>,
|
||||
pub tag: String,
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
pub language: Option<ScriptLang>,
|
||||
pub job_kind: JobKind,
|
||||
|
||||
}
|
||||
|
||||
impl CompletedJob {
|
||||
pub fn json_result(&self) -> Option<serde_json::Value> {
|
||||
self.result.clone()
|
||||
}
|
||||
}
|
||||
use windmill_queue::{get_queued_job, PushIsolationLevel};
|
||||
|
||||
async fn initialize_tracing() {
|
||||
use std::sync::Once;
|
||||
@@ -96,6 +54,37 @@ fn next_worker_name() -> String {
|
||||
format!("{id}/{thread_name}")
|
||||
}
|
||||
|
||||
pub async fn get_job_by_id<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
id: Uuid,
|
||||
) -> windmill_common::error::Result<(Option<Job>, Transaction<'c, Postgres>)> {
|
||||
let cjob_option = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let job_option = match cjob_option {
|
||||
Some(job) => Some(Job::CompletedJob(job)),
|
||||
None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob),
|
||||
};
|
||||
if job_option.is_some() {
|
||||
Ok((job_option, tx))
|
||||
} else {
|
||||
// check if a job had been moved in-between queries
|
||||
let cjob_option = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
Ok((cjob_option.map(Job::CompletedJob), tx))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ApiServer {
|
||||
pub addr: std::net::SocketAddr,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
@@ -132,15 +121,15 @@ impl ApiServer {
|
||||
}
|
||||
}
|
||||
|
||||
// async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
|
||||
// tracing::info!(
|
||||
// "{:#?}",
|
||||
// get_job_by_id(db.begin().await?, "test-workspace", id)
|
||||
// .await?
|
||||
// .0
|
||||
// );
|
||||
// Ok(())
|
||||
// }
|
||||
async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
|
||||
tracing::info!(
|
||||
"{:#?}",
|
||||
get_job_by_id(db.begin().await?, "test-workspace", id)
|
||||
.await?
|
||||
.0
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
|
||||
cjob.flow_status.clone().and_then(|fs| {
|
||||
@@ -310,7 +299,7 @@ mod suspend_resume {
|
||||
|
||||
server.close().await.unwrap();
|
||||
|
||||
let result = completed_job(flow, &db).await.json_result().unwrap();
|
||||
let result = completed_job(flow, &db).await.result.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
json!({
|
||||
@@ -348,7 +337,7 @@ mod suspend_resume {
|
||||
.arg("port", json!(port))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
server.close().await.unwrap();
|
||||
@@ -411,7 +400,7 @@ mod suspend_resume {
|
||||
|
||||
server.close().await.unwrap();
|
||||
|
||||
let result = completed_job(flow, &db).await.json_result().unwrap();
|
||||
let result = completed_job(flow, &db).await.result.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
json!( {"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by unknown", "canceler": "unknown"}}),
|
||||
@@ -566,7 +555,7 @@ def main(last, port):
|
||||
.arg("port", json!(server.addr.port()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(server.close().await, attempts);
|
||||
@@ -595,7 +584,7 @@ def main(last, port):
|
||||
.arg("port", json!(server.addr.port()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(server.close().await, attempts);
|
||||
@@ -637,7 +626,7 @@ def main(last, port):
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await;
|
||||
|
||||
let result = job.json_result().unwrap();
|
||||
let result = job.result.unwrap();
|
||||
assert_eq!(server.close().await, attempts);
|
||||
assert!(result["error"]
|
||||
.as_object()
|
||||
@@ -701,7 +690,7 @@ def main(error, port):
|
||||
.arg("port", json!(server.addr.port()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await;
|
||||
let result = cjob.json_result().clone().unwrap();
|
||||
let result = cjob.result.clone().unwrap();
|
||||
let failed_module = get_module(&cjob, "a").unwrap();
|
||||
match failed_module {
|
||||
FlowStatusModule::Failure { .. } => {}
|
||||
@@ -759,7 +748,7 @@ async fn test_iteration(db: Pool<Postgres>) {
|
||||
.arg("items", json!([]))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!([]));
|
||||
|
||||
@@ -768,7 +757,7 @@ async fn test_iteration(db: Pool<Postgres>) {
|
||||
.arg("items", json!((0..257).collect::<Vec<_>>()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert!(matches!(result, serde_json::Value::Array(_)));
|
||||
assert!(result[2]["error"]
|
||||
@@ -816,7 +805,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
|
||||
.arg("items", json!([]))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!([]));
|
||||
|
||||
@@ -826,7 +815,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await;
|
||||
// println!("{:#?}", job);
|
||||
let result = job.json_result().unwrap();
|
||||
let result = job.result.unwrap();
|
||||
assert!(matches!(result, serde_json::Value::Array(_)));
|
||||
assert!(result[2]["error"]
|
||||
.as_object()
|
||||
@@ -957,12 +946,8 @@ fn spawn_test_worker(
|
||||
let worker_name: String = next_worker_name();
|
||||
let ip: &str = Default::default();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let future = async move {
|
||||
let base_internal_url = format!("http://localhost:{}", port);
|
||||
let mut wc = WORKER_CONFIG.write().await;
|
||||
*wc = load_worker_config(&db).await.unwrap();
|
||||
drop(wc);
|
||||
windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
|
||||
&db,
|
||||
worker_instance,
|
||||
@@ -971,7 +956,6 @@ fn spawn_test_worker(
|
||||
1,
|
||||
ip,
|
||||
rx,
|
||||
tx2,
|
||||
&base_internal_url,
|
||||
None,
|
||||
Arc::new(RwLock::new(None)),
|
||||
@@ -1011,8 +995,8 @@ async fn listen_for_uuid_on(
|
||||
}
|
||||
|
||||
async fn completed_job(uuid: Uuid, db: &Pool<Postgres>) -> CompletedJob {
|
||||
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1").bind(uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -1122,7 +1106,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
println!("deno flow iteration: {}", i);
|
||||
let job = run_job_in_new_worker_until_complete(&db, job.clone(), port).await;
|
||||
// println!("job: {:#?}", job.flow_status);
|
||||
let result = job.json_result().unwrap();
|
||||
let result = job.result.unwrap();
|
||||
assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i);
|
||||
}
|
||||
}
|
||||
@@ -1158,7 +1142,7 @@ async fn test_identity(db: Pool<Postgres>) {
|
||||
let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!(42));
|
||||
}
|
||||
@@ -1347,7 +1331,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
@@ -1402,7 +1386,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
|
||||
let job = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, job.clone(), port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!([[42]]));
|
||||
}
|
||||
@@ -1447,7 +1431,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
|
||||
.arg("n", json!(123))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(json!("last step saw 123"), result);
|
||||
|
||||
@@ -1456,7 +1440,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
|
||||
let result = cjob.json_result().unwrap();
|
||||
let result = cjob.result.unwrap();
|
||||
assert_eq!(json!(-123), result);
|
||||
}
|
||||
|
||||
@@ -1505,7 +1489,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
|
||||
.arg("n", json!(123))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(json!("last step saw [123]"), result);
|
||||
|
||||
@@ -1514,7 +1498,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
|
||||
let result = cjob.json_result().unwrap();
|
||||
let result = cjob.result.unwrap();
|
||||
assert_eq!(json!([-123]), result);
|
||||
}
|
||||
|
||||
@@ -1568,7 +1552,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
|
||||
port,
|
||||
)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}");
|
||||
@@ -1603,7 +1587,7 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
|
||||
port,
|
||||
)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}");
|
||||
@@ -1640,7 +1624,7 @@ func main(derp string) (string, error) {
|
||||
.arg("derp", json!("world"))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("hello world"));
|
||||
@@ -1671,7 +1655,7 @@ echo "hello $msg"
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
|
||||
assert_eq!(job.json_result(), Some(json!("hello world")));
|
||||
assert_eq!(job.result, Some(json!("hello world")));
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
@@ -1698,7 +1682,7 @@ def main():
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("hello world"));
|
||||
@@ -1731,7 +1715,7 @@ def main():
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(3));
|
||||
@@ -1763,7 +1747,7 @@ def main():
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("test-workspace"));
|
||||
@@ -1819,7 +1803,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(0));
|
||||
@@ -1859,7 +1843,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
|
||||
let job = run_job_in_new_worker_until_complete(&db, flow, port).await;
|
||||
|
||||
assert_eq!(
|
||||
job.json_result().unwrap(),
|
||||
job.result.unwrap(),
|
||||
serde_json::json!( {"error": {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
|
||||
);
|
||||
}
|
||||
@@ -1900,7 +1884,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([]));
|
||||
@@ -1955,7 +1939,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(9));
|
||||
@@ -2023,7 +2007,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([1, 2]));
|
||||
@@ -2059,7 +2043,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([1, 3]));
|
||||
@@ -2097,7 +2081,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
|
||||
@@ -2134,7 +2118,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
|
||||
@@ -2181,7 +2165,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2218,7 +2202,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2282,7 +2266,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([1, 2, 3]));
|
||||
@@ -2339,7 +2323,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
println!("{:#?}", result);
|
||||
@@ -2404,7 +2388,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(0))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert!(result["from failure module"]["error"]
|
||||
@@ -2420,7 +2404,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(1))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert!(result["from failure module"]["error"]
|
||||
@@ -2436,7 +2420,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(2))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert!(result["from failure module"]["error"]
|
||||
@@ -2452,7 +2436,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(3))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(json!({ "l": [0, 1, 2] }), result);
|
||||
}
|
||||
@@ -2602,7 +2586,6 @@ async fn test_rust_client(db: Pool<Postgres>) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
initialize_tracing().await;
|
||||
@@ -2659,7 +2642,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2718,7 +2702,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2793,7 +2778,8 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2853,7 +2839,8 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.173.0
|
||||
version: 1.168.3
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -2386,46 +2386,6 @@ paths:
|
||||
- content
|
||||
- language
|
||||
|
||||
/scripts/hub/query:
|
||||
get:
|
||||
summary: query hub scripts by similarity
|
||||
operationId: queryHubScripts
|
||||
tags:
|
||||
- script
|
||||
parameters:
|
||||
- name: text
|
||||
description: query text
|
||||
in: query
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: kind
|
||||
description: query scripts kind
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
description: query limit
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
"200":
|
||||
description: script details
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
|
||||
/w/{workspace}/scripts/list:
|
||||
get:
|
||||
summary: list all available scripts
|
||||
@@ -3954,27 +3914,6 @@ paths:
|
||||
required:
|
||||
- database_length
|
||||
|
||||
/w/{workspace}/jobs/completed/count:
|
||||
get:
|
||||
summary: get completed count
|
||||
operationId: getCompletedCount
|
||||
tags:
|
||||
- job
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
responses:
|
||||
"200":
|
||||
description: completed count
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
database_length:
|
||||
type: integer
|
||||
required:
|
||||
- database_length
|
||||
|
||||
/w/{workspace}/jobs/queue/cancel_all:
|
||||
post:
|
||||
summary: cancel all jobs
|
||||
@@ -5252,65 +5191,6 @@ paths:
|
||||
items:
|
||||
$ref: "#/components/schemas/WorkerPing"
|
||||
|
||||
/workers/list_worker_groups:
|
||||
get:
|
||||
summary: list workers
|
||||
operationId: listWorkerGroups
|
||||
tags:
|
||||
- worker
|
||||
responses:
|
||||
"200":
|
||||
description: a list of workers
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
config: {}
|
||||
required:
|
||||
- name
|
||||
- config
|
||||
|
||||
/workers/worker_group/{name}:
|
||||
post:
|
||||
summary: Update Worker Group
|
||||
operationId: updateWorkerGroup
|
||||
tags:
|
||||
- worker
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/Name"
|
||||
requestBody:
|
||||
description: worker group
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
responses:
|
||||
"200":
|
||||
description: Update a worker group
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
delete:
|
||||
summary: Delete Worker Group
|
||||
operationId: deleteWorkerGroup
|
||||
tags:
|
||||
- worker
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/Name"
|
||||
responses:
|
||||
"200":
|
||||
description: Delete a worker group
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/acls/get/{kind}/{path}:
|
||||
get:
|
||||
summary: get granular acls
|
||||
@@ -7231,8 +7111,6 @@ components:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
worker_group:
|
||||
type: string
|
||||
required:
|
||||
- worker
|
||||
- worker_instance
|
||||
@@ -7240,7 +7118,6 @@ components:
|
||||
- started_at
|
||||
- ip
|
||||
- jobs_executed
|
||||
- worker_group
|
||||
|
||||
UserWorkspaceList:
|
||||
type: object
|
||||
|
||||
@@ -508,7 +508,6 @@ pub async fn get_hub_app_by_id(
|
||||
&format!("https://hub.windmill.dev/apps/{id}/json"),
|
||||
&email,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
|
||||
@@ -160,7 +160,6 @@ pub async fn get_hub_flow_by_id(
|
||||
&format!("https://hub.windmill.dev/flows/{id}/json"),
|
||||
&email,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
|
||||
@@ -15,7 +15,6 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
use windmill_common::{db::UserDB, users::username_to_permissioned_as};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
@@ -24,6 +23,7 @@ use windmill_common::{
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{query_scalar, FromRow, Postgres, Transaction};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -212,7 +212,6 @@ async fn _check_nb_of_groups(db: &DB) -> Result<()> {
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
async fn create_group(
|
||||
authed: ApiAuthed,
|
||||
Extension(_db): Extension<DB>,
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
use crate::{db::ApiAuthed, jobs::CompletedJob};
|
||||
use axum::{
|
||||
extract::{Path, Query},
|
||||
routing::{get, post},
|
||||
@@ -15,7 +15,7 @@ use axum::{
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sqlx::{types::Uuid, FromRow};
|
||||
use sqlx::types::Uuid;
|
||||
use std::{
|
||||
fmt::{Display, Formatter},
|
||||
vec,
|
||||
@@ -102,15 +102,6 @@ pub struct Input {
|
||||
success: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, FromRow)]
|
||||
pub struct CompletedJobMini {
|
||||
id: Uuid,
|
||||
created_at: chrono::DateTime<chrono::Utc>,
|
||||
args: Option<serde_json::Value>,
|
||||
created_by: String,
|
||||
success: bool,
|
||||
}
|
||||
|
||||
async fn get_input_history(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -123,13 +114,13 @@ async fn get_input_history(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let sql = &format!(
|
||||
"select id, created_at, created_by, args, success from completed_job \
|
||||
"select * from completed_job \
|
||||
where {} = $1 and job_kind = $2 and workspace_id = $3 \
|
||||
order by created_at desc limit $4 offset $5",
|
||||
r.runnable_type.column_name()
|
||||
);
|
||||
|
||||
let query = sqlx::query_as::<_, CompletedJobMini>(sql);
|
||||
let query = sqlx::query_as::<_, CompletedJob>(sql);
|
||||
|
||||
let query = match r.runnable_type {
|
||||
RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::{
|
||||
users::{check_scopes, require_owner_of_path, OptAuthed},
|
||||
utils::require_super_admin,
|
||||
variables::get_workspace_key,
|
||||
workers::{CUSTOM_TAGS, CUSTOM_TAGS_PER_WORKSPACE},
|
||||
BASE_URL,
|
||||
};
|
||||
use anyhow::Context;
|
||||
@@ -28,12 +29,10 @@ use hmac::Mac;
|
||||
use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use sql_builder::{prelude::*, quote, SqlBuilder};
|
||||
use sqlx::types::JsonRawValue;
|
||||
use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction};
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use urlencoding::encode;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::worker::CUSTOM_TAGS_PER_WORKSPACE;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{self, to_anyhow, Error},
|
||||
@@ -45,7 +44,7 @@ use windmill_common::{
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::{job_is_complete, push, PushIsolationLevel};
|
||||
use windmill_queue::{get_queued_job, push, PushIsolationLevel};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
let cors = CorsLayer::new()
|
||||
@@ -111,7 +110,6 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/queue/list", get(list_queue_jobs))
|
||||
.route("/queue/count", get(count_queue_jobs))
|
||||
.route("/queue/cancel_all", post(cancel_all))
|
||||
.route("/completed/count", get(count_completed_jobs))
|
||||
.route(
|
||||
"/completed/list",
|
||||
get(list_completed_jobs).layer(cors.clone()),
|
||||
@@ -232,15 +230,15 @@ async fn cancel_job_api(
|
||||
tx.commit().await?;
|
||||
Ok(id.to_string())
|
||||
} else {
|
||||
let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?;
|
||||
tx.commit().await?;
|
||||
if job_is_complete(&db, id, &w_id).await.unwrap_or(false) {
|
||||
return Ok(format!("queued job id {} is already completed", id));
|
||||
} else {
|
||||
return Err(error::Error::NotFound(format!(
|
||||
"queued job id {} does not exist",
|
||||
id
|
||||
)));
|
||||
}
|
||||
let err = match job_o {
|
||||
Some(Job::CompletedJob(_)) => {
|
||||
return Ok(format!("queued job id {} is already completed", id))
|
||||
}
|
||||
_ => error::Error::NotFound(format!("queued job id {} does not exist", id)),
|
||||
};
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,15 +273,15 @@ async fn force_cancel(
|
||||
tx.commit().await?;
|
||||
Ok(id.to_string())
|
||||
} else {
|
||||
let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?;
|
||||
tx.commit().await?;
|
||||
if job_is_complete(&db, id, &w_id).await.unwrap_or(false) {
|
||||
return Ok(format!("queued job id {} is already completed", id));
|
||||
} else {
|
||||
return Err(error::Error::NotFound(format!(
|
||||
"queued job id {} does not exist",
|
||||
id
|
||||
)));
|
||||
}
|
||||
let err = match job_o {
|
||||
Some(Job::CompletedJob(_)) => {
|
||||
return Ok(format!("queued job id {} is already completed", id))
|
||||
}
|
||||
_ => error::Error::NotFound(format!("queued job id {} does not exist", id)),
|
||||
};
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,29 +344,12 @@ pub async fn get_path_tag_limits_cache_for_hash(
|
||||
async fn get_job(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::Result<Response> {
|
||||
let cjob_option =
|
||||
sqlx::query("SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id)
|
||||
.bind(&w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
if let Some(job) = cjob_option {
|
||||
let job = Job::CompletedJob(CompletedJob::from_row(&job)?);
|
||||
Ok(Json(job).into_response())
|
||||
} else {
|
||||
let job_o = sqlx::query_as::<_, QueuedJob>(
|
||||
"SELECT *
|
||||
FROM queue WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(&w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.map(Job::QueuedJob);
|
||||
let job: Job<'_> = not_found_if_none(job_o, "Job", id.to_string())?;
|
||||
Ok(Json(job).into_response())
|
||||
}
|
||||
) -> error::JsonResult<Job> {
|
||||
let tx = db.begin().await?;
|
||||
let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?;
|
||||
let job = not_found_if_none(job_o, "Job", id.to_string())?;
|
||||
tx.commit().await?;
|
||||
Ok(Json(job))
|
||||
}
|
||||
|
||||
async fn get_job_logs(
|
||||
@@ -387,8 +368,39 @@ async fn get_job_logs(
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
pub async fn get_job_by_id<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
id: Uuid,
|
||||
) -> error::Result<(Option<Job>, Transaction<'c, Postgres>)> {
|
||||
let cjob_option = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let job_option = match cjob_option {
|
||||
Some(job) => Some(Job::CompletedJob(job)),
|
||||
None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob),
|
||||
};
|
||||
if job_option.is_some() {
|
||||
Ok((job_option, tx))
|
||||
} else {
|
||||
// check if a job had been moved in-between queries
|
||||
let cjob_option = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
Ok((cjob_option.map(Job::CompletedJob), tx))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize)]
|
||||
pub struct CompletedJob<'rows> {
|
||||
pub struct CompletedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -396,15 +408,15 @@ pub struct CompletedJob<'rows> {
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub started_at: chrono::DateTime<chrono::Utc>,
|
||||
pub duration_ms: i64,
|
||||
pub duration_ms: i32,
|
||||
pub success: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none", borrow)]
|
||||
pub result: Option<&'rows JsonRawValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub result: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logs: Option<String>,
|
||||
pub deleted: bool,
|
||||
@@ -434,64 +446,6 @@ pub struct CompletedJob<'rows> {
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
impl<'row> CompletedJob<'row> {
|
||||
pub fn json_result(&self) -> Option<serde_json::Value> {
|
||||
self.result
|
||||
.as_ref()
|
||||
.map(|r| serde_json::from_str(r.get()).ok())
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize)]
|
||||
pub struct ListableCompletedJob {
|
||||
pub r#type: String,
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub started_at: chrono::DateTime<chrono::Utc>,
|
||||
pub duration_ms: i64,
|
||||
pub success: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub deleted: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_code: Option<String>,
|
||||
pub canceled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_by: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_reason: Option<String>,
|
||||
pub job_kind: JobKind,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<ScriptLang>,
|
||||
pub is_skipped: bool,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
impl<'a> IntoResponse for CompletedJob<'a> {
|
||||
fn into_response(self) -> Response {
|
||||
Json(self).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
pub struct RunJobQuery {
|
||||
scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
|
||||
@@ -677,7 +631,6 @@ struct ListableQueuedJob {
|
||||
pub language: Option<ScriptLang>,
|
||||
pub email: String,
|
||||
pub suspend: Option<i32>,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
async fn list_queue_jobs(
|
||||
@@ -696,7 +649,7 @@ async fn list_queue_jobs(
|
||||
"scheduled_for",
|
||||
"script_hash",
|
||||
"script_path",
|
||||
"null as args",
|
||||
"args",
|
||||
"job_kind",
|
||||
"schedule_path",
|
||||
"permissioned_as",
|
||||
@@ -705,7 +658,6 @@ async fn list_queue_jobs(
|
||||
"same_worker",
|
||||
"email",
|
||||
"suspend",
|
||||
"tag",
|
||||
],
|
||||
)
|
||||
.sql()?;
|
||||
@@ -752,33 +704,72 @@ async fn count_queue_jobs(
|
||||
))
|
||||
}
|
||||
|
||||
async fn count_completed_jobs(
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
) -> error::JsonResult<QueueStats> {
|
||||
Ok(Json(
|
||||
sqlx::query_as!(
|
||||
QueueStats,
|
||||
"SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_jobs(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(pagination): Query<Pagination>,
|
||||
Query(lq): Query<ListCompletedQuery>,
|
||||
) -> error::JsonResult<Vec<Job<'static>>> {
|
||||
) -> error::JsonResult<Vec<Job>> {
|
||||
check_scopes(&authed, || format!("listjobs"))?;
|
||||
|
||||
// TODO: todo!("rewrite this to just run list_queue_jobs and list_completed_jobs separately and return as one");
|
||||
let (per_page, offset) = paginate(pagination);
|
||||
let lqc = lq.clone();
|
||||
|
||||
let sqlq = list_queue_jobs_query(
|
||||
&w_id,
|
||||
&ListQueueQuery {
|
||||
script_path_start: lq.script_path_start,
|
||||
script_path_exact: lq.script_path_exact,
|
||||
script_hash: lq.script_hash,
|
||||
created_by: lq.created_by,
|
||||
started_before: lq.started_before,
|
||||
started_after: lq.started_after,
|
||||
created_before: lq.created_before,
|
||||
created_after: lq.created_after,
|
||||
created_or_started_before: lq.created_or_started_before,
|
||||
created_or_started_after: lq.created_or_started_after,
|
||||
running: None,
|
||||
parent_job: lq.parent_job,
|
||||
order_desc: Some(true),
|
||||
job_kinds: lq.job_kinds,
|
||||
suspended: lq.suspended,
|
||||
args: lq.args,
|
||||
tag: lq.tag,
|
||||
schedule_path: lq.schedule_path,
|
||||
},
|
||||
&[
|
||||
"'QueuedJob' as typ",
|
||||
"id",
|
||||
"workspace_id",
|
||||
"parent_job",
|
||||
"created_by",
|
||||
"created_at",
|
||||
"started_at",
|
||||
"scheduled_for",
|
||||
"running",
|
||||
"script_hash",
|
||||
"script_path",
|
||||
"null as args",
|
||||
"null as duration_ms",
|
||||
"null as success",
|
||||
"false as deleted",
|
||||
"canceled",
|
||||
"canceled_by",
|
||||
"job_kind",
|
||||
"schedule_path",
|
||||
"permissioned_as",
|
||||
"is_flow_step",
|
||||
"language",
|
||||
"false as is_skipped",
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"suspend",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
"concurrent_limit",
|
||||
"concurrency_time_window_s",
|
||||
],
|
||||
);
|
||||
let sqlc = list_completed_jobs_query(
|
||||
&w_id,
|
||||
per_page + offset,
|
||||
@@ -817,74 +808,13 @@ async fn list_jobs(
|
||||
"null as concurrency_time_window_s",
|
||||
],
|
||||
);
|
||||
|
||||
let sql = if lq.success.is_none() {
|
||||
let sqlq = list_queue_jobs_query(
|
||||
&w_id,
|
||||
&ListQueueQuery {
|
||||
script_path_start: lq.script_path_start,
|
||||
script_path_exact: lq.script_path_exact,
|
||||
script_hash: lq.script_hash,
|
||||
created_by: lq.created_by,
|
||||
started_before: lq.started_before,
|
||||
started_after: lq.started_after,
|
||||
created_before: lq.created_before,
|
||||
created_after: lq.created_after,
|
||||
created_or_started_before: lq.created_or_started_before,
|
||||
created_or_started_after: lq.created_or_started_after,
|
||||
running: None,
|
||||
parent_job: lq.parent_job,
|
||||
order_desc: Some(true),
|
||||
job_kinds: lq.job_kinds,
|
||||
suspended: lq.suspended,
|
||||
args: lq.args,
|
||||
tag: lq.tag,
|
||||
schedule_path: lq.schedule_path,
|
||||
},
|
||||
&[
|
||||
"'QueuedJob' as typ",
|
||||
"id",
|
||||
"workspace_id",
|
||||
"parent_job",
|
||||
"created_by",
|
||||
"created_at",
|
||||
"started_at",
|
||||
"scheduled_for",
|
||||
"running",
|
||||
"script_hash",
|
||||
"script_path",
|
||||
"null as args",
|
||||
"null as duration_ms",
|
||||
"null as success",
|
||||
"false as deleted",
|
||||
"canceled",
|
||||
"canceled_by",
|
||||
"job_kind",
|
||||
"schedule_path",
|
||||
"permissioned_as",
|
||||
"is_flow_step",
|
||||
"language",
|
||||
"false as is_skipped",
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"suspend",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
"concurrent_limit",
|
||||
"concurrency_time_window_s",
|
||||
],
|
||||
);
|
||||
|
||||
format!(
|
||||
"{} UNION ALL {} LIMIT {} OFFSET {};",
|
||||
&sqlq.subquery()?,
|
||||
&sqlc.subquery()?,
|
||||
per_page,
|
||||
offset
|
||||
)
|
||||
} else {
|
||||
sqlc.query()?
|
||||
};
|
||||
let sql = format!(
|
||||
"{} UNION ALL {} ORDER BY created_at DESC LIMIT {} OFFSET {};",
|
||||
&sqlq.subquery()?,
|
||||
&sqlc.subquery()?,
|
||||
per_page,
|
||||
offset
|
||||
);
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let jobs: Vec<UnifiedJob> = sqlx::query_as(&sql).fetch_all(&mut *tx).await?;
|
||||
tx.commit().await?;
|
||||
@@ -1131,8 +1061,8 @@ pub async fn cancel_suspended_job(
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct SuspendedJobFlow<'a> {
|
||||
pub job: Job<'a>,
|
||||
pub struct SuspendedJobFlow {
|
||||
pub job: Job,
|
||||
pub approvers: Vec<Approval>,
|
||||
}
|
||||
|
||||
@@ -1146,7 +1076,7 @@ pub async fn get_suspended_job_flow(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, job, resume_id, secret)): Path<(String, Uuid, u32, String)>,
|
||||
Query(approver): Query<QueryApprover>,
|
||||
) -> error::Result<Response> {
|
||||
) -> error::JsonResult<SuspendedJobFlow> {
|
||||
let mut tx = db.begin().await?;
|
||||
let key = get_workspace_key(&w_id, &mut tx).await?;
|
||||
let mut mac = HmacSha256::new_from_slice(key.as_bytes()).map_err(to_anyhow)?;
|
||||
@@ -1174,29 +1104,7 @@ pub async fn get_suspended_job_flow(
|
||||
.await?
|
||||
.flatten()
|
||||
.ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?;
|
||||
let cjob_option =
|
||||
sqlx::query("SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(flow_id)
|
||||
.bind(&w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
let mut _rows = None;
|
||||
let flow_o = if let Some(job) = cjob_option {
|
||||
_rows = Some(job);
|
||||
Some(Job::CompletedJob(CompletedJob::from_row(
|
||||
_rows.as_ref().unwrap(),
|
||||
)?))
|
||||
} else {
|
||||
sqlx::query_as::<_, QueuedJob>(
|
||||
"SELECT *
|
||||
FROM queue WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(flow_id)
|
||||
.bind(&w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.map(Job::QueuedJob)
|
||||
};
|
||||
let (flow_o, mut tx) = get_job_by_id(tx, &w_id, flow_id).await?;
|
||||
let flow = not_found_if_none(flow_o, "Parent Flow", job.to_string())?;
|
||||
|
||||
let flow_status = flow
|
||||
@@ -1233,7 +1141,7 @@ pub async fn get_suspended_job_flow(
|
||||
approvers_from_status
|
||||
};
|
||||
|
||||
Ok(Json(SuspendedJobFlow { job: flow, approvers }).into_response())
|
||||
Ok(Json(SuspendedJobFlow { job: flow, approvers }))
|
||||
}
|
||||
|
||||
pub async fn create_job_signature(
|
||||
@@ -1313,12 +1221,12 @@ pub async fn get_resume_urls(
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum Job<'a> {
|
||||
pub enum Job {
|
||||
QueuedJob(QueuedJob),
|
||||
CompletedJob(CompletedJob<'a>),
|
||||
CompletedJob(CompletedJob),
|
||||
}
|
||||
|
||||
impl<'a> Job<'a> {
|
||||
impl Job {
|
||||
pub fn raw_flow(&self) -> Option<FlowValue> {
|
||||
let value = match self {
|
||||
Job::QueuedJob(job) => job.raw_flow.clone(),
|
||||
@@ -1348,7 +1256,8 @@ struct UnifiedJob {
|
||||
running: Option<bool>,
|
||||
script_hash: Option<ScriptHash>,
|
||||
script_path: Option<String>,
|
||||
duration_ms: Option<i64>,
|
||||
args: Option<serde_json::Value>,
|
||||
duration_ms: Option<i32>,
|
||||
success: Option<bool>,
|
||||
deleted: bool,
|
||||
canceled: bool,
|
||||
@@ -1368,7 +1277,7 @@ struct UnifiedJob {
|
||||
concurrency_time_window_s: Option<i32>,
|
||||
}
|
||||
|
||||
impl<'a> From<UnifiedJob> for Job<'a> {
|
||||
impl From<UnifiedJob> for Job {
|
||||
fn from(uj: UnifiedJob) -> Self {
|
||||
match uj.typ.as_ref() {
|
||||
"CompletedJob" => Job::CompletedJob(CompletedJob {
|
||||
@@ -1382,7 +1291,7 @@ impl<'a> From<UnifiedJob> for Job<'a> {
|
||||
success: uj.success.unwrap(),
|
||||
script_hash: uj.script_hash,
|
||||
script_path: uj.script_path,
|
||||
args: None,
|
||||
args: uj.args,
|
||||
result: None,
|
||||
logs: None,
|
||||
flow_status: None,
|
||||
@@ -1412,7 +1321,7 @@ impl<'a> From<UnifiedJob> for Job<'a> {
|
||||
started_at: uj.started_at,
|
||||
script_hash: uj.script_hash,
|
||||
script_path: uj.script_path,
|
||||
args: None,
|
||||
args: uj.args,
|
||||
running: uj.running.unwrap(),
|
||||
scheduled_for: uj.scheduled_for.unwrap(),
|
||||
logs: None,
|
||||
@@ -1607,12 +1516,12 @@ fn add_raw_string(
|
||||
return args;
|
||||
}
|
||||
|
||||
async fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) -> error::Result<()> {
|
||||
fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) -> error::Result<()> {
|
||||
if let Some(tag) = tag {
|
||||
if tag == "" {
|
||||
return Ok(());
|
||||
}
|
||||
let custom_tags_per_w = CUSTOM_TAGS_PER_WORKSPACE.read().await;
|
||||
let custom_tags_per_w = &*CUSTOM_TAGS_PER_WORKSPACE;
|
||||
if custom_tags_per_w.0.contains(&tag.to_string()) {
|
||||
Ok(())
|
||||
} else if custom_tags_per_w.1.contains_key(tag)
|
||||
@@ -1626,7 +1535,7 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) ->
|
||||
} else {
|
||||
return Err(error::Error::BadRequest(format!(
|
||||
"Tag {tag} cannot be used on workspace {w_id}: (CUSTOM_TAGS: {:?})",
|
||||
custom_tags_per_w
|
||||
*CUSTOM_TAGS
|
||||
)));
|
||||
}
|
||||
} else {
|
||||
@@ -1655,7 +1564,7 @@ pub async fn run_flow_by_path(
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.flatten();
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
@@ -1705,7 +1614,7 @@ pub async fn run_job_by_path(
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1908,7 +1817,7 @@ pub async fn run_wait_result_job_by_path_get(
|
||||
check_scopes(&authed, || format!("run:script/{script_path}"))?;
|
||||
|
||||
let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2079,7 +1988,7 @@ async fn run_wait_result_script_by_path_internal(
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2143,7 +2052,7 @@ pub async fn run_wait_result_script_by_hash(
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2258,7 +2167,7 @@ async fn run_wait_result_flow_by_path_internal(
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.flatten();
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2314,7 +2223,7 @@ async fn run_preview_job(
|
||||
}
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
|
||||
check_tag_available_for_workspace(&w_id, &preview.tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &preview.tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2422,7 +2331,7 @@ async fn run_preview_flow_job(
|
||||
}
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
|
||||
check_tag_available_for_workspace(&w_id, &raw_flow.tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &raw_flow.tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2479,7 +2388,7 @@ pub async fn run_job_by_hash(
|
||||
let scheduled_for = run_query.get_scheduled_for(&db).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
check_tag_available_for_workspace(&w_id, &tag).await?;
|
||||
check_tag_available_for_workspace(&w_id, &tag)?;
|
||||
let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -2687,7 +2596,7 @@ async fn list_completed_jobs(
|
||||
Path(w_id): Path<String>,
|
||||
Query(pagination): Query<Pagination>,
|
||||
Query(lq): Query<ListCompletedQuery>,
|
||||
) -> error::JsonResult<Vec<ListableCompletedJob>> {
|
||||
) -> error::JsonResult<Vec<CompletedJob>> {
|
||||
check_scopes(&authed, || format!("listjobs"))?;
|
||||
|
||||
let (per_page, offset) = paginate(pagination);
|
||||
@@ -2708,6 +2617,9 @@ async fn list_completed_jobs(
|
||||
"success",
|
||||
"script_hash",
|
||||
"script_path",
|
||||
"args",
|
||||
"result",
|
||||
"null as logs",
|
||||
"deleted",
|
||||
"canceled",
|
||||
"canceled_by",
|
||||
@@ -2725,92 +2637,83 @@ async fn list_completed_jobs(
|
||||
"visible_to_owner",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
"'CompletedJob' as type",
|
||||
],
|
||||
)
|
||||
.sql()?;
|
||||
let jobs = sqlx::query_as::<_, ListableCompletedJob>(&sql)
|
||||
let jobs = sqlx::query_as::<_, CompletedJob>(&sql)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
Ok(Json(jobs))
|
||||
}
|
||||
|
||||
async fn get_completed_job<'a>(
|
||||
async fn get_completed_job(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::Result<Response> {
|
||||
let job_o = sqlx::query("SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
) -> error::JsonResult<CompletedJob> {
|
||||
let job_o = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
let job = not_found_if_none(job_o, "Completed Job", id.to_string())?;
|
||||
Ok(CompletedJob::from_row(&job)?.into_response())
|
||||
}
|
||||
|
||||
#[derive(FromRow)]
|
||||
pub struct RawResult<'a> {
|
||||
pub result: &'a JsonRawValue,
|
||||
}
|
||||
|
||||
impl<'a> IntoResponse for RawResult<'a> {
|
||||
fn into_response(self) -> Response {
|
||||
Json(self.result).into_response()
|
||||
}
|
||||
Ok(Json(job))
|
||||
}
|
||||
|
||||
async fn get_completed_job_result(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::Result<Response> {
|
||||
let result_o =
|
||||
sqlx::query("SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
) -> error::JsonResult<Option<serde_json::Value>> {
|
||||
let result_o = sqlx::query_scalar!(
|
||||
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
id,
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
let result = not_found_if_none(result_o, "Completed Job", id.to_string())?;
|
||||
Ok(RawResult::from_row(&result)?.into_response())
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CompletedJobResult<'c> {
|
||||
struct CompletedJobResult {
|
||||
completed: bool,
|
||||
result: Option<&'c JsonRawValue>,
|
||||
result: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
async fn get_completed_job_result_maybe(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::Result<Response> {
|
||||
let result_o =
|
||||
sqlx::query("SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2")
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
) -> error::JsonResult<CompletedJobResult> {
|
||||
let result_o = sqlx::query_scalar!(
|
||||
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
id,
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
if let Some(result) = result_o {
|
||||
let res = RawResult::from_row(&result)?;
|
||||
Ok(Json(CompletedJobResult { completed: true, result: Some(res.result) }).into_response())
|
||||
Ok(Json(CompletedJobResult { completed: true, result }))
|
||||
} else {
|
||||
Ok(Json(CompletedJobResult { completed: false, result: None }).into_response())
|
||||
Ok(Json(CompletedJobResult { completed: false, result: None }))
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_completed_job<'a>(
|
||||
async fn delete_completed_job(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::Result<Response> {
|
||||
) -> error::JsonResult<CompletedJob> {
|
||||
check_scopes(&authed, || format!("deletejob"))?;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
let job_o = sqlx::query(
|
||||
let job_o = sqlx::query_as::<_, CompletedJob>(
|
||||
"UPDATE completed_job SET logs = '', result = null, deleted = true WHERE id = $1 AND workspace_id = $2 \
|
||||
RETURNING *",
|
||||
)
|
||||
@@ -2833,5 +2736,5 @@ async fn delete_completed_job<'a>(
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(CompletedJob::from_row(&job)?.into_response())
|
||||
Ok(Json(job))
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::oauth2::AllClients;
|
||||
use crate::saml::{SamlSsoLogin, ServiceProviderExt};
|
||||
use crate::scim::has_scim_token;
|
||||
use crate::tracing_init::MyOnFailure;
|
||||
use crate::workers::ALL_TAGS;
|
||||
use crate::{
|
||||
oauth2::{build_oauth_clients, SlackVerifier},
|
||||
tracing_init::{MyMakeSpan, MyOnResponse},
|
||||
@@ -35,7 +36,6 @@ use tower_http::{
|
||||
};
|
||||
use windmill_common::db::UserDB;
|
||||
use windmill_common::utils::rd_string;
|
||||
use windmill_common::worker::ALL_TAGS;
|
||||
|
||||
use windmill_common::error::AppError;
|
||||
|
||||
@@ -72,7 +72,7 @@ mod workspaces;
|
||||
pub const GIT_VERSION: &str =
|
||||
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
|
||||
|
||||
pub use users::delete_expired_items;
|
||||
pub use users::delete_expired_items_perdiodically;
|
||||
|
||||
pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
|
||||
lazy_static::lazy_static! {
|
||||
@@ -153,7 +153,7 @@ pub async fn run_server(
|
||||
port_tx: tokio::sync::oneshot::Sender<u16>,
|
||||
) -> anyhow::Result<()> {
|
||||
if let Some(mut rsmq) = rsmq.clone() {
|
||||
for tag in ALL_TAGS.read().await.iter() {
|
||||
for tag in ALL_TAGS.clone() {
|
||||
let r =
|
||||
rsmq_async::RsmqConnection::create_queue(&mut rsmq, &tag, None, None, None).await;
|
||||
if let Err(e) = r {
|
||||
|
||||
@@ -341,7 +341,6 @@ struct CreateAccount {
|
||||
refresh_token: Option<String>,
|
||||
expires_in: i64,
|
||||
}
|
||||
|
||||
async fn create_account(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
|
||||
@@ -97,21 +97,6 @@ async fn create_schedule(
|
||||
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if ns.on_recovery.is_some() {
|
||||
return Err(Error::BadRequest(
|
||||
"on_recovery is only available in enterprise version".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if ns.on_failure_times.is_some() && ns.on_failure_times.unwrap() > 1 {
|
||||
return Err(Error::BadRequest(
|
||||
"on_failure with a number of times > 1 is only available in enterprise version"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
|
||||
check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
|
||||
check_flow_conflict(
|
||||
|
||||
@@ -19,7 +19,7 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use sql_builder::prelude::*;
|
||||
use sql_builder::SqlBuilder;
|
||||
@@ -41,8 +41,7 @@ use windmill_common::{
|
||||
},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{
|
||||
list_elems_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
|
||||
Pagination, StripPath,
|
||||
list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
|
||||
@@ -81,7 +80,6 @@ pub fn global_service() -> Router {
|
||||
.route("/hub/list", get(list_hub_scripts))
|
||||
.route("/hub/get/*path", get(get_hub_script_by_path))
|
||||
.route("/hub/get_full/*path", get(get_full_hub_script_by_path))
|
||||
.route("/hub/query", get(query_hub_scripts))
|
||||
}
|
||||
|
||||
pub fn global_unauthed_service() -> Router {
|
||||
@@ -211,28 +209,6 @@ async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serd
|
||||
Ok(Json(asks))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct HubScriptsQuery {
|
||||
text: String,
|
||||
kind: Option<String>,
|
||||
limit: Option<i64>,
|
||||
}
|
||||
async fn query_hub_scripts(
|
||||
ApiAuthed { email, .. }: ApiAuthed,
|
||||
Query(query): Query<HubScriptsQuery>,
|
||||
) -> JsonResult<serde_json::Value> {
|
||||
let asks = query_elems_from_hub(
|
||||
&HTTP_CLIENT,
|
||||
"https://hub.windmill.dev/scripts/query",
|
||||
&email,
|
||||
&query.text,
|
||||
&query.kind,
|
||||
&query.limit,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(asks))
|
||||
}
|
||||
|
||||
fn hash_script(ns: &NewScript) -> i64 {
|
||||
let mut dh = DefaultHasher::new();
|
||||
ns.hash(&mut dh);
|
||||
|
||||
@@ -41,13 +41,13 @@ use time::OffsetDateTime;
|
||||
use tower_cookies::{Cookie, Cookies};
|
||||
use tracing::{Instrument, Span};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{self, to_anyhow, Error, JsonResult, Result},
|
||||
users::SUPERADMIN_SECRET_EMAIL,
|
||||
utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
const TTL_TOKEN_CACHE_S: u64 = 60; // 60s
|
||||
pub const TTL_TOKEN_DB_H: u32 = 72;
|
||||
@@ -712,7 +712,7 @@ async fn list_users(
|
||||
SELECT usr.*, usage.*
|
||||
FROM usr
|
||||
, LATERAL (
|
||||
SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions
|
||||
SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0) executions
|
||||
FROM completed_job
|
||||
WHERE workspace_id = $1
|
||||
AND job_kind NOT IN ('flow', 'flowpreview')
|
||||
@@ -2240,20 +2240,6 @@ pub struct LoginUserInfo {
|
||||
pub displayName: Option<String>,
|
||||
}
|
||||
|
||||
async fn _check_nb_of_user(db: &DB) -> Result<()> {
|
||||
let nb_groups =
|
||||
sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",)
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
if nb_groups.unwrap_or(0) >= 50 {
|
||||
return Err(Error::BadRequest(
|
||||
"You have reached the maximum number of oauth users accounts (50) without an enterprise license"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub async fn login_externally(
|
||||
db: DB,
|
||||
email: &String,
|
||||
@@ -2306,10 +2292,6 @@ pub async fn login_externally(
|
||||
if (name.is_none() || name == Some(String::new())) && user.is_some() {
|
||||
name = user.clone().unwrap().displayName;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
_check_nb_of_user(&db).await?;
|
||||
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO password (email, name, company, login_type, verified) VALUES ($1, \
|
||||
$2, $3, '{}', true)",
|
||||
@@ -2359,58 +2341,71 @@ pub async fn login_externally(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_expired_items(db: &DB) -> () {
|
||||
let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
|
||||
"DELETE FROM token WHERE expiration <= now()
|
||||
pub async fn delete_expired_items_perdiodically(
|
||||
db: &DB,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> () {
|
||||
loop {
|
||||
let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
|
||||
"DELETE FROM token WHERE expiration <= now()
|
||||
RETURNING concat(substring(token for 10), '*****')",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match tokens_deleted_r {
|
||||
Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
match tokens_deleted_r {
|
||||
Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let pip_resolution_r = sqlx::query_scalar!(
|
||||
"DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
let pip_resolution_r = sqlx::query_scalar!(
|
||||
"DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match pip_resolution_r {
|
||||
Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
|
||||
}
|
||||
match pip_resolution_r {
|
||||
Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let deleted_cache = sqlx::query_scalar!(
|
||||
let deleted_cache = sqlx::query_scalar!(
|
||||
"DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_cache {
|
||||
Ok(res) => tracing::debug!("deleted {} cache resource: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
|
||||
}
|
||||
match deleted_cache {
|
||||
Ok(res) => tracing::debug!("deleted {} cache resource: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
|
||||
}
|
||||
|
||||
if *JOB_RETENTION_SECS > 0 {
|
||||
let deleted_jobs = sqlx::query_scalar!(
|
||||
if *JOB_RETENTION_SECS > 0 {
|
||||
let deleted_jobs = sqlx::query_scalar!(
|
||||
"DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
|
||||
*JOB_RETENTION_SECS as i64
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_jobs {
|
||||
Ok(deleted_jobs) => {
|
||||
tracing::info!(
|
||||
"deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
|
||||
deleted_jobs.len(),
|
||||
deleted_jobs
|
||||
)
|
||||
match deleted_jobs {
|
||||
Ok(deleted_jobs) => {
|
||||
tracing::info!(
|
||||
"deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
|
||||
deleted_jobs.len(),
|
||||
deleted_jobs
|
||||
)
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(600)) => (),
|
||||
_ = rx. recv() => {
|
||||
println!("received killpill for delete expired tokens");
|
||||
break;
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,44 +7,74 @@
|
||||
*/
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
extract::{Extension, Query},
|
||||
routing::get,
|
||||
Json, Router,
|
||||
};
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{self, JsonResult},
|
||||
error::JsonResult,
|
||||
utils::{paginate, Pagination},
|
||||
worker::ALL_TAGS,
|
||||
DB,
|
||||
};
|
||||
|
||||
use std::collections::HashMap;
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::sync::atomic::Ordering;
|
||||
#[cfg(feature = "benchmark")]
|
||||
use windmill_queue::IDLE_WORKERS;
|
||||
|
||||
use crate::{db::ApiAuthed, utils::require_super_admin};
|
||||
use crate::db::ApiAuthed;
|
||||
|
||||
#[cfg(not(feature = "benchmark"))]
|
||||
pub fn global_service() -> Router {
|
||||
use axum::routing::post;
|
||||
|
||||
let router = Router::new()
|
||||
Router::new()
|
||||
.route("/list", get(list_worker_pings))
|
||||
.route("/custom_tags", get(get_custom_tags))
|
||||
.route("/list_worker_groups", get(get_worker_groups))
|
||||
.route(
|
||||
"/worker_group/:name",
|
||||
post(update_worker_group).delete(delete_worker_group),
|
||||
);
|
||||
#[cfg(feature = "benchmark")]
|
||||
return router.route("/toggle", get(toggle));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "benchmark"))]
|
||||
return router;
|
||||
#[cfg(feature = "benchmark")]
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/toggle", get(toggle))
|
||||
.route("/list", get(list_worker_pings))
|
||||
.route("/custom_tags", get(get_custom_tags))
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
pub static ref CUSTOM_TAGS_PER_WORKSPACE: (Vec<String>, HashMap<String, Vec<String>>) = process_custom_tags(std::env::var("CUSTOM_TAGS")
|
||||
.ok());
|
||||
|
||||
pub static ref ALL_TAGS: Vec<String> = [CUSTOM_TAGS_PER_WORKSPACE.0.clone(), CUSTOM_TAGS_PER_WORKSPACE.1.keys().map(|x| x.to_string()).collect_vec()].concat();
|
||||
|
||||
}
|
||||
|
||||
fn process_custom_tags(o: Option<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
|
||||
let regex = Regex::new(r"^(\w+)\(((?:\w+)\+?)+\)$").unwrap();
|
||||
if let Some(s) = o {
|
||||
let mut global = vec![];
|
||||
let mut specific: HashMap<String, Vec<String>> = HashMap::new();
|
||||
for e in s.split(",") {
|
||||
if let Some(cap) = regex.captures(e) {
|
||||
let tag = cap.get(1).unwrap().as_str().to_string();
|
||||
let workspaces = cap.get(2).unwrap().as_str().split("+");
|
||||
specific.insert(tag, workspaces.map(|x| x.to_string()).collect_vec());
|
||||
} else {
|
||||
global.push(e.to_string());
|
||||
}
|
||||
}
|
||||
(global, specific)
|
||||
} else {
|
||||
(vec![], HashMap::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize)]
|
||||
@@ -56,7 +86,6 @@ struct WorkerPing {
|
||||
ip: String,
|
||||
jobs_executed: i32,
|
||||
custom_tags: Option<Vec<String>>,
|
||||
worker_group: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -75,7 +104,7 @@ async fn list_worker_pings(
|
||||
|
||||
let rows = sqlx::query_as!(
|
||||
WorkerPing,
|
||||
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
per_page as i64,
|
||||
offset as i64
|
||||
)
|
||||
@@ -92,75 +121,5 @@ async fn toggle(Query(query): Query<EnableWorkerQuery>) -> JsonResult<bool> {
|
||||
}
|
||||
|
||||
async fn get_custom_tags() -> Json<Vec<String>> {
|
||||
Json(ALL_TAGS.read().await.clone().into())
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, FromRow)]
|
||||
struct WorkerGroup {
|
||||
name: String,
|
||||
config: serde_json::Value,
|
||||
}
|
||||
|
||||
async fn get_worker_groups(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
) -> error::JsonResult<Vec<WorkerGroup>> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
|
||||
let rows = sqlx::query_as!(WorkerGroup, "SELECT * FROM worker_group_config")
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn update_worker_group(
|
||||
Path(name): Path<String>,
|
||||
Extension(db): Extension<DB>,
|
||||
authed: ApiAuthed,
|
||||
Json(config): Json<serde_json::Value>,
|
||||
) -> error::Result<String> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO worker_group_config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
|
||||
&name,
|
||||
config
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
Ok(format!("Updated worker group {name}"))
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
async fn update_worker_group() -> String {
|
||||
"Worker groups available only in enterprise version".to_string()
|
||||
}
|
||||
|
||||
async fn delete_worker_group(
|
||||
Path(name): Path<String>,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
authed: ApiAuthed,
|
||||
) -> error::Result<String> {
|
||||
let tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let deleted = sqlx::query!(
|
||||
"DELETE FROM worker_group_config WHERE name = $1 RETURNING name",
|
||||
name,
|
||||
)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
|
||||
if deleted.len() == 0 {
|
||||
return Err(error::Error::NotFound(format!(
|
||||
"Worker group {name} not found",
|
||||
name = name
|
||||
)));
|
||||
}
|
||||
Ok(format!("Deleted worker group {name}"))
|
||||
Json(ALL_TAGS.clone())
|
||||
}
|
||||
|
||||
@@ -74,19 +74,13 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/edit_error_handler", post(edit_error_handler));
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
if std::env::var("STRIPE_KEY").is_err() {
|
||||
return router;
|
||||
} else {
|
||||
tracing::info!("stripe enabled");
|
||||
|
||||
return router
|
||||
.route("/checkout", get(stripe_checkout))
|
||||
.route("/billing_portal", get(stripe_portal));
|
||||
}
|
||||
}
|
||||
tracing::info!("stripe enabled");
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
let router = router
|
||||
.route("/checkout", get(stripe_checkout))
|
||||
.route("/billing_portal", get(stripe_portal));
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
router
|
||||
}
|
||||
pub fn global_service() -> Router {
|
||||
@@ -849,20 +843,6 @@ lazy_static::lazy_static! {
|
||||
|
||||
}
|
||||
|
||||
async fn _check_nb_of_workspaces(db: &DB) -> Result<()> {
|
||||
let nb_workspaces = sqlx::query_scalar!("SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",)
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
if nb_workspaces.unwrap_or(0) >= 3 {
|
||||
return Err(Error::BadRequest(
|
||||
"You have reached the maximum number of workspaces (3 outside of default group 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
async fn create_workspace(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
@@ -872,10 +852,6 @@ async fn create_workspace(
|
||||
if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
_check_nb_of_workspaces(&db).await?;
|
||||
|
||||
let mut tx: Transaction<'_, Postgres> = db.begin().await?;
|
||||
|
||||
check_name_conflict(&mut tx, &nw.id).await?;
|
||||
|
||||
@@ -44,5 +44,3 @@ reqwest = { workspace = true, optional = true }
|
||||
tracing-subscriber = { workspace = true, optional = true }
|
||||
lazy_static.workspace = true
|
||||
tracing-flame = { version = "^0", optional = true }
|
||||
itertools.workspace = true
|
||||
regex.workspace = true
|
||||
@@ -1,5 +1,4 @@
|
||||
pub const WORKER_S3_BUCKET_SYNC: &str = "worker_s3_bucket_sync";
|
||||
pub const CUSTOM_TAGS_SETTING: &str = "custom_tags";
|
||||
|
||||
pub const ENV_SETTINGS: [&str; 54] = [
|
||||
"DISABLE_NSJAIL",
|
||||
@@ -42,6 +41,8 @@ pub const ENV_SETTINGS: [&str; 54] = [
|
||||
"INSTANCE_EVENTS_WEBHOOK",
|
||||
"CLOUD_HOSTED",
|
||||
"GLOBAL_CACHE_INTERVAL",
|
||||
"WORKER_TAGS",
|
||||
"CUSTOM_TAGS",
|
||||
"JOB_RETENTION_SECS",
|
||||
"WAIT_RESULT_FAST_POLL_DURATION_SECS",
|
||||
"WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
|
||||
@@ -54,6 +55,4 @@ pub const ENV_SETTINGS: [&str; 54] = [
|
||||
"SMTP_TLS_IMPLICIT",
|
||||
"CREATE_WORKSPACE_REQUIRE_SUPERADMIN",
|
||||
"GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE",
|
||||
"MAX_WAIT_FOR_SIGTERM",
|
||||
"WORKER_GROUP",
|
||||
];
|
||||
|
||||
@@ -27,7 +27,6 @@ pub mod scripts;
|
||||
pub mod users;
|
||||
pub mod utils;
|
||||
pub mod variables;
|
||||
pub mod worker;
|
||||
|
||||
#[cfg(feature = "tracing_init")]
|
||||
pub mod tracing_init;
|
||||
@@ -78,15 +77,12 @@ pub async fn shutdown_signal(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "prometheus")]
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
#[cfg(feature = "prometheus")]
|
||||
pub async fn serve_metrics(
|
||||
addr: SocketAddr,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
ready_worker_endpoint: bool,
|
||||
) -> JoinHandle<()> {
|
||||
) -> Result<(), hyper::Error> {
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use axum::{routing::get, Router};
|
||||
@@ -108,18 +104,13 @@ pub async fn serve_metrics(
|
||||
router
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = axum::Server::bind(&addr)
|
||||
.serve(router.into_make_service())
|
||||
.with_graceful_shutdown(async {
|
||||
rx.recv().await.ok();
|
||||
println!("Graceful shutdown of metrics");
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error serving metrics: {}", e);
|
||||
}
|
||||
})
|
||||
axum::Server::bind(&addr)
|
||||
.serve(router.into_make_service())
|
||||
.with_graceful_shutdown(async {
|
||||
rx.recv().await.ok();
|
||||
println!("Graceful shutdown of metrics");
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn metrics() -> Result<String, Error> {
|
||||
|
||||
@@ -274,7 +274,6 @@ pub async fn get_hub_script_by_path(
|
||||
&format!("https://hub.windmill.dev/raw/{path}.ts"),
|
||||
email,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.text()
|
||||
@@ -304,7 +303,6 @@ pub async fn get_full_hub_script_by_path(
|
||||
&format!("https://hub.windmill.dev/raw2/{path}"),
|
||||
email,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.json::<HubScript>()
|
||||
|
||||
@@ -79,32 +79,7 @@ pub async fn list_elems_from_hub(
|
||||
url: &str,
|
||||
email: &str,
|
||||
) -> Result<serde_json::Value> {
|
||||
let rows = http_get_from_hub(http_client, url, email, false, None)
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await
|
||||
.map_err(crate::error::to_anyhow)?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
#[cfg(feature = "reqwest")]
|
||||
pub async fn query_elems_from_hub(
|
||||
http_client: &reqwest::Client,
|
||||
url: &str,
|
||||
email: &str,
|
||||
query_text: &str,
|
||||
query_kind: &Option<String>,
|
||||
query_limit: &Option<i64>,
|
||||
) -> Result<serde_json::Value> {
|
||||
let mut query_params = vec![("text", query_text)];
|
||||
if let Some(query_kind) = query_kind {
|
||||
query_params.push(("kind", query_kind.as_str()));
|
||||
}
|
||||
let query_limit = query_limit.unwrap_or(0).to_string();
|
||||
if query_limit.parse::<i64>().unwrap() > 0 {
|
||||
query_params.push(("limit", query_limit.as_str()));
|
||||
}
|
||||
let rows = http_get_from_hub(http_client, url, email, false, Some(query_params))
|
||||
let rows = http_get_from_hub(http_client, url, email, false)
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await
|
||||
@@ -118,9 +93,8 @@ pub async fn http_get_from_hub(
|
||||
url: &str,
|
||||
email: &str,
|
||||
plain: bool,
|
||||
query_params: Option<Vec<(&str, &str)>>,
|
||||
) -> Result<reqwest::Response> {
|
||||
let mut request = http_client
|
||||
let response = http_client
|
||||
.get(url)
|
||||
.header(
|
||||
"Accept",
|
||||
@@ -130,15 +104,10 @@ pub async fn http_get_from_hub(
|
||||
"application/json"
|
||||
},
|
||||
)
|
||||
.header("X-email", email);
|
||||
|
||||
if let Some(query_params) = query_params {
|
||||
for (key, value) in query_params {
|
||||
request = request.query(&[(key, value)]);
|
||||
}
|
||||
}
|
||||
|
||||
let response = request.send().await.map_err(crate::error::to_anyhow)?;
|
||||
.header("X-email", email)
|
||||
.send()
|
||||
.await
|
||||
.map_err(crate::error::to_anyhow)?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, DB};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
|
||||
|
||||
pub static ref DEFAULT_TAGS : Vec<String> = vec![
|
||||
"deno".to_string(),
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"powershell".to_string(),
|
||||
"nativets".to_string(),
|
||||
"mysql".to_string(),
|
||||
"graphql".to_string(),
|
||||
"bun".to_string(),
|
||||
"postgresql".to_string(),
|
||||
"bigquery".to_string(),
|
||||
"snowflake".to_string(),
|
||||
"graphql".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
"other".to_string()];
|
||||
|
||||
|
||||
pub static ref WORKER_CONFIG: Arc<RwLock<WorkerConfig>> = Arc::new(RwLock::new(WorkerConfig {
|
||||
worker_tags: Default::default(),
|
||||
dedicated_worker: Default::default(),
|
||||
}));
|
||||
|
||||
|
||||
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
|
||||
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
|
||||
pub static ref CUSTOM_TAGS_PER_WORKSPACE: Arc<RwLock<(Vec<String>, HashMap<String, Vec<String>>)>> = Arc::new(RwLock::new((vec![], HashMap::new())));
|
||||
|
||||
pub static ref ALL_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(vec![]));
|
||||
|
||||
static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^(\w+)\(((?:\w+)\+?)+\)$").unwrap();
|
||||
|
||||
}
|
||||
|
||||
pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {
|
||||
let q = sqlx::query!(
|
||||
"SELECT value FROM global_settings WHERE name = $1",
|
||||
CUSTOM_TAGS_SETTING
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let tags = if let Some(q) = q {
|
||||
if let Ok(v) = serde_json::from_value::<Vec<String>>(q.value.clone()) {
|
||||
v
|
||||
} else {
|
||||
tracing::error!(
|
||||
"Could not parse custom tags setting as vec of strings, found: {:#?}",
|
||||
&q.value
|
||||
);
|
||||
vec![]
|
||||
}
|
||||
} else {
|
||||
CUSTOM_TAGS.clone()
|
||||
};
|
||||
|
||||
let custom_tags = process_custom_tags(tags);
|
||||
|
||||
{
|
||||
let l = CUSTOM_TAGS_PER_WORKSPACE.read().await;
|
||||
if l.clone() == custom_tags {
|
||||
tracing::info!("Custom tags setting unchanged, skipping update");
|
||||
return Ok(());
|
||||
} else {
|
||||
tracing::info!("Custom tags setting changed, updating");
|
||||
}
|
||||
}
|
||||
{
|
||||
let mut l = CUSTOM_TAGS_PER_WORKSPACE.write().await;
|
||||
*l = custom_tags.clone()
|
||||
}
|
||||
{
|
||||
let mut l = ALL_TAGS.write().await;
|
||||
*l = [
|
||||
custom_tags.0.clone(),
|
||||
custom_tags.1.keys().map(|x| x.to_string()).collect_vec(),
|
||||
]
|
||||
.concat();
|
||||
}
|
||||
Ok(())
|
||||
// pub static ref CUSTOM_TAGS_PER_WORKSPACE: (Vec<String>, HashMap<String, Vec<String>>) = process_custom_tags(std::env::var("CUSTOM_TAGS")
|
||||
// .ok());
|
||||
|
||||
// pub static ref ALL_TAGS: Vec<String> = [CUSTOM_TAGS_PER_WORKSPACE.0.clone(), CUSTOM_TAGS_PER_WORKSPACE.1.keys().map(|x| x.to_string()).collect_vec()].concat();
|
||||
}
|
||||
|
||||
fn process_custom_tags(tags: Vec<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
|
||||
let mut global = vec![];
|
||||
let mut specific: HashMap<String, Vec<String>> = HashMap::new();
|
||||
for e in tags {
|
||||
if let Some(cap) = CUSTOM_TAG_REGEX.captures(&e) {
|
||||
let tag = cap.get(1).unwrap().as_str().to_string();
|
||||
let workspaces = cap.get(2).unwrap().as_str().split("+");
|
||||
specific.insert(tag, workspaces.map(|x| x.to_string()).collect_vec());
|
||||
} else {
|
||||
global.push(e.to_string());
|
||||
}
|
||||
}
|
||||
(global, specific)
|
||||
}
|
||||
|
||||
pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
|
||||
let wc = WORKER_CONFIG.read().await;
|
||||
let tags = wc.worker_tags.as_slice();
|
||||
sqlx::query!(
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
|
||||
worker_instance,
|
||||
worker_name,
|
||||
ip,
|
||||
tags,
|
||||
*WORKER_GROUP,
|
||||
wc.dedicated_worker.as_ref().map(|x| format!("{}:{}", x.workspace_id, x.path))
|
||||
)
|
||||
.execute(db)
|
||||
.await
|
||||
.expect("insert worker_ping initial value");
|
||||
}
|
||||
|
||||
pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
|
||||
let config: WorkerConfigOpt = sqlx::query_scalar!(
|
||||
"SELECT config FROM worker_group_config WHERE name = $1",
|
||||
*WORKER_GROUP
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.flatten()
|
||||
.map(|x| serde_json::from_value(x).ok())
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
let dedicated_worker = config.dedicated_worker.map(|x| {
|
||||
let splitted = x.split(':').to_owned().collect_vec();
|
||||
if splitted.len() != 2 {
|
||||
panic!("DEDICATED_WORKER setting should be in the form of <workspace>:<script_path>")
|
||||
} else {
|
||||
let workspace = splitted[0];
|
||||
let script_path = splitted[1];
|
||||
WorkspacedPath { workspace_id: workspace.to_string(), path: script_path.to_string() }
|
||||
}
|
||||
});
|
||||
Ok(WorkerConfig {
|
||||
worker_tags: config
|
||||
.worker_tags
|
||||
.or_else(|| {
|
||||
if let Some(ref dedicated_worker) = dedicated_worker.as_ref() {
|
||||
Some(vec![format!(
|
||||
"{}:{}",
|
||||
dedicated_worker.workspace_id, dedicated_worker.path
|
||||
)])
|
||||
} else {
|
||||
std::env::var("WORKER_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect())
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| DEFAULT_TAGS.clone()),
|
||||
dedicated_worker,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Debug)]
|
||||
pub struct WorkspacedPath {
|
||||
pub workspace_id: String,
|
||||
pub path: String,
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct WorkerConfigOpt {
|
||||
pub worker_tags: Option<Vec<String>>,
|
||||
pub dedicated_worker: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for WorkerConfigOpt {
|
||||
fn default() -> Self {
|
||||
Self { worker_tags: Default::default(), dedicated_worker: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Debug)]
|
||||
pub struct WorkerConfig {
|
||||
pub worker_tags: Vec<String>,
|
||||
pub dedicated_worker: Option<WorkspacedPath>,
|
||||
}
|
||||
@@ -37,4 +37,3 @@ tokio.workspace = true
|
||||
futures-core.workspace = true
|
||||
itertools.workspace = true
|
||||
async-recursion.workspace = true
|
||||
bigdecimal.workspace = true
|
||||
@@ -8,16 +8,17 @@
|
||||
|
||||
use std::{collections::HashMap, sync::atomic::AtomicBool, vec};
|
||||
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::Context;
|
||||
use async_recursion::async_recursion;
|
||||
use bigdecimal::ToPrimitive;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use itertools::Itertools;
|
||||
use reqwest::Client;
|
||||
use rsmq_async::RsmqConnection;
|
||||
use serde_json::json;
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::time::Instant;
|
||||
use tracing::{instrument, Instrument};
|
||||
use ulid::Ulid;
|
||||
use uuid::Uuid;
|
||||
@@ -36,13 +37,9 @@ use windmill_common::{
|
||||
schedule::{schedule_to_user, Schedule},
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
users::{username_to_permissioned_as, SUPERADMIN_SECRET_EMAIL},
|
||||
worker::WORKER_CONFIG,
|
||||
DB, METRICS_ENABLED,
|
||||
};
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
|
||||
use crate::{
|
||||
schedule::{get_schedule_opt, push_scheduled_job},
|
||||
QueueTransaction,
|
||||
@@ -69,6 +66,57 @@ lazy_static::lazy_static! {
|
||||
"Total number of jobs pulled from the queue."
|
||||
)
|
||||
.unwrap();
|
||||
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
|
||||
|
||||
pub static ref DEFAULT_TAGS : Vec<String> = vec![
|
||||
"deno".to_string(),
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"powershell".to_string(),
|
||||
"nativets".to_string(),
|
||||
"mysql".to_string(),
|
||||
"graphql".to_string(),
|
||||
"bun".to_string(),
|
||||
"postgresql".to_string(),
|
||||
"bigquery".to_string(),
|
||||
"snowflake".to_string(),
|
||||
"graphql".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
"other".to_string()];
|
||||
|
||||
pub static ref DEDICATED_WORKER: Option<(String, String)> = std::env::var("DEDICATED_WORKER")
|
||||
.ok()
|
||||
.map(|x| {
|
||||
let splitted = x.split(':').to_owned().collect_vec();
|
||||
if splitted.len() != 2 {
|
||||
panic!("DEDICATED_WORKER should be in the form of <workspace>:<script_path>")
|
||||
} else {
|
||||
let workspace = splitted[0];
|
||||
let script_path = splitted[1];
|
||||
(workspace.to_string(), script_path.to_string())
|
||||
}
|
||||
});
|
||||
|
||||
pub static ref ACCEPTED_TAGS: Vec<String> = {
|
||||
let worker_tags = std::env::var("WORKER_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect())
|
||||
.unwrap_or_else(|| DEFAULT_TAGS.clone());
|
||||
if let Some(ref dedicated_worker) = DEDICATED_WORKER.as_ref() {
|
||||
vec![format!("{}:{}", dedicated_worker.0, dedicated_worker.1)]
|
||||
} else {
|
||||
worker_tags
|
||||
}
|
||||
};
|
||||
|
||||
pub static ref IS_WORKER_TAGS_DEFINED: bool = std::env::var("WORKER_TAGS").ok().is_some();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// When compiled in 'benchmark' mode, this flags is exposed via the /workers/toggle endpoint
|
||||
// and make it possible to disable to current active workers (such that they don't pull any)
|
||||
@@ -236,8 +284,6 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|x| x.to_i64())
|
||||
.flatten()
|
||||
} else {
|
||||
tracing::warn!("Could not parse flow status");
|
||||
None
|
||||
@@ -254,7 +300,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
.flatten();
|
||||
let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
|
||||
let job_id = queued_job.id.clone();
|
||||
let _duration: i64 = sqlx::query_scalar!(
|
||||
let _duration = sqlx::query_scalar!(
|
||||
"INSERT INTO completed_job AS cj
|
||||
( workspace_id
|
||||
, id
|
||||
@@ -379,7 +425,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
if !is_flow && _duration > 1000 {
|
||||
let additional_usage = _duration / 1000;
|
||||
let w_id = &queued_job.workspace_id;
|
||||
let premium_workspace = *windmill_common::worker::CLOUD_HOSTED
|
||||
let premium_workspace = *CLOUD_HOSTED
|
||||
&& sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
@@ -390,7 +436,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $3",
|
||||
if premium_workspace { w_id } else { &queued_job.email },
|
||||
premium_workspace,
|
||||
additional_usage as i32)
|
||||
additional_usage)
|
||||
.execute(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("updating usage: {e}")));
|
||||
@@ -1026,8 +1072,8 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
|
||||
|
||||
let job_uuid: Uuid = pulled_job.id;
|
||||
let min_started_at: Option<DateTime<Utc>> = script_path_live_stats.min_started_at;
|
||||
let avg_script_duration: Option<i64> = sqlx::query_scalar!(
|
||||
"SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM
|
||||
let avg_script_duration: Option<i32> = sqlx::query_scalar!(
|
||||
"SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM
|
||||
(SELECT duration_ms FROM completed_job WHERE script_path = $1
|
||||
ORDER BY started_at
|
||||
DESC LIMIT 10) AS t",
|
||||
@@ -1107,7 +1153,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
// TODO: REDIS: Race conditions / replace last_ping
|
||||
|
||||
// TODO: shuffle this list to have fairness
|
||||
let mut all_tags = WORKER_CONFIG.read().await.worker_tags.clone();
|
||||
let mut all_tags = ACCEPTED_TAGS.clone();
|
||||
|
||||
let mut msg: Option<_> = None;
|
||||
let mut tag = None;
|
||||
@@ -1164,8 +1210,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
* suspend_until is non-null
|
||||
* and suspend = 0 when the resume messages are received
|
||||
* or suspend_until <= now() if it has timed out */
|
||||
let config = WORKER_CONFIG.read().await;
|
||||
let tags = config.worker_tags.as_slice();
|
||||
|
||||
let r = if suspend_first {
|
||||
sqlx::query_as::<_, QueuedJob>("UPDATE queue
|
||||
@@ -1182,21 +1226,17 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING *")
|
||||
.bind(tags)
|
||||
.bind(ACCEPTED_TAGS.as_slice())
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
drop(config);
|
||||
|
||||
if r.is_none() {
|
||||
// #[cfg(feature = "benchmark")]
|
||||
// let instant = Instant::now();
|
||||
|
||||
let config = WORKER_CONFIG.read().await;
|
||||
let tags = config.worker_tags.as_slice();
|
||||
|
||||
let r = sqlx::query_as::<_, QueuedJob>(
|
||||
"UPDATE queue
|
||||
SET running = true
|
||||
@@ -1213,9 +1253,10 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
)
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(tags)
|
||||
.bind(ACCEPTED_TAGS.as_slice())
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
// #[cfg(feature = "benchmark")]
|
||||
// println!("pull query: {:?}", instant.elapsed());
|
||||
|
||||
@@ -1312,35 +1353,15 @@ pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
w_id,
|
||||
job_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await;
|
||||
|
||||
if let Err(job_removed) = job_removed {
|
||||
tracing::error!(
|
||||
"Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"
|
||||
);
|
||||
} else {
|
||||
let job_removed = job_removed.unwrap().flatten().unwrap_or(0);
|
||||
if job_removed != 1 {
|
||||
tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling");
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!("Job {job_id} deleted");
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
|
||||
.unwrap_or(0)
|
||||
== 1;
|
||||
tracing::debug!("Job {job_id} deleted: {job_removed}");
|
||||
Ok(tx)
|
||||
}
|
||||
|
||||
pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result<bool> {
|
||||
Ok(sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
|
||||
id,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await?
|
||||
.unwrap_or(false))
|
||||
}
|
||||
|
||||
pub async fn get_queued_job<'c>(
|
||||
id: Uuid,
|
||||
w_id: &str,
|
||||
|
||||
@@ -71,7 +71,6 @@ jsonwebtoken = { workspace = true, optional = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
pem = { workspace = true, optional = true }
|
||||
urlencoding.workspace = true
|
||||
nix.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
deno_fetch.workspace = true
|
||||
|
||||
@@ -7,8 +7,6 @@ const bo = await Bun.build({
|
||||
|
||||
const fs = require("fs/promises");
|
||||
|
||||
const captureVersion = /(^\@?[^\@]+)(?:\@(.+))?/;
|
||||
|
||||
if (!bo.success) {
|
||||
bo.logs.forEach((l) => console.log(l));
|
||||
process.exit(1);
|
||||
@@ -19,7 +17,7 @@ if (!bo.success) {
|
||||
const { intersect } = require("semver-intersect");
|
||||
const dependencies: Record<string, string[]> = {};
|
||||
for (const i of imports) {
|
||||
let [_, name, version] = i.path.match(captureVersion) ?? [];
|
||||
let [name, version] = i.path.split("@");
|
||||
if (version == undefined) {
|
||||
if (dependencies[name] == undefined) {
|
||||
dependencies[name] = [];
|
||||
|
||||
@@ -123,7 +123,6 @@ pub async fn handle_bash_job(
|
||||
&job.workspace_id,
|
||||
"bash run",
|
||||
job.timeout,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -261,7 +260,6 @@ pub async fn handle_powershell_job(
|
||||
&job.workspace_id,
|
||||
"bash/powershell run",
|
||||
job.timeout,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -105,7 +105,6 @@ pub async fn gen_lockfile(
|
||||
w_id,
|
||||
"bun build",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -172,7 +171,6 @@ pub async fn install_lockfile(
|
||||
w_id,
|
||||
"bun install",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -412,7 +410,6 @@ plugin(p)
|
||||
&job.workspace_id,
|
||||
"bun run",
|
||||
job.timeout,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
use async_recursion::async_recursion;
|
||||
use nix::sys::signal::{self, Signal};
|
||||
use nix::unistd::Pid;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{fs::File, io::AsyncReadExt};
|
||||
use windmill_api_client::{types::CreateResource, Client};
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
jobs::QueuedJob,
|
||||
variables::ContextualVariable,
|
||||
};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
use anyhow::Result;
|
||||
use std::{
|
||||
@@ -39,9 +37,7 @@ use futures::{
|
||||
stream, StreamExt,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
AuthedClient, MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGTERM, TIMEOUT_DURATION, WHITELIST_ENVS,
|
||||
};
|
||||
use crate::{AuthedClient, MAX_RESULT_SIZE, TIMEOUT_DURATION, WHITELIST_ENVS};
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn create_args_and_out_file(
|
||||
@@ -254,7 +250,6 @@ pub async fn handle_child(
|
||||
_w_id: &str,
|
||||
child_name: &str,
|
||||
custom_timeout: Option<i32>,
|
||||
sigterm: bool,
|
||||
) -> error::Result<()> {
|
||||
let start = Instant::now();
|
||||
let update_job_interval = Duration::from_millis(500);
|
||||
@@ -393,21 +388,6 @@ pub async fn handle_child(
|
||||
}
|
||||
};
|
||||
|
||||
if sigterm {
|
||||
if let Some(id) = child.id() {
|
||||
signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
|
||||
for _ in 0..*MAX_WAIT_FOR_SIGTERM {
|
||||
if child.try_wait().is_ok_and(|x| x.is_some()) {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
if child.try_wait().is_ok_and(|x| x.is_some()) {
|
||||
set_reason.await;
|
||||
return Ok(Err(kill_reason));
|
||||
}
|
||||
}
|
||||
}
|
||||
/* send SIGKILL and reap child process */
|
||||
let (_, kill) = future::join(set_reason, child.kill()).await;
|
||||
kill.map(|()| Err(kill_reason))
|
||||
|
||||
@@ -112,7 +112,6 @@ pub async fn generate_deno_lock(
|
||||
w_id,
|
||||
"deno cache",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -316,7 +315,6 @@ run().catch(async (e) => {{
|
||||
&job.workspace_id,
|
||||
"deno run",
|
||||
job.timeout,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
// logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
|
||||
|
||||
@@ -198,7 +198,6 @@ func Run(req Req) (interface{{}}, error){{
|
||||
&job.workspace_id,
|
||||
"go build",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -275,7 +274,6 @@ func Run(req Req) (interface{{}}, error){{
|
||||
&job.workspace_id,
|
||||
"go run",
|
||||
job.timeout,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
@@ -333,7 +331,6 @@ pub async fn install_go_dependencies(
|
||||
w_id,
|
||||
"go init",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -394,7 +391,6 @@ pub async fn install_go_dependencies(
|
||||
&w_id,
|
||||
&format!("go {mod_command}"),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lockfile generation failed: {e:?}")))?;
|
||||
|
||||
@@ -6,7 +6,6 @@ mod snowflake_executor;
|
||||
mod bash_executor;
|
||||
mod bun_executor;
|
||||
mod common;
|
||||
mod config;
|
||||
mod dedicated_worker;
|
||||
mod deno_executor;
|
||||
mod global_cache;
|
||||
@@ -18,4 +17,5 @@ mod pg_executor;
|
||||
mod python_executor;
|
||||
mod worker;
|
||||
mod worker_flow;
|
||||
|
||||
pub use worker::*;
|
||||
|
||||
@@ -218,15 +218,15 @@ pub fn pg_cell_to_json_value(
|
||||
Type::DATE => get_basic(row, column, column_i, |a: chrono::NaiveDate| {
|
||||
Ok(JSONValue::String(a.to_string()))
|
||||
})?,
|
||||
Type::TIME => get_basic(row, column, column_i, |a: chrono::NaiveTime| {
|
||||
Ok(JSONValue::String(a.to_string()))
|
||||
})?,
|
||||
Type::TIMESTAMPTZ => get_basic(row, column, column_i, |a: chrono::DateTime<Utc>| {
|
||||
Ok(JSONValue::String(a.to_string()))
|
||||
})?,
|
||||
Type::UUID => get_basic(row, column, column_i, |a: uuid::Uuid| {
|
||||
Ok(JSONValue::String(a.to_string()))
|
||||
})?,
|
||||
// Type::DATE => get_basic(row, column, column_i, |a: chrono::NaiveDate| {
|
||||
// Ok(JSONValue::String(a.to_string()))
|
||||
// })?,
|
||||
Type::JSON | Type::JSONB => get_basic(row, column, column_i, |a: JSONValue| Ok(a))?,
|
||||
Type::FLOAT4 => get_basic(row, column, column_i, |a: f32| {
|
||||
Ok(f64_to_json_number(a.into())?)
|
||||
|
||||
@@ -143,7 +143,6 @@ pub async fn pip_compile(
|
||||
&w_id,
|
||||
"pip-compile",
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
@@ -479,7 +478,6 @@ mount {{
|
||||
&job.workspace_id,
|
||||
"python run",
|
||||
job.timeout,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
@@ -639,7 +637,6 @@ pub async fn handle_python_reqs(
|
||||
&w_id,
|
||||
&format!("pip install {req}"),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
|
||||
@@ -28,10 +28,12 @@ use windmill_common::{
|
||||
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
|
||||
users::SUPERADMIN_SECRET_EMAIL,
|
||||
utils::{rd_string, StripPath},
|
||||
worker::{update_ping, CLOUD_HOSTED, WORKER_CONFIG},
|
||||
DB, IS_READY, METRICS_ENABLED,
|
||||
};
|
||||
use windmill_queue::{canceled_job_to_result, get_queued_job, pull, HTTP_CLIENT};
|
||||
use windmill_queue::{
|
||||
canceled_job_to_result, get_queued_job, pull, ACCEPTED_TAGS, CLOUD_HOSTED, DEDICATED_WORKER,
|
||||
HTTP_CLIENT, IS_WORKER_TAGS_DEFINED,
|
||||
};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
@@ -251,11 +253,6 @@ lazy_static::lazy_static! {
|
||||
.and_then(|x| x.parse::<u64>().ok())
|
||||
.unwrap_or_else(|| if *CLOUD_HOSTED { DEFAULT_CLOUD_TIMEOUT } else { DEFAULT_SELFHOSTED_TIMEOUT });
|
||||
|
||||
pub static ref MAX_WAIT_FOR_SIGTERM: u64 = std::env::var("MAX_WAIT_FOR_SIGTERM")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<u64>().ok())
|
||||
.unwrap_or_else(|| 5);
|
||||
|
||||
pub static ref TIMEOUT_DURATION: Duration = Duration::from_secs(*TIMEOUT);
|
||||
|
||||
pub static ref SCRIPT_TOKEN_EXPIRY: i32 = std::env::var("SCRIPT_TOKEN_EXPIRY")
|
||||
@@ -348,7 +345,6 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
_num_workers: u32,
|
||||
ip: &str,
|
||||
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
killpill_tx: tokio::sync::broadcast::Sender<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
_sync_barrier: Arc<RwLock<Option<Barrier>>>,
|
||||
@@ -387,7 +383,7 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
|
||||
let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);
|
||||
|
||||
update_ping(worker_instance, &worker_name, ip, db).await;
|
||||
insert_initial_ping(worker_instance, &worker_name, ip, db).await;
|
||||
|
||||
let uptime_metric =
|
||||
prometheus::register_counter!(WORKER_UPTIME_OPTS.clone().const_label("name", &worker_name))
|
||||
@@ -608,23 +604,19 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
let vacuum_shift = rand::thread_rng().gen_range(0..VACUUM_PERIOD);
|
||||
|
||||
IS_READY.store(true, Ordering::Relaxed);
|
||||
tracing::info!(worker = %worker_name, "listening for jobs, config: {:#?}", WORKER_CONFIG.read().await);
|
||||
tracing::info!(worker = %worker_name, "listening for jobs");
|
||||
|
||||
let (dedicated_worker_tx, dedicated_worker_handle) = if let Some(_wp) =
|
||||
WORKER_CONFIG.read().await.dedicated_worker.clone()
|
||||
let (dedicated_worker_tx, dedicated_worker_handle) = if let Some((_workspace, _script_path)) =
|
||||
DEDICATED_WORKER.clone()
|
||||
{
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
{
|
||||
tracing::error!("Dedicated worker is an enterprise feature");
|
||||
killpill_tx.send(()).expect("send");
|
||||
return;
|
||||
}
|
||||
panic!("Dedicated worker is an enterprise feature");
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
let (dedicated_worker_tx, dedicated_worker_rx) =
|
||||
mpsc::channel::<QueuedJob>(MAX_BUFFERED_DEDICATED_JOBS);
|
||||
let mut killpill_rx = killpill_rx.resubscribe();
|
||||
let killpill_rx = killpill_rx.resubscribe();
|
||||
let db = db.clone();
|
||||
let worker_dir = worker_dir.clone();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
@@ -648,52 +640,19 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
.execute(&db)
|
||||
.await
|
||||
{
|
||||
tracing::error!("failed to create token for dedicated worker: {:?}", e);
|
||||
killpill_tx.send(()).expect("send");
|
||||
panic!("failed to create token for dedicated worker: {:?}", e)
|
||||
};
|
||||
|
||||
let (content, lock, _language, envs) = {
|
||||
let r;
|
||||
loop {
|
||||
let q = sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>, Option<Vec<String>>)>(
|
||||
let (content, lock, _language, envs) = sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>, Option<Vec<String>>)>(
|
||||
"SELECT content, lock, language, envs FROM script WHERE path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
|
||||
deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
|
||||
)
|
||||
.bind(&_wp.path)
|
||||
.bind(&_wp.workspace_id)
|
||||
.bind(&_script_path)
|
||||
.bind(&_workspace)
|
||||
.fetch_optional(&db)
|
||||
.await;
|
||||
if let Ok(q) = q {
|
||||
if let Some(wp) = q {
|
||||
r = wp;
|
||||
break;
|
||||
} else {
|
||||
tracing::error!(
|
||||
"Failed to fetch script `{}` in workspace {} for dedicated worker. Retrying in 10s.",
|
||||
_wp.path,
|
||||
_wp.workspace_id
|
||||
);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = killpill_rx.recv() => {
|
||||
tracing::info!("Killing dedicated worker while it was attempting to fetch script");
|
||||
return;
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_secs(10)) => {
|
||||
continue;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tracing::error!("Failed to fetch script for dedicated worker");
|
||||
killpill_tx.send(()).expect("send");
|
||||
return;
|
||||
}
|
||||
}
|
||||
r
|
||||
};
|
||||
.await.expect("Failed to fetch script for dedicated worker")
|
||||
.expect(&format!("Failed to fetch script `{_script_path}` in workspace {_workspace} for dedicated worker"));
|
||||
|
||||
let worker_envs = build_envs(envs).expect("failed to build envs");
|
||||
if let Err(e) = start_worker(
|
||||
@@ -704,8 +663,8 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
&job_dir,
|
||||
&worker_name,
|
||||
worker_envs,
|
||||
&_wp.workspace_id,
|
||||
&_wp.path,
|
||||
&_workspace,
|
||||
&_script_path,
|
||||
&token,
|
||||
job_completed_tx,
|
||||
dedicated_worker_rx,
|
||||
@@ -742,13 +701,9 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
|
||||
let copy_tx = _copy_to_bucket_tx.clone();
|
||||
|
||||
if last_ping.elapsed().as_secs() > NUM_SECS_PING {
|
||||
let wc = WORKER_CONFIG.read().await;
|
||||
let tags = wc.worker_tags.as_slice();
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
|
||||
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
|
||||
jobs_executed,
|
||||
tags,
|
||||
&worker_name
|
||||
)
|
||||
.execute(db)
|
||||
@@ -1309,6 +1264,25 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
|
||||
tracing::error!(job_id = %job.id, "error handling job: {err:?} {} {} {}", job.id, job.workspace_id, job.created_by);
|
||||
}
|
||||
|
||||
async fn insert_initial_ping(
|
||||
worker_instance: &str,
|
||||
worker_name: &str,
|
||||
ip: &str,
|
||||
db: &Pool<Postgres>,
|
||||
) {
|
||||
let tags = ACCEPTED_TAGS.clone();
|
||||
sqlx::query!(
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags) VALUES ($1, $2, $3, $4) ON CONFLICT (worker) DO NOTHING",
|
||||
worker_instance,
|
||||
worker_name,
|
||||
ip,
|
||||
if *IS_WORKER_TAGS_DEFINED { Some(tags.as_slice()) } else { None }
|
||||
)
|
||||
.execute(db)
|
||||
.await
|
||||
.expect("insert worker_ping initial value");
|
||||
}
|
||||
|
||||
fn extract_error_value(log_lines: &str, i: i32) -> serde_json::Value {
|
||||
return json!({"message": format!("ExitCode: {i}, last log lines:\n{}", ANSI_ESCAPE_RE.replace_all(log_lines.trim(), "").to_string()), "name": "ExecutionErr"});
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ async function login(email: string, password: string): Promise<string> {
|
||||
});
|
||||
}
|
||||
|
||||
export const VERSION = "v1.173.0";
|
||||
export const VERSION = "v1.168.3";
|
||||
|
||||
export async function main({
|
||||
host,
|
||||
@@ -242,28 +242,20 @@ export async function main({
|
||||
const jobsSent = Array(num_workers).fill(0);
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
|
||||
async function getQueueCount() {
|
||||
return (
|
||||
await (
|
||||
await fetch(
|
||||
config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count",
|
||||
{ headers: { ["Authorization"]: "Bearer " + config.token } }
|
||||
)
|
||||
).json()
|
||||
).database_length;
|
||||
}
|
||||
|
||||
const initial_queue_length = await getQueueCount();
|
||||
|
||||
console.log("Initial queue length:", initial_queue_length);
|
||||
|
||||
const updateState = setInterval(async () => {
|
||||
const elapsed = start ? Math.ceil((Date.now() - start) / 1000) : 0;
|
||||
const sum = jobsSent.reduce((a, b) => a + b, 0);
|
||||
let queue_length = -1;
|
||||
while (queue_length === -1) {
|
||||
try {
|
||||
queue_length = await getQueueCount();
|
||||
queue_length = (
|
||||
await (
|
||||
await fetch(
|
||||
host + "/api/w/" + config.workspace_id + "/jobs/queue/count",
|
||||
{ headers: { ["Authorization"]: "Bearer " + config.token } }
|
||||
)
|
||||
).json()
|
||||
).database_length;
|
||||
} catch (e) {
|
||||
console.log(
|
||||
`queue count not reachable. waiting... `
|
||||
@@ -276,9 +268,7 @@ export async function main({
|
||||
enc(
|
||||
`elapsed: ${elapsed}/${seconds} | jobs sent: ${JSON.stringify(
|
||||
jobsSent
|
||||
)} (sum: ${sum} thr: ${(sum / elapsed).toFixed(2)}) - processed (sum: ${
|
||||
sum - queue_length
|
||||
} thr: ${((sum - queue_length) / elapsed).toFixed(
|
||||
)} (sum: ${sum} thr: ${(sum / elapsed).toFixed(
|
||||
2
|
||||
)}) | queue: ${queue_length} \r`
|
||||
)
|
||||
|
||||
@@ -33,10 +33,7 @@ export async function pushApp(
|
||||
if (raw) {
|
||||
// deleting old app if it exists in raw mode
|
||||
try {
|
||||
app = await AppService.getAppByPath({
|
||||
workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
});
|
||||
app = await AppService.getAppByPath({ workspace, path: remotePath });
|
||||
} catch {
|
||||
//ignore
|
||||
}
|
||||
@@ -48,7 +45,7 @@ export async function pushApp(
|
||||
}
|
||||
await AppService.updateApp({
|
||||
workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
requestBody: {
|
||||
...newApp,
|
||||
},
|
||||
@@ -59,7 +56,7 @@ export async function pushApp(
|
||||
await AppService.createApp({
|
||||
workspace,
|
||||
requestBody: {
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
...newApp,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// windmill
|
||||
export { setClient } from "https://deno.land/x/windmill@v1.95.1/mod.ts";
|
||||
export * from "https://deno.land/x/windmill@v1.95.1/windmill-api/index.ts";
|
||||
export { SEP } from "https://deno.land/std@0.201.0/path/separator.ts";
|
||||
|
||||
// cliffy
|
||||
export { Command } from "https://deno.land/x/cliffy@v1.0.0-rc.2/command/mod.ts";
|
||||
export { Table } from "https://deno.land/x/cliffy@v1.0.0-rc.2/table/table.ts";
|
||||
|
||||
12
cli/flow.ts
12
cli/flow.ts
@@ -1,6 +1,6 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { GlobalOptions, isSuperset } from "./types.ts";
|
||||
import { SEP, log } from "./deps.ts";
|
||||
import { log } from "./deps.ts";
|
||||
import {
|
||||
colors,
|
||||
Command,
|
||||
@@ -42,8 +42,8 @@ export async function pushFlow(
|
||||
// flow doesn't exist
|
||||
}
|
||||
|
||||
if (!localFlowPath.endsWith(SEP)) {
|
||||
localFlowPath += SEP;
|
||||
if (!localFlowPath.endsWith("/")) {
|
||||
localFlowPath += "/";
|
||||
}
|
||||
const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
|
||||
const localFlow = yamlParse(localFlowRaw) as FlowFile;
|
||||
@@ -74,9 +74,9 @@ export async function pushFlow(
|
||||
log.info(colors.bold.yellow(`Updating flow ${remotePath}...`));
|
||||
await FlowService.updateFlow({
|
||||
workspace: workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
requestBody: {
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
...localFlow,
|
||||
},
|
||||
});
|
||||
@@ -85,7 +85,7 @@ export async function pushFlow(
|
||||
await FlowService.createFlow({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
...localFlow,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import {
|
||||
colors,
|
||||
Command,
|
||||
Folder,
|
||||
FolderService,
|
||||
log,
|
||||
SEP,
|
||||
Table,
|
||||
} from "./deps.ts";
|
||||
import { colors, Command, Folder, FolderService, log, Table } from "./deps.ts";
|
||||
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
|
||||
import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
|
||||
|
||||
@@ -46,13 +38,13 @@ export async function pushFolder(
|
||||
localFolder: FolderFile,
|
||||
raw: boolean
|
||||
): Promise<void> {
|
||||
if (name.startsWith(SEP)) {
|
||||
if (name.startsWith("/")) {
|
||||
name = name.substring(1);
|
||||
}
|
||||
if (name.startsWith("f" + SEP)) {
|
||||
if (name.startsWith("f/")) {
|
||||
name = name.substring(2);
|
||||
}
|
||||
name = name.split(SEP)[0];
|
||||
name = name.split("/")[0];
|
||||
log.debug(`Processing local folder ${name}`);
|
||||
|
||||
if (raw) {
|
||||
|
||||
@@ -31,7 +31,7 @@ addEventListener("error", (event) => {
|
||||
}
|
||||
});
|
||||
|
||||
export const VERSION = "v1.173.0";
|
||||
export const VERSION = "v1.168.3";
|
||||
|
||||
let command: any = new Command()
|
||||
.name("wmill")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { GlobalOptions } from "./types.ts";
|
||||
import { colors, Command, JSZip, log } from "./deps.ts";
|
||||
import { colors, Command, JSZip } from "./deps.ts";
|
||||
import { Workspace } from "./workspace.ts";
|
||||
import { getHeaders } from "./utils.ts";
|
||||
|
||||
@@ -45,8 +45,6 @@ export async function downloadZip(
|
||||
colors.red("Failed to request tarball from API " + zipResponse.statusText)
|
||||
);
|
||||
throw new Error(await zipResponse.text());
|
||||
} else {
|
||||
log.debug(`Downloaded zip/tarball successfully`);
|
||||
}
|
||||
const blob = await zipResponse.blob();
|
||||
return await JSZip.loadAsync(blob as any);
|
||||
|
||||
@@ -34,7 +34,7 @@ export async function pushResource(
|
||||
try {
|
||||
resource = await ResourceService.getResource({
|
||||
workspace: workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
});
|
||||
} catch {
|
||||
// flow doesn't exist
|
||||
@@ -47,7 +47,7 @@ export async function pushResource(
|
||||
|
||||
await ResourceService.updateResource({
|
||||
workspace: workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
requestBody: { ...localResource },
|
||||
});
|
||||
} else {
|
||||
@@ -63,7 +63,7 @@ export async function pushResource(
|
||||
await ResourceService.createResource({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
...localResource,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -83,9 +83,7 @@ export async function handleFile(
|
||||
log.debug(`Processing local script ${path}`);
|
||||
|
||||
alreadySynced.push(path);
|
||||
const remotePath = path
|
||||
.substring(0, path.indexOf("."))
|
||||
.replaceAll("\\", "/");
|
||||
const remotePath = path.substring(0, path.indexOf("."));
|
||||
const metaPath = remotePath + ".script.json";
|
||||
let typed = undefined;
|
||||
try {
|
||||
@@ -107,7 +105,7 @@ export async function handleFile(
|
||||
try {
|
||||
remote = await ScriptService.getScriptByPath({
|
||||
workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
});
|
||||
log.debug(`Script ${remotePath} exists on remote`);
|
||||
} catch {
|
||||
@@ -144,7 +142,7 @@ export async function handleFile(
|
||||
content,
|
||||
description: typed?.description ?? "",
|
||||
language: language as NewScript.language,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
summary: typed?.summary ?? "",
|
||||
is_template: typed?.is_template,
|
||||
kind: typed?.kind,
|
||||
@@ -164,7 +162,7 @@ export async function handleFile(
|
||||
content,
|
||||
description: typed?.description ?? "",
|
||||
language: language as NewScript.language,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
summary: typed?.summary ?? "",
|
||||
is_template: typed?.is_template,
|
||||
kind: typed?.kind,
|
||||
|
||||
21
cli/sync.ts
21
cli/sync.ts
@@ -20,7 +20,6 @@ import {
|
||||
yamlStringify,
|
||||
yamlParse,
|
||||
ScheduleService,
|
||||
SEP,
|
||||
} from "./deps.ts";
|
||||
import {
|
||||
getTypeStrFromPath,
|
||||
@@ -238,7 +237,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
|
||||
},
|
||||
};
|
||||
}
|
||||
return _internal_folder("." + SEP, zip);
|
||||
return _internal_folder("./", zip);
|
||||
}
|
||||
|
||||
async function* readDirRecursiveWithIgnore(
|
||||
@@ -354,27 +353,19 @@ async function compareDynFSElement(
|
||||
}
|
||||
|
||||
const isNotWmillFile = (p: string, isDirectory: boolean) => {
|
||||
if (p.endsWith(SEP)) {
|
||||
if (p.endsWith("/")) {
|
||||
return false;
|
||||
}
|
||||
if (isDirectory) {
|
||||
return (
|
||||
!p.startsWith("u" + SEP) &&
|
||||
!p.startsWith("f" + SEP) &&
|
||||
!p.startsWith("g" + SEP)
|
||||
);
|
||||
return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/");
|
||||
}
|
||||
|
||||
try {
|
||||
const typ = getTypeStrFromPath(p);
|
||||
if (typ == "resource-type") {
|
||||
return p.includes(SEP);
|
||||
return p.includes("/");
|
||||
} else {
|
||||
return (
|
||||
!p.startsWith("u" + SEP) &&
|
||||
!p.startsWith("f" + SEP) &&
|
||||
!p.startsWith("g" + SEP)
|
||||
);
|
||||
return !p.startsWith("u/") && !p.startsWith("f/") && !p.startsWith("g/");
|
||||
}
|
||||
} catch {
|
||||
return true;
|
||||
@@ -382,7 +373,7 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
|
||||
};
|
||||
|
||||
export const isWhitelisted = (p: string) => {
|
||||
return p == "." + SEP || p == "" || p == "u" || p == "f" || p == "g";
|
||||
return p == "./" || p == "" || p == "u" || p == "f" || p == "g";
|
||||
};
|
||||
export async function ignoreF() {
|
||||
try {
|
||||
|
||||
@@ -62,7 +62,7 @@ export async function pushVariable(
|
||||
try {
|
||||
variable = await VariableService.getVariable({
|
||||
workspace: workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
decryptSecret: plainSecrets,
|
||||
});
|
||||
log.debug(`Variable ${remotePath} exists on remote`);
|
||||
@@ -80,7 +80,7 @@ export async function pushVariable(
|
||||
|
||||
await VariableService.updateVariable({
|
||||
workspace,
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
alreadyEncrypted: !plainSecrets,
|
||||
requestBody: {
|
||||
...localVariable,
|
||||
@@ -94,7 +94,7 @@ export async function pushVariable(
|
||||
workspace,
|
||||
alreadyEncrypted: !plainSecrets,
|
||||
requestBody: {
|
||||
path: remotePath.replaceAll("\\", "/"),
|
||||
path: remotePath,
|
||||
...localVariable,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -95,7 +95,6 @@ services:
|
||||
- METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001)
|
||||
- NUM_WORKERS=4
|
||||
- WORKER_TAGS=nativets,postgresql,mysql,graphql,snowflake
|
||||
- WORKER_GROUP=native
|
||||
# LICENSE_KEY is only needed for the enterprise edition
|
||||
# - LICENSE_KEY=${WM_LICENSE_KEY}
|
||||
depends_on:
|
||||
|
||||
811
frontend/package-lock.json
generated
811
frontend/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "windmill-components",
|
||||
"version": "1.173.0",
|
||||
"version": "1.168.3",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"build": "vite build",
|
||||
@@ -17,11 +17,12 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@floating-ui/core": "^1.3.1",
|
||||
"@modyfi/vite-plugin-yaml": "^1.0.4",
|
||||
"@playwright/test": "^1.34.3",
|
||||
"@rgossiaux/svelte-headlessui": "^2.0.0",
|
||||
"@sveltejs/adapter-static": "^2.0.3",
|
||||
"@sveltejs/kit": "^1.25.0",
|
||||
"@sveltejs/package": "^2.2.2",
|
||||
"@rgossiaux/svelte-headlessui": "^1.0.2",
|
||||
"@sveltejs/adapter-static": "^2.0.2",
|
||||
"@sveltejs/kit": "^1.20.1",
|
||||
"@sveltejs/package": "^2.2.0",
|
||||
"@tailwindcss/forms": "^0.5.3",
|
||||
"@tailwindcss/typography": "^0.5.8",
|
||||
"@types/d3": "^7.4.0",
|
||||
@@ -37,7 +38,7 @@
|
||||
"d3-dag": "^0.11.5",
|
||||
"eslint": "^8.47.0",
|
||||
"eslint-config-prettier": "^8.6.0",
|
||||
"eslint-plugin-svelte": "^2.33.1",
|
||||
"eslint-plugin-svelte3": "^4.0.0",
|
||||
"monaco-editor-workers": "~0.39.1",
|
||||
"ol": "^7.4.0",
|
||||
"openapi-typescript-codegen": "^0.25.0",
|
||||
@@ -50,7 +51,7 @@
|
||||
"simple-svelte-autocomplete": "^2.5.1",
|
||||
"style-to-object": "^0.4.1",
|
||||
"stylelint-config-recommended": "^13.0.0",
|
||||
"svelte": "^4.0.0",
|
||||
"svelte": "^3.59.2",
|
||||
"svelte-awesome": "^3.2.1",
|
||||
"svelte-awesome-color-picker": "^2.4.5",
|
||||
"svelte-check": "^3.4.3",
|
||||
@@ -61,7 +62,7 @@
|
||||
"svelte-popperjs": "^1.3.2",
|
||||
"svelte-preprocess": "^5.0.1",
|
||||
"svelte-range-slider-pips": "^2.1.1",
|
||||
"svelte-splitpanes": "^0.8.0",
|
||||
"svelte-splitpanes": "^0.7.14",
|
||||
"svelte2tsx": "^0.6.16",
|
||||
"tailwindcss": "^3.3.2",
|
||||
"tslib": "^2.6.1",
|
||||
@@ -70,20 +71,6 @@
|
||||
"vite-plugin-monaco-editor": "^1.1.0",
|
||||
"yootils": "^0.3.1"
|
||||
},
|
||||
"overrides": {
|
||||
"@rgossiaux/svelte-headlessui": {
|
||||
"svelte": "$svelte"
|
||||
},
|
||||
"ag-grid-svelte": {
|
||||
"svelte": "$svelte"
|
||||
},
|
||||
"svelte-chartjs": {
|
||||
"svelte": "$svelte"
|
||||
},
|
||||
"svelte-timezone-picker": {
|
||||
"svelte": "$svelte"
|
||||
}
|
||||
},
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"@aws-crypto/sha256-js": "^4.0.0",
|
||||
@@ -92,7 +79,7 @@
|
||||
"@leeoniya/ufuzzy": "^1.0.8",
|
||||
"@popperjs/core": "^2.11.6",
|
||||
"@redocly/json-to-json-schema": "^0.0.1",
|
||||
"@tanstack/svelte-table": "^8.9.9",
|
||||
"@tanstack/svelte-table": "^8.9.2",
|
||||
"ag-grid-svelte": "^0.3.0",
|
||||
"ansi_up": "^5.2.1",
|
||||
"chart.js": "^4.3.0",
|
||||
@@ -106,14 +93,14 @@
|
||||
"graphql": "^16.7.1",
|
||||
"highlight.js": "^11.8.0",
|
||||
"lodash": "^4.17.21",
|
||||
"lucide-svelte": "^0.277.0",
|
||||
"lucide-svelte": "^0.246.0",
|
||||
"monaco-graphql": "^1.3.0",
|
||||
"monaco-languageclient": "~6.0.3",
|
||||
"openai": "^4.3.0",
|
||||
"openai": "^4.0.0-beta.12",
|
||||
"quill": "^1.3.7",
|
||||
"svelte-autosize": "^1.0.1",
|
||||
"svelte-carousel": "^1.0.25",
|
||||
"svelte-chartjs": "^3.1.2",
|
||||
"svelte-chartjs": "^3.1.0",
|
||||
"svelte-dnd-action": "^0.9.22",
|
||||
"svelte-exmarkdown": "^2.1.0",
|
||||
"svelte-infinite-loading": "^1.3.8",
|
||||
@@ -128,8 +115,8 @@
|
||||
"yjs": "^13.6.7"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@sveltejs/kit": "^1.25.0",
|
||||
"svelte": "^4.0.0"
|
||||
"@sveltejs/kit": "^1.20.1",
|
||||
"svelte": "^3.59.2"
|
||||
},
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
|
||||
@@ -369,7 +369,7 @@
|
||||
size="sm"
|
||||
variant="border"
|
||||
color={key === resource_type ? 'blue' : 'light'}
|
||||
btnClasses={key === resource_type ? '!border-2' : 'm-[1px]'}
|
||||
btnClasses={key === resource_type ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
|
||||
on:click={() => {
|
||||
manual = false
|
||||
resource_type = key
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { JobService, Preview } from '$lib/gen'
|
||||
import {
|
||||
dbSchemas,
|
||||
workspaceStore,
|
||||
type DBSchema,
|
||||
type GraphqlSchema,
|
||||
type SQLSchema
|
||||
} from '$lib/stores'
|
||||
import { dbSchemas, workspaceStore, type DBSchema, type GraphqlSchema } from '$lib/stores'
|
||||
import Button from './common/button/Button.svelte'
|
||||
import Drawer from './common/drawer/Drawer.svelte'
|
||||
import DrawerContent from './common/drawer/DrawerContent.svelte'
|
||||
@@ -186,51 +180,6 @@ GROUP BY
|
||||
cols.append(col)
|
||||
schema[dataset.dataset_id][row[0]] = cols
|
||||
return schema
|
||||
`,
|
||||
lang: 'python3'
|
||||
},
|
||||
snowflake: {
|
||||
code: `# requirements:
|
||||
# snowflake-connector-python==3.2.0
|
||||
from typing import Any
|
||||
import snowflake.connector as sf
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
def main(args):
|
||||
if not args["database"]:
|
||||
raise Exception("a selected database is required for the schema explorer")
|
||||
p_key = serialization.load_pem_private_key(
|
||||
args["private_key"].encode(), password=None, backend=default_backend()
|
||||
)
|
||||
pkb = p_key.private_bytes(
|
||||
encoding=serialization.Encoding.DER,
|
||||
format=serialization.PrivateFormat.PKCS8,
|
||||
encryption_algorithm=serialization.NoEncryption(),
|
||||
)
|
||||
ctx = sf.connect(
|
||||
user=args["username"],
|
||||
account=args["account_identifier"],
|
||||
private_key=pkb,
|
||||
warehouse=args["warehouse"],
|
||||
database=args["database"],
|
||||
schema=args["schema"],
|
||||
role=args["role"],
|
||||
)
|
||||
cs = ctx.cursor()
|
||||
rows = cs.execute("select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE from information_schema.columns where table_schema != 'INFORMATION_SCHEMA'")
|
||||
schema = dict()
|
||||
for row in rows:
|
||||
if row[0] not in schema:
|
||||
schema[row[0]] = dict()
|
||||
if row[1] not in schema[row[0]]:
|
||||
schema[row[0]][row[1]] = dict()
|
||||
schema[row[0]][row[1]][row[3]] = {
|
||||
"type": row[2],
|
||||
"required": row[5] == "YES",
|
||||
}
|
||||
if row[4] is not None:
|
||||
schema[row[0]][row[1]][row[3]]["default"] = row[4]
|
||||
return schema
|
||||
`,
|
||||
lang: 'python3'
|
||||
}
|
||||
@@ -261,18 +210,19 @@ def main(args):
|
||||
if (!testResult.success) {
|
||||
console.error(testResult.result?.['error']?.['message'])
|
||||
} else {
|
||||
if (resourceType !== undefined) {
|
||||
if (resourceType !== 'graphql') {
|
||||
$dbSchemas[resourcePath] = {
|
||||
lang: resourceType as SQLSchema['lang'],
|
||||
schema: testResult.result,
|
||||
publicOnly: !!testResult.result.public || !!testResult.result.PUBLIC
|
||||
}
|
||||
} else {
|
||||
$dbSchemas[resourcePath] = {
|
||||
lang: 'graphql',
|
||||
schema: testResult.result
|
||||
}
|
||||
if (resourceType === 'postgresql') {
|
||||
$dbSchemas[resourcePath] = {
|
||||
lang: 'postgresql',
|
||||
schema: testResult.result,
|
||||
publicOnly: true
|
||||
}
|
||||
} else if (
|
||||
resourceType !== undefined &&
|
||||
['mysql', 'graphql', 'bigquery'].includes(resourceType)
|
||||
) {
|
||||
$dbSchemas[resourcePath] = {
|
||||
lang: resourceType as 'mysql' | 'graphql' | 'bigquery',
|
||||
schema: testResult.result
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -303,8 +253,8 @@ def main(args):
|
||||
}
|
||||
|
||||
function formatSchema(dbSchema: DBSchema) {
|
||||
if (dbSchema.lang !== 'graphql' && dbSchema.publicOnly) {
|
||||
return dbSchema.schema.public || dbSchema.schema.PUBLIC || dbSchema
|
||||
if (dbSchema.lang === 'postgresql' && dbSchema.publicOnly) {
|
||||
return dbSchema.schema.public || dbSchema
|
||||
} else if (dbSchema.lang === 'mysql' && Object.keys(dbSchema.schema).length === 1) {
|
||||
return dbSchema.schema[Object.keys(dbSchema.schema)[0]]
|
||||
} else {
|
||||
@@ -349,7 +299,7 @@ def main(args):
|
||||
>Refresh
|
||||
</Button>
|
||||
</svelte:fragment>
|
||||
{#if dbSchema.lang !== 'graphql' && (dbSchema.schema?.public || dbSchema.schema?.PUBLIC)}
|
||||
{#if dbSchema.lang === 'postgresql'}
|
||||
<ToggleButtonGroup class="mb-4" bind:selected={dbSchema.publicOnly}>
|
||||
<ToggleButton value={true} label="Public" />
|
||||
<ToggleButton value={false} label="All" />
|
||||
|
||||
@@ -70,7 +70,6 @@
|
||||
export let yContent: Text | undefined = undefined
|
||||
export let awareness: any | undefined = undefined
|
||||
export let folding = false
|
||||
export let args: Record<string, any> | undefined = undefined
|
||||
|
||||
languages.typescript.typescriptDefaults.setModeConfiguration({
|
||||
completionItems: false,
|
||||
@@ -189,27 +188,6 @@
|
||||
}
|
||||
}
|
||||
|
||||
export function append(code): void {
|
||||
if (editor) {
|
||||
const lineCount = editor.getModel()?.getLineCount() || 0
|
||||
const lastLineLength = editor.getModel()?.getLineLength(lineCount) || 0
|
||||
const range: IRange = {
|
||||
startLineNumber: lineCount,
|
||||
startColumn: lastLineLength + 1,
|
||||
endLineNumber: lineCount,
|
||||
endColumn: lastLineLength + 1
|
||||
}
|
||||
editor.executeEdits('append', [
|
||||
{
|
||||
range,
|
||||
text: code,
|
||||
forceMoveMarkers: true
|
||||
}
|
||||
])
|
||||
editor.revealLine(lineCount)
|
||||
}
|
||||
}
|
||||
|
||||
export function format() {
|
||||
if (editor) {
|
||||
code = getCode()
|
||||
@@ -225,8 +203,7 @@
|
||||
let command: Disposable | undefined = undefined
|
||||
|
||||
let sqlSchemaCompletor: Disposable | undefined = undefined
|
||||
$: args &&
|
||||
(dbSchema = $dbSchemas[(lang === 'graphql' ? args.api : args.database)?.replace('$res:', '')])
|
||||
$: dbSchema = $dbSchemas[Object.keys($dbSchemas)[0]]
|
||||
$: dbSchema && ['sql', 'graphql'].includes(lang) && addDBSchemaCompletions()
|
||||
$: (!dbSchema || lang !== 'sql') && sqlSchemaCompletor && sqlSchemaCompletor.dispose()
|
||||
$: (!dbSchema || lang !== 'graphql') && graphqlService && graphqlService.setSchemaConfig([])
|
||||
@@ -244,7 +221,7 @@
|
||||
introspectionJSON: schema
|
||||
}
|
||||
])
|
||||
} else {
|
||||
} else if (schemaLang === 'mysql' || schemaLang === 'postgresql') {
|
||||
if (sqlSchemaCompletor) {
|
||||
sqlSchemaCompletor.dispose()
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
import { capitalize, toCamel } from '$lib/utils'
|
||||
import type { Schema, SchemaProperty, SupportedLanguage } from '$lib/common'
|
||||
import ScriptVersionHistory from './ScriptVersionHistory.svelte'
|
||||
import ScriptGen from './copilot/ScriptGen.svelte'
|
||||
import { ScriptGen } from './codeGen'
|
||||
import type DiffEditor from './DiffEditor.svelte'
|
||||
import { getResetCode } from '$lib/script_helpers'
|
||||
import type { Script } from '$lib/gen'
|
||||
@@ -62,7 +62,6 @@
|
||||
export let collabUsers: { name: string }[] = []
|
||||
export let scriptPath: string | undefined = undefined
|
||||
export let diffEditor: DiffEditor | undefined = undefined
|
||||
export let args: Record<string, any>
|
||||
|
||||
let contextualVariablePicker: ItemPicker
|
||||
let variablePicker: ItemPicker
|
||||
@@ -531,7 +530,7 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<ScriptGen {editor} {diffEditor} {lang} {iconOnly} {args} />
|
||||
<ScriptGen {editor} {diffEditor} {lang} {iconOnly} />
|
||||
|
||||
<!-- <Popover
|
||||
notClickable
|
||||
|
||||
@@ -5,24 +5,15 @@
|
||||
type Flow,
|
||||
type FlowModule,
|
||||
DraftService,
|
||||
type PathScript,
|
||||
ScriptService,
|
||||
Script
|
||||
type PathScript
|
||||
} from '$lib/gen'
|
||||
import { initHistory, push, redo, undo } from '$lib/history'
|
||||
import {
|
||||
enterpriseLicense,
|
||||
existsOpenaiResourcePath,
|
||||
hubScripts,
|
||||
userStore,
|
||||
workspaceStore
|
||||
} from '$lib/stores'
|
||||
import { encodeState, formatCron, sleep } from '$lib/utils'
|
||||
import { initHistory, redo, undo } from '$lib/history'
|
||||
import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores'
|
||||
import { encodeState, formatCron } from '$lib/utils'
|
||||
import { sendUserToast } from '$lib/toast'
|
||||
import type { Drawer } from '$lib/components/common'
|
||||
|
||||
import { faCalendarAlt, faSave } from '@fortawesome/free-solid-svg-icons'
|
||||
import { setContext, tick } from 'svelte'
|
||||
import { setContext } from 'svelte'
|
||||
import { writable, type Writable } from 'svelte/store'
|
||||
import CenteredPage from './CenteredPage.svelte'
|
||||
import { Badge, Button, Kbd, UndoRedo } from './common'
|
||||
@@ -30,30 +21,17 @@
|
||||
import FlowEditor from './flows/FlowEditor.svelte'
|
||||
import ScriptEditorDrawer from './flows/content/ScriptEditorDrawer.svelte'
|
||||
import type { FlowState } from './flows/flowState'
|
||||
import { dfs as dfsApply } from './flows/flowStore'
|
||||
import { dfs, getPreviousIds } from './flows/previousResults'
|
||||
import { dfs } from './flows/flowStore'
|
||||
import FlowImportExportMenu from './flows/header/FlowImportExportMenu.svelte'
|
||||
import FlowPreviewButtons from './flows/header/FlowPreviewButtons.svelte'
|
||||
import { loadFlowSchedule, type Schedule } from './flows/scheduleUtils'
|
||||
import type { FlowEditorContext } from './flows/types'
|
||||
import { cleanInputs, emptyFlowModuleState } from './flows/utils'
|
||||
import { cleanInputs } from './flows/utils'
|
||||
import { Pen } from 'lucide-svelte'
|
||||
import { loadHubScripts } from '$lib/scripts'
|
||||
import { createEventDispatcher } from 'svelte'
|
||||
import Awareness from './Awareness.svelte'
|
||||
import { getAllModules } from './flows/flowExplorer'
|
||||
import {
|
||||
stepCopilot,
|
||||
type FlowCopilotModule,
|
||||
glueCopilot,
|
||||
type FlowCopilotContext
|
||||
} from './copilot/flow'
|
||||
import type { Schema, SchemaProperty } from '$lib/common'
|
||||
import FlowCopilotDrawer from './copilot/FlowCopilotDrawer.svelte'
|
||||
import FlowCopilotStatus from './copilot/FlowCopilotStatus.svelte'
|
||||
import { fade } from 'svelte/transition'
|
||||
import { loadFlowModuleState } from './flows/flowStateUtils'
|
||||
import FlowCopilotInputsModal from './copilot/FlowCopilotInputsModal.svelte'
|
||||
|
||||
export let initialPath: string = ''
|
||||
export let selectedId: string | undefined
|
||||
@@ -350,7 +328,7 @@
|
||||
return [
|
||||
'settings-metadata',
|
||||
'constants',
|
||||
...dfsApply($flowStore.value.modules, (module) => module.id)
|
||||
...dfs($flowStore.value.modules, (module) => module.id)
|
||||
]
|
||||
}
|
||||
|
||||
@@ -370,500 +348,18 @@
|
||||
onClick: () => window.open(`/flows/add?template=${initialPath}`)
|
||||
})
|
||||
}
|
||||
|
||||
let flowCopilotContext: FlowCopilotContext = {
|
||||
drawerStore: writable<Drawer | undefined>(undefined),
|
||||
modulesStore: writable<FlowCopilotModule[]>([]),
|
||||
currentStepStore: writable<string | undefined>(undefined),
|
||||
genFlow: undefined
|
||||
}
|
||||
|
||||
setContext('FlowCopilotContext', flowCopilotContext)
|
||||
|
||||
const {
|
||||
drawerStore: copilotDrawerStore,
|
||||
modulesStore: copilotModulesStore,
|
||||
currentStepStore: copilotCurrentStepStore
|
||||
} = flowCopilotContext
|
||||
|
||||
let doneTs = 0
|
||||
async function getHubCompletions(text: string, idx: number, type: 'trigger' | 'script') {
|
||||
try {
|
||||
// make sure we display the results of the last request last
|
||||
const ts = Date.now()
|
||||
const scriptIds = await ScriptService.queryHubScripts({
|
||||
text: `${text}`,
|
||||
limit: 3,
|
||||
kind: type
|
||||
})
|
||||
if (ts < doneTs) return
|
||||
doneTs = ts
|
||||
|
||||
const scripts = scriptIds
|
||||
.map((qs) => {
|
||||
const s = $hubScripts?.find((hs) => hs.ask_id === Number(qs.id))
|
||||
return s
|
||||
})
|
||||
.filter((s) => !!s)
|
||||
|
||||
$copilotModulesStore[idx].hubCompletions = scripts as {
|
||||
path: string
|
||||
summary: string
|
||||
approved: boolean
|
||||
kind: string
|
||||
app: string
|
||||
ask_id: number
|
||||
}[]
|
||||
} catch (err) {
|
||||
if (err.name !== 'CancelError') throw err
|
||||
}
|
||||
}
|
||||
|
||||
let abortController: AbortController | undefined = undefined
|
||||
let copilotLoading = false
|
||||
let flowCopilotMode: 'trigger' | 'sequence' = 'trigger'
|
||||
let copilotStatus: string = ''
|
||||
let copilotFlowInputs: Record<string, SchemaProperty> = {}
|
||||
let copilotFlowRequiredInputs: string[] = []
|
||||
let openCopilotInputsModal = false
|
||||
|
||||
function setInitCopilotModules(mode: typeof flowCopilotMode) {
|
||||
$copilotModulesStore = [
|
||||
{
|
||||
id: 'a',
|
||||
type: mode === 'trigger' ? 'trigger' : 'script',
|
||||
description: '',
|
||||
code: '',
|
||||
hubCompletions: [],
|
||||
selectedCompletion: undefined,
|
||||
source: undefined,
|
||||
lang: undefined
|
||||
},
|
||||
{
|
||||
id: 'b',
|
||||
type: 'script',
|
||||
description: '',
|
||||
code: '',
|
||||
hubCompletions: [],
|
||||
selectedCompletion: undefined,
|
||||
source: undefined,
|
||||
lang: undefined
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
$: setInitCopilotModules(flowCopilotMode)
|
||||
|
||||
function applyCopilotFlowInputs() {
|
||||
const properties = {
|
||||
...($flowStore.schema?.properties as Record<string, SchemaProperty> | undefined),
|
||||
...copilotFlowInputs
|
||||
}
|
||||
const required = [
|
||||
...(($flowStore.schema?.required as string[] | undefined) ?? []),
|
||||
...copilotFlowRequiredInputs
|
||||
]
|
||||
$flowStore.schema = {
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
properties,
|
||||
required,
|
||||
type: 'object'
|
||||
}
|
||||
}
|
||||
|
||||
function clearFlowInputsFromStep(id: string | undefined) {
|
||||
const module: FlowModule | undefined = dfs(id, $flowStore)[0]
|
||||
if (module?.value.type === 'rawscript') {
|
||||
// clear step inputs that start with flow_input. but not flow_input.iter
|
||||
for (const key in module.value.input_transforms) {
|
||||
const input = module.value.input_transforms[key]
|
||||
if (
|
||||
input.type === 'javascript' &&
|
||||
input.expr.includes('flow_input.') &&
|
||||
!input.expr.includes('flow_input.iter')
|
||||
) {
|
||||
module.value.input_transforms[key] = {
|
||||
type: 'static',
|
||||
value: undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
$flowStore = $flowStore
|
||||
}
|
||||
|
||||
async function finishStepGen() {
|
||||
copilotFlowInputs = {}
|
||||
copilotFlowRequiredInputs = []
|
||||
setInitCopilotModules(flowCopilotMode)
|
||||
copilotStatus = "Done! Just check the step's inputs and you're good to go!"
|
||||
await sleep(3000)
|
||||
copilotStatus = ''
|
||||
}
|
||||
|
||||
async function genFlow(idx: number, flowModules: FlowModule[], stepOnly = false) {
|
||||
try {
|
||||
push(history, $flowStore)
|
||||
let module = stepOnly ? $copilotModulesStore[0] : $copilotModulesStore[idx]
|
||||
|
||||
copilotLoading = true
|
||||
copilotStatus = "Generating code for step '" + module.id + "'..."
|
||||
$copilotCurrentStepStore = module.id
|
||||
focusCopilot()
|
||||
|
||||
if (!stepOnly && flowModules.length > idx) {
|
||||
select('')
|
||||
await tick()
|
||||
flowModules.splice(idx, flowModules.length - idx)
|
||||
$flowStore = $flowStore
|
||||
focusCopilot()
|
||||
}
|
||||
|
||||
if (idx === 0 && !stepOnly) {
|
||||
$flowStore.schema = {
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
properties: {},
|
||||
required: [],
|
||||
type: 'object'
|
||||
}
|
||||
}
|
||||
|
||||
if (module.type === 'trigger') {
|
||||
if (!$scheduleStore.cron) {
|
||||
$scheduleStore.cron = '0 */15 * * *'
|
||||
}
|
||||
$scheduleStore.enabled = true
|
||||
}
|
||||
|
||||
let hubScript:
|
||||
| {
|
||||
content: string
|
||||
lockfile?: string | undefined
|
||||
schema?: any
|
||||
language: string
|
||||
summary?: string | undefined
|
||||
}
|
||||
| undefined = undefined
|
||||
|
||||
if (module.source === 'hub' && module.selectedCompletion) {
|
||||
hubScript = await ScriptService.getHubScriptByPath({
|
||||
path: module.selectedCompletion.path
|
||||
})
|
||||
}
|
||||
|
||||
const flowModule = {
|
||||
id: module.id,
|
||||
stop_after_if:
|
||||
module.type === 'trigger'
|
||||
? {
|
||||
expr: 'result == undefined || Array.isArray(result) && result.length == 0',
|
||||
skip_if_stopped: true
|
||||
}
|
||||
: undefined,
|
||||
value: {
|
||||
input_transforms: {},
|
||||
content: '',
|
||||
language: (hubScript ? hubScript.language : module.lang ?? 'bun') as Script.language,
|
||||
type: 'rawscript' as const
|
||||
},
|
||||
summary: module.selectedCompletion?.summary ?? module.description
|
||||
}
|
||||
|
||||
$flowStateStore[module.id] = emptyFlowModuleState()
|
||||
if (stepOnly) {
|
||||
flowModules.splice(idx, 0, flowModule)
|
||||
} else if (idx === 1 && $copilotModulesStore[idx - 1].type === 'trigger') {
|
||||
const loopModule: FlowModule = {
|
||||
id: module.id + '_loop',
|
||||
value: {
|
||||
type: 'forloopflow',
|
||||
iterator: {
|
||||
type: 'javascript',
|
||||
expr: 'results.a'
|
||||
},
|
||||
skip_failures: true,
|
||||
modules: [flowModule]
|
||||
}
|
||||
}
|
||||
const loopState = await loadFlowModuleState(loopModule)
|
||||
$flowStateStore[loopModule.id] = loopState
|
||||
flowModules.push(loopModule)
|
||||
} else {
|
||||
flowModules.push(flowModule)
|
||||
}
|
||||
|
||||
$copilotDrawerStore?.closeDrawer()
|
||||
select(module.id)
|
||||
await tick()
|
||||
focusCopilot()
|
||||
|
||||
let isFirstInLoop = false
|
||||
const parents = dfs(module.id, $flowStore).slice(1)
|
||||
if (
|
||||
parents[0]?.value.type === 'forloopflow' &&
|
||||
parents[0].value.modules[0].id === module.id
|
||||
) {
|
||||
isFirstInLoop = true
|
||||
}
|
||||
const prevNodeId = getPreviousIds(module.id, $flowStore, false)[0]
|
||||
const pastModule: FlowModule | undefined = dfs(prevNodeId, $flowStore, false)[0]
|
||||
|
||||
if (hubScript) {
|
||||
module.editor?.setCode(hubScript.content)
|
||||
} else if (module.source === 'custom') {
|
||||
module.editor?.setCode('')
|
||||
const deltaStore = writable<string>('')
|
||||
const unsubscribe = deltaStore.subscribe(async (delta) => {
|
||||
module.editor?.append(delta)
|
||||
})
|
||||
|
||||
abortController = new AbortController()
|
||||
await stepCopilot(
|
||||
module,
|
||||
deltaStore,
|
||||
pastModule?.value.type === 'rawscript' ? pastModule.value.content : '',
|
||||
pastModule?.value.type === 'rawscript' ? pastModule.value.language : undefined,
|
||||
pastModule === undefined,
|
||||
isFirstInLoop,
|
||||
abortController
|
||||
)
|
||||
unsubscribe()
|
||||
} else {
|
||||
throw new Error('Invalid copilot module source')
|
||||
}
|
||||
|
||||
copilotStatus = "Generating inputs for step '" + module.id + "'..."
|
||||
await sleep(500) // make sure code was parsed
|
||||
|
||||
try {
|
||||
if (flowModule.value.type === 'rawscript') {
|
||||
const stepSchema: Schema = JSON.parse(JSON.stringify($flowStateStore[module.id].schema)) // deep copy
|
||||
if (module.source === 'hub' && pastModule !== undefined && $existsOpenaiResourcePath) {
|
||||
// ask AI to set step inputs
|
||||
abortController = new AbortController()
|
||||
const inputs = await glueCopilot(
|
||||
Object.keys(flowModule.value.input_transforms),
|
||||
pastModule.value.type === 'rawscript' ? pastModule.value.content : '',
|
||||
pastModule.value.type === 'rawscript' ? pastModule.value.language : undefined,
|
||||
isFirstInLoop,
|
||||
abortController
|
||||
)
|
||||
|
||||
// create flow inputs used by AI for autocompletion
|
||||
copilotFlowInputs = {}
|
||||
copilotFlowRequiredInputs = []
|
||||
Object.entries(inputs).forEach(([key, expr]) => {
|
||||
if (
|
||||
key in stepSchema.properties &&
|
||||
expr.includes('flow_input.') &&
|
||||
!expr.includes('flow_input.iter') &&
|
||||
(!$flowStore.schema || !(key in $flowStore.schema.properties)) // prevent overriding flow inputs
|
||||
) {
|
||||
copilotFlowInputs[key] = stepSchema.properties[key]
|
||||
if (stepSchema.required.includes(key)) {
|
||||
copilotFlowRequiredInputs.push(key)
|
||||
}
|
||||
}
|
||||
})
|
||||
if (!stepOnly) {
|
||||
applyCopilotFlowInputs()
|
||||
}
|
||||
|
||||
// set step inputs
|
||||
Object.entries(inputs).forEach(([key, expr]) => {
|
||||
flowModule.value.input_transforms[key] = {
|
||||
type: 'javascript',
|
||||
expr
|
||||
}
|
||||
})
|
||||
} else {
|
||||
if (module.source === 'hub' && pastModule !== undefined && !$existsOpenaiResourcePath) {
|
||||
sendUserToast(
|
||||
'For better input generation, enable Windmill AI in the workspace settings',
|
||||
true
|
||||
)
|
||||
}
|
||||
|
||||
// create possible flow inputs for autocompletion
|
||||
copilotFlowInputs = {}
|
||||
copilotFlowRequiredInputs = []
|
||||
Object.keys(flowModule.value.input_transforms).forEach((key) => {
|
||||
if (key !== 'prev_output') {
|
||||
const schema = $flowStateStore[module.id].schema
|
||||
const schemaProperty = Object.entries(schema.properties).find(
|
||||
(x) => x[0] === key
|
||||
)?.[1]
|
||||
if (
|
||||
schemaProperty &&
|
||||
(!$flowStore.schema || !(key in $flowStore.schema.properties)) // prevent overriding flow inputs
|
||||
) {
|
||||
copilotFlowInputs[key] = schemaProperty
|
||||
if (schema.required.includes(key)) {
|
||||
copilotFlowRequiredInputs.push(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
if (!stepOnly) {
|
||||
applyCopilotFlowInputs()
|
||||
}
|
||||
|
||||
// programatically set step inputs
|
||||
for (const key of Object.keys(flowModule.value.input_transforms)) {
|
||||
flowModule.value.input_transforms[key] = {
|
||||
type: 'javascript',
|
||||
expr:
|
||||
key === 'prev_output'
|
||||
? isFirstInLoop
|
||||
? 'flow_input.iter.value'
|
||||
: pastModule
|
||||
? 'results.' + pastModule.id
|
||||
: 'flow_input.' + key
|
||||
: 'flow_input.' + key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$flowStore = $flowStore // force rerendering
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
}
|
||||
|
||||
if (stepOnly) {
|
||||
$copilotCurrentStepStore = undefined
|
||||
copilotLoading = false
|
||||
copilotStatus = ''
|
||||
if (Object.keys(copilotFlowInputs).length > 0) {
|
||||
openCopilotInputsModal = true
|
||||
} else {
|
||||
finishStepGen()
|
||||
}
|
||||
} else {
|
||||
copilotStatus =
|
||||
"Waiting for the user to validate code and inputs of step '" + module.id + "'"
|
||||
}
|
||||
} catch (err) {
|
||||
if (stepOnly) {
|
||||
copilotStatus = ''
|
||||
$copilotCurrentStepStore = undefined
|
||||
setInitCopilotModules(flowCopilotMode)
|
||||
}
|
||||
if (err?.message) {
|
||||
sendUserToast('Failed to generate code: ' + err.message, true)
|
||||
} else {
|
||||
sendUserToast('Failed to generate code', true)
|
||||
console.error(err)
|
||||
}
|
||||
} finally {
|
||||
copilotLoading = false
|
||||
}
|
||||
}
|
||||
|
||||
flowCopilotContext.genFlow = genFlow
|
||||
|
||||
async function finishCopilotFlowBuilder() {
|
||||
copilotLoading = true
|
||||
select('Input')
|
||||
$copilotCurrentStepStore = 'Input'
|
||||
copilotStatus = 'Setting flow inputs...'
|
||||
|
||||
// filter out unused flow inputs
|
||||
const flowInputs: Record<string, SchemaProperty> = {}
|
||||
const required = new Set<string>()
|
||||
function getFlowInputs(modules: FlowModule[]) {
|
||||
for (const module of modules) {
|
||||
if (module.value.type === 'rawscript') {
|
||||
for (const moduleAttr of Object.keys(module.value.input_transforms)) {
|
||||
const input = module.value.input_transforms[moduleAttr]
|
||||
if (
|
||||
input.type === 'javascript' &&
|
||||
input.expr.includes('flow_input.') &&
|
||||
!input.expr.includes('flow_input.iter')
|
||||
) {
|
||||
const flowAttr = input.expr.split('.')[1]
|
||||
const schema = $flowStateStore[module.id].schema
|
||||
const schemaProperty = Object.entries(schema.properties).find(
|
||||
(x) => x[0] === moduleAttr
|
||||
)?.[1]
|
||||
if (schemaProperty) {
|
||||
flowInputs[flowAttr] = schemaProperty
|
||||
required.add(flowAttr)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (module.value.type === 'forloopflow') {
|
||||
getFlowInputs(module.value.modules)
|
||||
}
|
||||
}
|
||||
}
|
||||
getFlowInputs($flowStore.value.modules)
|
||||
|
||||
$flowStore.schema = {
|
||||
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
||||
properties: flowInputs,
|
||||
required: Array.from(required),
|
||||
type: 'object'
|
||||
}
|
||||
|
||||
copilotStatus = "Done! Just check the flow's inputs and you're good to go!"
|
||||
$copilotCurrentStepStore = undefined
|
||||
copilotLoading = false
|
||||
await sleep(3000)
|
||||
copilotStatus = ''
|
||||
}
|
||||
|
||||
function focusCopilot() {
|
||||
document.querySelectorAll('.splitpanes__splitter').forEach((el) => {
|
||||
el.classList.add('hidden')
|
||||
})
|
||||
document.querySelectorAll('#flow-graph *').forEach((el) => {
|
||||
if (el instanceof HTMLElement) {
|
||||
el.style.pointerEvents = 'none'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function blurCopilot() {
|
||||
document.querySelectorAll('.splitpanes__splitter').forEach((el) => {
|
||||
el.classList.remove('hidden')
|
||||
})
|
||||
document.querySelectorAll('#flow-graph *').forEach((el) => {
|
||||
if (el instanceof HTMLElement) {
|
||||
el.style.pointerEvents = ''
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
$: $copilotCurrentStepStore === undefined && blurCopilot()
|
||||
</script>
|
||||
|
||||
<svelte:window on:keydown={onKeyDown} />
|
||||
|
||||
{#if !$userStore?.operator}
|
||||
<FlowCopilotDrawer {getHubCompletions} {genFlow} bind:flowCopilotMode />
|
||||
<FlowCopilotInputsModal
|
||||
on:confirmed={async () => {
|
||||
applyCopilotFlowInputs()
|
||||
finishStepGen()
|
||||
}}
|
||||
on:canceled={async () => {
|
||||
clearFlowInputsFromStep($copilotModulesStore[0]?.id)
|
||||
finishStepGen()
|
||||
}}
|
||||
bind:open={openCopilotInputsModal}
|
||||
inputs={Object.keys(copilotFlowInputs)}
|
||||
/>
|
||||
<ScriptEditorDrawer bind:this={$scriptEditorDrawer} />
|
||||
|
||||
<div class="flex flex-col flex-1 h-screen">
|
||||
<!-- Nav between steps-->
|
||||
<div
|
||||
class="justify-between flex flex-row items-center pl-2.5 pr-6 space-x-4 scrollbar-hidden max-h-12 h-full relative"
|
||||
class="justify-between flex flex-row items-center pl-2.5 pr-6 space-x-4 scrollbar-hidden max-h-12 h-full"
|
||||
>
|
||||
{#if $copilotCurrentStepStore !== undefined}
|
||||
<div transition:fade class="absolute inset-0 bg-gray-500 bg-opacity-75 z-[900] !m-0" />
|
||||
{/if}
|
||||
<div class="flex w-full max-w-md gap-4 items-center">
|
||||
<div class="min-w-64 w-full">
|
||||
<input
|
||||
@@ -933,14 +429,6 @@
|
||||
<Awareness />
|
||||
{/if}
|
||||
|
||||
<FlowCopilotStatus
|
||||
{copilotLoading}
|
||||
bind:copilotStatus
|
||||
{genFlow}
|
||||
{finishCopilotFlowBuilder}
|
||||
{abortController}
|
||||
/>
|
||||
|
||||
<FlowImportExportMenu />
|
||||
|
||||
<FlowPreviewButtons />
|
||||
|
||||
@@ -47,12 +47,6 @@
|
||||
|
||||
let propertyType = getPropertyType(arg)
|
||||
|
||||
function updatePropertyType() {
|
||||
propertyType = arg.type
|
||||
}
|
||||
|
||||
$: arg?.type && arg.type !== propertyType && updatePropertyType()
|
||||
|
||||
function getPropertyType(arg: InputTransform | any): 'static' | 'javascript' {
|
||||
let type: 'static' | 'javascript' = arg?.type ?? 'static'
|
||||
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
<script lang="ts">
|
||||
import { createPopperActions } from 'svelte-popperjs'
|
||||
import type { PopoverPlacement } from './Popover.model'
|
||||
import Portal from 'svelte-portal'
|
||||
import { fade } from 'svelte/transition'
|
||||
|
||||
export let placement: PopoverPlacement = 'bottom'
|
||||
|
||||
const [popperRef, popperContent, getInstance] = createPopperActions({ placement })
|
||||
|
||||
export async function refresh() {
|
||||
await getInstance()?.update()
|
||||
}
|
||||
|
||||
let showTooltip = false
|
||||
export function open() {
|
||||
showTooltip = true
|
||||
}
|
||||
export function close() {
|
||||
showTooltip = false
|
||||
}
|
||||
</script>
|
||||
|
||||
<fragment use:popperRef>
|
||||
<slot />
|
||||
</fragment>
|
||||
{#if showTooltip}
|
||||
<Portal>
|
||||
<div
|
||||
use:popperContent
|
||||
class="z-[901] rounded-lg shadow-md border p-4 bg-surface"
|
||||
transition:fade={{ duration: 200 }}
|
||||
>
|
||||
<slot name="content" />
|
||||
</div>
|
||||
</Portal>
|
||||
{/if}
|
||||
110
frontend/src/lib/components/Modal.svelte
Normal file
110
frontend/src/lib/components/Modal.svelte
Normal file
@@ -0,0 +1,110 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher } from 'svelte'
|
||||
|
||||
export let open: boolean = false
|
||||
export let z = 'z-30'
|
||||
|
||||
const dispatch = createEventDispatcher()
|
||||
|
||||
export function closeDrawer(): void {
|
||||
document.body.style.overflow = 'auto'
|
||||
|
||||
open = false
|
||||
dispatch('close')
|
||||
}
|
||||
|
||||
export function openDrawer(): void {
|
||||
document.body.style.overflow = 'hidden'
|
||||
open = true
|
||||
dispatch('open')
|
||||
}
|
||||
|
||||
function handleKeyUp(event: KeyboardEvent): void {
|
||||
const key = event.key
|
||||
if (key === 'Escape' || key === 'Esc') {
|
||||
if (open) {
|
||||
event.preventDefault()
|
||||
closeDrawer()
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:window on:keyup={handleKeyUp} />
|
||||
|
||||
{#if open}
|
||||
<div class="blurred-background" />
|
||||
|
||||
<div class="fixed top-0 w-screen h-screen {z}">
|
||||
<div
|
||||
class="fixed right-0 top-0 flex flex-col w-3/4 sm:w-2/3 lg:w-1/2 h-screen border border-gray-300 shadow-xl"
|
||||
>
|
||||
{#if open}
|
||||
<div class="flex flex-row justify-between p-2 bg-surface border-b border-gray-200">
|
||||
<button
|
||||
on:click={() => {
|
||||
open = false
|
||||
closeDrawer()
|
||||
}}
|
||||
>
|
||||
<svg
|
||||
class="w-6 h-6"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
stroke-width="2"
|
||||
d="M6 18L18 6M6 6l12 12"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
<p class="font-semibold text-gray-800"><slot name="title" /></p>
|
||||
<div />
|
||||
</div>
|
||||
<div class="flex flex-col bg-gray-50 pt-3 px-6 grow overflow-y-auto">
|
||||
<slot name="content" />
|
||||
</div>
|
||||
<div class="flex flex-col bg-surface border-gray-200 p-2">
|
||||
<div class="flex flex-row justify-between p-2">
|
||||
<button
|
||||
on:click={() => {
|
||||
closeDrawer()
|
||||
}}
|
||||
>
|
||||
<svg
|
||||
class="w-6 h-6"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
>
|
||||
<path
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round"
|
||||
stroke-width="2"
|
||||
d="M6 18L18 6M6 6l12 12"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
<span class="mr-4"><slot name="submission"> </slot></span>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<style lang="postcss">
|
||||
.blurred-background {
|
||||
/* @apply absolute sm:top-6 lg:top-8 left-28 sm:left-40 md:left-48; */ /* If we wanted to make the navbars visible */
|
||||
@apply fixed top-0 left-0;
|
||||
@apply bg-gray-400 opacity-75;
|
||||
@apply w-screen;
|
||||
@apply h-screen;
|
||||
z-index: 10;
|
||||
}
|
||||
</style>
|
||||
@@ -19,7 +19,7 @@
|
||||
import type { PickableProperties } from './flows/previousResults'
|
||||
import type DiffEditor from './DiffEditor.svelte'
|
||||
import type Editor from './Editor.svelte'
|
||||
import ScriptFix from './copilot/ScriptFix.svelte'
|
||||
import ScriptFix from './codeGen/ScriptFix.svelte'
|
||||
|
||||
export let mod: FlowModule
|
||||
export let schema: Schema
|
||||
@@ -135,13 +135,12 @@
|
||||
jobId={testJob?.id}
|
||||
result={testJob.result}>
|
||||
<svelte:fragment slot="copilot-fix">
|
||||
{#if lang && editor && diffEditor && stepArgs && testJob?.result?.error}
|
||||
{#if lang && editor && diffEditor && testJob?.result?.error}
|
||||
<ScriptFix
|
||||
error={JSON.stringify(testJob.result.error)}
|
||||
{lang}
|
||||
{editor}
|
||||
{diffEditor}
|
||||
args={stepArgs}
|
||||
/>
|
||||
{/if}
|
||||
</svelte:fragment>
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<span class="flex items-center space-x-2">
|
||||
<h1 class="!text-2xl font-semibold leading-6 tracking-tight">{title}</h1>
|
||||
{#if tooltip != '' || documentationLink}
|
||||
<Tooltip {documentationLink}>
|
||||
<Tooltip light {documentationLink} scale={0.9} wrapperClass="flex items-center">
|
||||
{tooltip}
|
||||
</Tooltip>
|
||||
{/if}
|
||||
@@ -21,7 +21,7 @@
|
||||
<span class="flex items-center space-x-2">
|
||||
<h2 class="!text-sm font-semibold">{title}</h2>
|
||||
{#if tooltip != '' || documentationLink}
|
||||
<Tooltip {documentationLink}>
|
||||
<Tooltip light {documentationLink} scale={0.9} wrapperClass="flex items-center">
|
||||
{tooltip}
|
||||
</Tooltip>
|
||||
{/if}
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
size="sm"
|
||||
variant="border"
|
||||
color={isPicked ? 'blue' : 'dark'}
|
||||
btnClasses={isPicked ? '!border-2' : 'm-[1px]'}
|
||||
btnClasses={isPicked ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
|
||||
disabled={notPickable}
|
||||
on:click={() => onClick(undefined)}
|
||||
>
|
||||
@@ -48,8 +48,8 @@
|
||||
<Button
|
||||
size="sm"
|
||||
variant="border"
|
||||
color={isPicked ? 'blue' : 'light'}
|
||||
btnClasses={isPicked ? '!border-2' : 'm-[1px]'}
|
||||
color={isPicked ? 'blue' : 'dark'}
|
||||
btnClasses={isPicked ? '!border-2 !bg-blue-50/75' : 'm-[1px]'}
|
||||
disabled={notPickable}
|
||||
on:click={() => onClick(r)}
|
||||
>
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
import Toggle from '$lib/components/Toggle.svelte'
|
||||
import Tooltip from '$lib/components/Tooltip.svelte'
|
||||
import { FlowService, ScheduleService, Script, ScriptService, type Flow } from '$lib/gen'
|
||||
import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores'
|
||||
import { userStore, workspaceStore } from '$lib/stores'
|
||||
import { canWrite, emptySchema, emptyString, formatCron, sendUserToast } from '$lib/utils'
|
||||
import { faList, faSave } from '@fortawesome/free-solid-svg-icons'
|
||||
import { createEventDispatcher } from 'svelte'
|
||||
@@ -324,7 +324,7 @@
|
||||
|
||||
<div class="flex flex-row items-center mb-2 gap-1">
|
||||
<div class="text-xl font-extrabold">Schedule</div>
|
||||
<Tooltip>Schedules use CRON syntax. Seconds are mandatory.</Tooltip>
|
||||
<Tooltip light>Schedules use CRON syntax. Seconds are mandatory.</Tooltip>
|
||||
</div>
|
||||
|
||||
<CronInput disabled={!can_write} bind:schedule bind:timezone bind:validCRON />
|
||||
@@ -372,7 +372,7 @@
|
||||
{/if}
|
||||
</div>
|
||||
<h2 class="border-b pb-1 mt-8 mb-2"
|
||||
>Error Handler <Tooltip>
|
||||
>Error Handler <Tooltip light>
|
||||
<div class="flex gap-20 items-start mt-3">
|
||||
<div class="text-tertiary text-sm"
|
||||
>The following args will be passed to the error handler:
|
||||
@@ -429,21 +429,12 @@
|
||||
|
||||
<div class="flex flex-row items-center justify-between">
|
||||
<div class="flex flex-row items-center mt-4 font-semibold text-sm gap-2">
|
||||
<p
|
||||
>{#if !$enterpriseLicense}<span class="text-normal text-2xs">(ee only)</span>{/if} Triggered
|
||||
when schedule failed</p
|
||||
>
|
||||
<select class="!w-14" bind:value={failedExact} disabled={!$enterpriseLicense}>
|
||||
<p>Triggered when schedule failed</p>
|
||||
<select class="!w-14" bind:value={failedExact}>
|
||||
<option value={false}>>=</option>
|
||||
<option value={true}>==</option>
|
||||
</select>
|
||||
<input
|
||||
type="number"
|
||||
class="!w-14 text-center"
|
||||
bind:value={failedTimes}
|
||||
disabled={!$enterpriseLicense}
|
||||
min="1"
|
||||
/>
|
||||
<input type="number" class="!w-14 text-center" bind:value={failedTimes} min="1" />
|
||||
<p>time{failedTimes > 1 ? 's in a row' : ''}</p>
|
||||
</div>
|
||||
</div>
|
||||
@@ -464,10 +455,7 @@
|
||||
{/if}
|
||||
|
||||
<h2 class="border-b pb-1 mt-8 mb-2"
|
||||
>Recovery Handler {#if !$enterpriseLicense}<span class="text-normal text-2xs"
|
||||
>(ee only)</span
|
||||
>{/if}
|
||||
<Tooltip
|
||||
>Recovery Handler <Tooltip light
|
||||
><div class="text-tertiary text-sm"
|
||||
>The following args will be passed to the recovery handler:
|
||||
<ul class="mt-1 ml-2">
|
||||
@@ -495,7 +483,7 @@
|
||||
{#if recoveryHandlerSelected === 'custom'}
|
||||
<div class="flex flex-row mb-2">
|
||||
<ScriptPicker
|
||||
disabled={!can_write || !$enterpriseLicense}
|
||||
disabled={!can_write}
|
||||
initialPath={recoveryHandlerPath}
|
||||
kind={Script.kind.SCRIPT}
|
||||
allowFlow={true}
|
||||
|
||||
@@ -160,19 +160,19 @@
|
||||
<Button
|
||||
size="sm"
|
||||
variant="border"
|
||||
color={isSelected ? 'blue' : 'light'}
|
||||
btnClasses={isSelected ? '!border-2' : 'm-[1px]'}
|
||||
color={isSelected ? 'blue' : 'dark'}
|
||||
btnClasses={isSelected ? '!bg-blue-50/75 dark:!bg-frost-900/50 ' : ''}
|
||||
on:click={() => {
|
||||
property.selectedType = argType;
|
||||
property.format = undefined;
|
||||
property.contentEncoding = undefined;
|
||||
property.enum_ = undefined;
|
||||
property.pattern = undefined;
|
||||
property.default = undefined;
|
||||
property.selectedType = argType
|
||||
property.format = undefined
|
||||
property.contentEncoding = undefined
|
||||
property.enum_ = undefined
|
||||
property.pattern = undefined
|
||||
property.default = undefined
|
||||
if (argType == 'array') {
|
||||
property.items = { type: 'string' }
|
||||
} else {
|
||||
property.items = undefined;
|
||||
property.items = undefined
|
||||
}
|
||||
}}
|
||||
>
|
||||
@@ -182,15 +182,14 @@
|
||||
<Button
|
||||
size="sm"
|
||||
variant="border"
|
||||
color={!property.selectedType ? 'blue' : 'light'}
|
||||
btnClasses={!property.selectedType ? '!border-2' : 'm-[1px]'}
|
||||
color={!property.selectedType ? 'blue' : 'dark'}
|
||||
btnClasses={!property.selectedType ? '!bg-blue-50/75 dark:!bg-frost-900/50' : ']'}
|
||||
on:click={() => {
|
||||
property.selectedType = undefined;
|
||||
property.selectedType = undefined
|
||||
}}
|
||||
>
|
||||
any
|
||||
</Button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
|
||||
@@ -483,7 +483,7 @@
|
||||
>
|
||||
<Toggle
|
||||
disabled={!$enterpriseLicense ||
|
||||
isCloudHosted() ||
|
||||
!isCloudHosted() ||
|
||||
script.language != Script.language.BUN}
|
||||
size="xs"
|
||||
checked={Boolean(script.dedicated_worker)}
|
||||
@@ -680,7 +680,7 @@
|
||||
>
|
||||
{title}
|
||||
{#if desc}
|
||||
<Tooltip {documentationLink}>
|
||||
<Tooltip {documentationLink} class="mb-0.5 ml-1">
|
||||
{desc}
|
||||
</Tooltip>
|
||||
{/if}
|
||||
|
||||
@@ -249,7 +249,6 @@
|
||||
kind={asKind(kind)}
|
||||
{template}
|
||||
{diffEditor}
|
||||
{args}
|
||||
/>
|
||||
{#if !noSyncFromGithub}
|
||||
<div class="py-1">
|
||||
@@ -303,7 +302,6 @@
|
||||
deno={lang == 'deno'}
|
||||
automaticLayout={true}
|
||||
{fixedOverflowWidgets}
|
||||
{args}
|
||||
/>
|
||||
<DiffEditor
|
||||
bind:this={diffEditor}
|
||||
@@ -366,7 +364,6 @@
|
||||
previewIsLoading={testIsLoading}
|
||||
{editor}
|
||||
{diffEditor}
|
||||
{args}
|
||||
/>
|
||||
</Pane>
|
||||
</Splitpanes>
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
export let size: 'sm' | 'xs' = 'sm'
|
||||
const id = (Math.random() + 1).toString(36).substring(10)
|
||||
const dispatch = createEventDispatcher()
|
||||
const bothOptions = Boolean(options.left) && Boolean(options.right);
|
||||
</script>
|
||||
|
||||
<span class="{$$props.class} z-auto">
|
||||
@@ -29,17 +28,17 @@
|
||||
: 'cursor-pointer'}"
|
||||
>
|
||||
{#if Boolean(options?.left)}
|
||||
<span
|
||||
class={twMerge(
|
||||
'mr-2 font-medium duration-50',
|
||||
bothOptions ? (checked ? 'text-disabled' : 'text-primary') : 'text-primary',
|
||||
size === 'xs' ? 'text-xs' : 'text-sm',
|
||||
textClass
|
||||
)}
|
||||
style={textStyle}
|
||||
>
|
||||
{options?.left}
|
||||
</span>
|
||||
<span
|
||||
class={twMerge(
|
||||
'mr-2 font-medium duration-50',
|
||||
disabled ? 'text-disabled' : 'text-primary',
|
||||
size === 'xs' ? 'text-xs' : 'text-sm',
|
||||
textClass
|
||||
)}
|
||||
style={textStyle}
|
||||
>
|
||||
{options?.left}
|
||||
</span>
|
||||
{/if}
|
||||
|
||||
<!-- svelte-ignore a11y-click-events-have-key-events -->
|
||||
@@ -67,20 +66,20 @@
|
||||
/>
|
||||
</div>
|
||||
{#if Boolean(options?.right)}
|
||||
<span
|
||||
class={twMerge(
|
||||
'ml-2 font-medium duration-50',
|
||||
bothOptions ? (checked ? 'text-primary' : 'text-disabled') : 'text-primary',
|
||||
size === 'xs' ? 'text-xs' : 'text-sm',
|
||||
textClass
|
||||
)}
|
||||
style={textStyle}
|
||||
>
|
||||
{options?.right}
|
||||
{#if options?.rightTooltip}
|
||||
<Tooltip>{options?.rightTooltip}</Tooltip>
|
||||
{/if}
|
||||
</span>
|
||||
<span
|
||||
class={twMerge(
|
||||
'ml-2 font-medium duration-50',
|
||||
disabled ? 'text-disabled' : 'text-primary',
|
||||
size === 'xs' ? 'text-xs' : 'text-sm',
|
||||
textClass
|
||||
)}
|
||||
style={textStyle}
|
||||
>
|
||||
{options?.right}
|
||||
{#if options?.rightTooltip}
|
||||
<Tooltip>{options?.rightTooltip}</Tooltip>
|
||||
{/if}
|
||||
</span>
|
||||
{/if}
|
||||
</label>
|
||||
</span>
|
||||
|
||||
@@ -1,22 +1,25 @@
|
||||
<script lang="ts">
|
||||
import { faInfoCircle } from '@fortawesome/free-solid-svg-icons'
|
||||
import Icon from 'svelte-awesome'
|
||||
import type { PopoverPlacement } from './Popover.model'
|
||||
import Popover from './Popover.svelte'
|
||||
import { ExternalLink, InfoIcon } from 'lucide-svelte'
|
||||
import { ExternalLink } from 'lucide-svelte'
|
||||
|
||||
export let light = false
|
||||
export let scale = 0.8
|
||||
export let wrapperClass = ''
|
||||
export let placement: PopoverPlacement | undefined = undefined
|
||||
export let documentationLink: string | undefined = undefined
|
||||
</script>
|
||||
|
||||
<Popover notClickable {placement} class={wrapperClass}>
|
||||
<div
|
||||
class="inline-flex w-3 mx-0.5 {light
|
||||
? 'text-tertiary-inverse'
|
||||
: 'text-tertiary'} {$$props.class} relative"
|
||||
>
|
||||
<InfoIcon class="-bottom-0.5 absolute" size={16} />
|
||||
</div>
|
||||
<Icon
|
||||
class="{light
|
||||
? 'text-gray-400 dark:text-gray-200 hover:text-tertiary dark:hover:text-gray-300'
|
||||
: ' text-tertiary dark:text-gray-300 hover:text-tertiary dark:hover:text-gray-400'} cursor-pointer transition-all font-thin flex h-4 p-0.5 w-4 justify-center items-center {$$props.class}"
|
||||
data={faInfoCircle}
|
||||
{scale}
|
||||
/>
|
||||
<svelte:fragment slot="text">
|
||||
<slot />
|
||||
{#if documentationLink}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user