Compare commits
1 Commits
instructio
...
fix-chartj
| Author | SHA1 | Date | |
|---|---|---|---|
|
b3fbf69b35 |
9
.env
9
.env
@@ -1,12 +1,19 @@
|
||||
DB_PASSWORD=changeme
|
||||
DATABASE_URL=postgres://postgres:changeme@db/windmill?sslmode=disable
|
||||
|
||||
WM_IMAGE=ghcr.io/windmill-labs/windmill:main
|
||||
WM_LICENSE_KEY=""
|
||||
|
||||
# For Enterprise Edition, comment the 2 lines above and uncomment below
|
||||
# WM_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
|
||||
# WINDMILL_IMAGE=ghcr.io/windmill-labs/windmill-ee:main
|
||||
# WM_LICENSE_KEY="<id>.<expiry>.<signature>"
|
||||
|
||||
|
||||
# this is the url that your instance is publicly exposed to
|
||||
WM_BASE_URL=http://localhost
|
||||
|
||||
# To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started
|
||||
# To have caddy take care of automatic TLS
|
||||
|
||||
|
||||
WM_REQUEST_SIZE_LIMIT=50097152 # 50MB (The size limit for any requests, including script/flow args inputs)
|
||||
|
||||
5
.github/DockerfileBackendTests
vendored
5
.github/DockerfileBackendTests
vendored
@@ -37,10 +37,9 @@ RUN apt-get update \
|
||||
make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
|
||||
libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
|
||||
libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
|
||||
libv8-dev tesseract-ocr nodejs npm clang libclang-dev\
|
||||
libv8-dev tesseract-ocr nodejs npm\
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
RUN wget https://golang.org/dl/go1.19.1.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.19.1.linux-amd64.tar.gz
|
||||
ENV PATH="${PATH}:/usr/local/go/bin"
|
||||
ENV GO_PATH=/usr/local/go/bin/go
|
||||
@@ -58,7 +57,7 @@ RUN /usr/local/bin/python3 -m pip install nltk
|
||||
RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
|
||||
|
||||
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
|
||||
COPY --from=oven/bun:1.0.0 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
|
||||
|
||||
RUN apt-get update \
|
||||
|
||||
2
.github/change-versions.sh
vendored
2
.github/change-versions.sh
vendored
@@ -5,7 +5,7 @@ echo "Updating versions to: $VERSION"
|
||||
|
||||
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
|
||||
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
|
||||
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/lib.ts
|
||||
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts
|
||||
sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
|
||||
sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
|
||||
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" typescript-client/package.json
|
||||
|
||||
2
.github/workflows/backend-test.yml
vendored
2
.github/workflows/backend-test.yml
vendored
@@ -42,4 +42,4 @@ jobs:
|
||||
mkdir frontend/build && cd backend && touch
|
||||
windmill-api/openapi-deref.yaml &&
|
||||
DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
|
||||
test --features enterprise --all -- --nocapture
|
||||
test --all -- --nocapture
|
||||
|
||||
21
.github/workflows/benchmark.yml
vendored
21
.github/workflows/benchmark.yml
vendored
@@ -19,28 +19,15 @@ jobs:
|
||||
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5
|
||||
windmill:
|
||||
image: ghcr.io/windmill-labs/windmill-ee:main
|
||||
image: ghcr.io/windmill-labs/windmill:main
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
WORKER_GROUP: main
|
||||
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow
|
||||
options: >-
|
||||
--pull always --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5 --health-cmd "curl
|
||||
http://localhost:8000/api/version"
|
||||
ports:
|
||||
- 8000:8000
|
||||
windmill-worker:
|
||||
image: ghcr.io/windmill-labs/windmill-ee:main
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
|
||||
DISABLE_SERVER: true
|
||||
WORKER_GROUP: dedicated
|
||||
DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
|
||||
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
|
||||
options: >-
|
||||
--pull always
|
||||
steps:
|
||||
- uses: denoland/setup-deno@v1
|
||||
with:
|
||||
@@ -50,9 +37,13 @@ jobs:
|
||||
ref: benchmarks
|
||||
- name: benchmark
|
||||
timeout-minutes: 10
|
||||
run: deno run --unstable -A -r
|
||||
run: deno run --unstable -A
|
||||
https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
|
||||
--host http://localhost:8000
|
||||
-e admin@windmill.dev
|
||||
-p changeme
|
||||
-c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json
|
||||
--branch ${GITHUB_REF##ref/head/}
|
||||
- name: Push changes
|
||||
run: |
|
||||
pwd
|
||||
|
||||
154
CHANGELOG.md
154
CHANGELOG.md
@@ -1,160 +1,6 @@
|
||||
# Changelog
|
||||
|
||||
|
||||
## [1.177.0](https://github.com/windmill-labs/windmill/compare/v1.176.0...v1.177.0) (2023-09-26)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add custom oauth support ([#2336](https://github.com/windmill-labs/windmill/issues/2336)) ([01277f4](https://github.com/windmill-labs/windmill/commit/01277f4d3b8bb04b955d5bbb2ed69c1c7c8f4f9e))
|
||||
* support automatic reconnection to pg ([ccaa05d](https://github.com/windmill-labs/windmill/commit/ccaa05d4bf5954c3fb8678239d2962cac6550a5a))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* fix resource type picker object reinitialization ([f0f15c4](https://github.com/windmill-labs/windmill/commit/f0f15c47cb35cc1e3cfa13549465803a1e970770))
|
||||
* **frontend:** Fix build ([#2330](https://github.com/windmill-labs/windmill/issues/2330)) ([46592af](https://github.com/windmill-labs/windmill/commit/46592affd3d51b54632a2a7a281c11141edcb4a5))
|
||||
* **frontend:** Fix markdown dark mode ([#2329](https://github.com/windmill-labs/windmill/issues/2329)) ([6c19740](https://github.com/windmill-labs/windmill/commit/6c197407185810f43c47d4107007bd69814a1d65))
|
||||
* set min size of components to 1 ([d298093](https://github.com/windmill-labs/windmill/commit/d298093e29bd9983c7631a8f8c80e47b768bb93c))
|
||||
|
||||
## [1.176.0](https://github.com/windmill-labs/windmill/compare/v1.175.0...v1.176.0) (2023-09-24)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add license key as superadmin setting ([#2321](https://github.com/windmill-labs/windmill/issues/2321)) ([304a259](https://github.com/windmill-labs/windmill/commit/304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254))
|
||||
* add running filter ([ea364ad](https://github.com/windmill-labs/windmill/commit/ea364ad9602647cbc9e8ee78fb5f17f0012105f6))
|
||||
* ai flow trigger menu ([#2317](https://github.com/windmill-labs/windmill/issues/2317)) ([95194ab](https://github.com/windmill-labs/windmill/commit/95194abeacc42416174ee9dd79b75f2204a40d33))
|
||||
* improved dedicated benchmarks + buffer fix ([#2313](https://github.com/windmill-labs/windmill/issues/2313)) ([fc93c2a](https://github.com/windmill-labs/windmill/commit/fc93c2a7cece95c00070a3a3391ae2bcb4513e85))
|
||||
* set instance settings from UI ([#2314](https://github.com/windmill-labs/windmill/issues/2314)) ([2f0e43b](https://github.com/windmill-labs/windmill/commit/2f0e43bfdbd1e196131f126c83b1d7dd2eea98d8))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add ability to test this step for flow step ([3585929](https://github.com/windmill-labs/windmill/commit/3585929bb758b0cfc2cbe43f66597b184e7b8ee0))
|
||||
* benchmark worker tags ([#2319](https://github.com/windmill-labs/windmill/issues/2319)) ([481bcd5](https://github.com/windmill-labs/windmill/commit/481bcd53cb07e4520d5fd81572cad74340c4eb64))
|
||||
* change cache implementation to remove async-timer ([4911b4b](https://github.com/windmill-labs/windmill/commit/4911b4b3fd6e3a9f6bccc4c8712b736e18dcb6e1))
|
||||
* fix upto preview issue with nested flows ([6492ff6](https://github.com/windmill-labs/windmill/commit/6492ff627a800832e12a31fd89a6070703988eb9))
|
||||
* flow steps appears in all static inputs ([c043847](https://github.com/windmill-labs/windmill/commit/c0438479aa3b6dc6349df01abdd9dcc434fe8781))
|
||||
* optimize performance for bun scripts without deps ([5b33f56](https://github.com/windmill-labs/windmill/commit/5b33f563e6e83605ae72338af351dcc97beb1a55))
|
||||
* overflow on workspace script picker ([5e4db0e](https://github.com/windmill-labs/windmill/commit/5e4db0ebab616305928cfa455af6833335e0fcf9))
|
||||
* tag id as flow ([#2318](https://github.com/windmill-labs/windmill/issues/2318)) ([f68cee4](https://github.com/windmill-labs/windmill/commit/f68cee4ebddbf6e774f80e91a8c89fb8dc213f91))
|
||||
|
||||
## [1.175.0](https://github.com/windmill-labs/windmill/compare/v1.174.0...v1.175.0) (2023-09-19)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add batch jobs ([#2306](https://github.com/windmill-labs/windmill/issues/2306)) ([5867e5d](https://github.com/windmill-labs/windmill/commit/5867e5d0f80fd515fab165659831b5ee9a8c3f97))
|
||||
* add dediacted worker env var ([#2296](https://github.com/windmill-labs/windmill/issues/2296)) ([e0c6eee](https://github.com/windmill-labs/windmill/commit/e0c6eee16e535b3a7d803a7978e463404f5fec30))
|
||||
* dedicated benchmarks ([#2297](https://github.com/windmill-labs/windmill/issues/2297)) ([c549239](https://github.com/windmill-labs/windmill/commit/c5492396843ddd9143ffe890696d0317c970de36))
|
||||
* **frontend:** Add component control doc ([#2295](https://github.com/windmill-labs/windmill/issues/2295)) ([26f8863](https://github.com/windmill-labs/windmill/commit/26f88636f0b972d4fe4931ed02135c38b27a56d2))
|
||||
* suggest adding openai key on workspace creation ([a6b3b2f](https://github.com/windmill-labs/windmill/commit/a6b3b2f63b317825a3d80218cbb606b9f610c221))
|
||||
* support pinned versions for bun in deployed scripts ([03806dc](https://github.com/windmill-labs/windmill/commit/03806dc3907cba724be14acb6aadf5be6e35cdb6))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add HOME to bun and deno ([0e3ecc7](https://github.com/windmill-labs/windmill/commit/0e3ecc7d6025c173135f20bacc33a0dc972ec222))
|
||||
* add queue_count to metrics ([9ced883](https://github.com/windmill-labs/windmill/commit/9ced8834a45151c6900b1eb33eca2cff4886a065))
|
||||
* ai improve prompts ([#2310](https://github.com/windmill-labs/windmill/issues/2310)) ([b647213](https://github.com/windmill-labs/windmill/commit/b647213b2c968b0cb1f90c97d94e8023c415dd55))
|
||||
* **frontend:** add missing key ([#2299](https://github.com/windmill-labs/windmill/issues/2299)) ([39d2467](https://github.com/windmill-labs/windmill/commit/39d24672ddd696372e55e9b4566f322a322385a8))
|
||||
* **frontend:** Always mount components ([#2309](https://github.com/windmill-labs/windmill/issues/2309)) ([34f94aa](https://github.com/windmill-labs/windmill/commit/34f94aa50e92254114c046fa8b7e900d93807937))
|
||||
* **frontend:** fix alignment ([#2307](https://github.com/windmill-labs/windmill/issues/2307)) ([f9fc6f1](https://github.com/windmill-labs/windmill/commit/f9fc6f19482e68c9ccba0014879fd8761662c36a))
|
||||
* **frontend:** Fix rich result styling + add title and hideDetails config ([#2294](https://github.com/windmill-labs/windmill/issues/2294)) ([732daef](https://github.com/windmill-labs/windmill/commit/732daef1c3515f7df3e09deac691bb585f9859cd))
|
||||
* **frontend:** fix tab styling + component bg ([#2308](https://github.com/windmill-labs/windmill/issues/2308)) ([5e773d3](https://github.com/windmill-labs/windmill/commit/5e773d386343f003425173207c166e3c4eeef956))
|
||||
* **frontend:** fix theme make default ([#2304](https://github.com/windmill-labs/windmill/issues/2304)) ([4629819](https://github.com/windmill-labs/windmill/commit/46298197c5333a81b9b8a004027ab9a856bdada4))
|
||||
* **frontend:** fix theme UI ([#2305](https://github.com/windmill-labs/windmill/issues/2305)) ([576f76b](https://github.com/windmill-labs/windmill/commit/576f76b1ffe9c50c8ccaca8c5e34d0ec03aebf3f))
|
||||
* validate more strongly usernames ([47094bb](https://github.com/windmill-labs/windmill/commit/47094bb8d1c6f4ba621d42515dede061fd04afdd))
|
||||
|
||||
## [1.174.0](https://github.com/windmill-labs/windmill/compare/v1.173.0...v1.174.0) (2023-09-15)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* ai gen support all langs ([#2276](https://github.com/windmill-labs/windmill/issues/2276)) ([39590b3](https://github.com/windmill-labs/windmill/commit/39590b3d2592b2d08117c0f70829c13f1efb4885))
|
||||
* bun absolute/relative imports + tests ([#2286](https://github.com/windmill-labs/windmill/issues/2286)) ([e5ce85b](https://github.com/windmill-labs/windmill/commit/e5ce85b9affe665342f24b1d39ce3d03db09b941))
|
||||
* **frontend:** Global CSS editor ([#2178](https://github.com/windmill-labs/windmill/issues/2178)) ([7e9ee39](https://github.com/windmill-labs/windmill/commit/7e9ee39aa69bc31766b5e4f4aab498c8f14067cd))
|
||||
|
||||
## [1.173.0](https://github.com/windmill-labs/windmill/compare/v1.172.1...v1.173.0) (2023-09-14)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* cli sync on windows ([#2283](https://github.com/windmill-labs/windmill/issues/2283)) ([c371cb3](https://github.com/windmill-labs/windmill/commit/c371cb397ab3d0c534e2c553d1dfb1ad5176d2a6))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* accept jobs whose duration > 24 days ([2c00894](https://github.com/windmill-labs/windmill/commit/2c00894122aa8caee59b20625935284de6902950))
|
||||
|
||||
## [1.172.1](https://github.com/windmill-labs/windmill/compare/v1.172.0...v1.172.1) (2023-09-14)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* improve splitpane + improve deleting conditional tab ([1629008](https://github.com/windmill-labs/windmill/commit/1629008eb2eb48ff9cc2cf6b3a351efcf682244d))
|
||||
* update to svelte 4 ([#2280](https://github.com/windmill-labs/windmill/issues/2280)) ([90c10d8](https://github.com/windmill-labs/windmill/commit/90c10d803b4c47a9e1ac5b9e49e2a614344299a9))
|
||||
|
||||
## [1.172.0](https://github.com/windmill-labs/windmill/compare/v1.171.0...v1.172.0) (2023-09-13)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* improve ai flow ([#2270](https://github.com/windmill-labs/windmill/issues/2270)) ([b23417a](https://github.com/windmill-labs/windmill/commit/b23417ab5b9938bbdf9db6449102760ff8c80152))
|
||||
* worker groups admin panel ([#2277](https://github.com/windmill-labs/windmill/issues/2277)) ([070b162](https://github.com/windmill-labs/windmill/commit/070b16222bc666866284180b3878f4d4f27bfa85))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* ai flow nits ([#2272](https://github.com/windmill-labs/windmill/issues/2272)) ([8f6f46d](https://github.com/windmill-labs/windmill/commit/8f6f46de199d58133b9faa77cdbcbcfd6cb962f7))
|
||||
|
||||
## [1.171.0](https://github.com/windmill-labs/windmill/compare/v1.170.0...v1.171.0) (2023-09-12)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* attempt to SIGTERM before SIGKILL for bash ([f40bbba](https://github.com/windmill-labs/windmill/commit/f40bbba519a97cbb1ec142c335f038dbebcd4e7c))
|
||||
* zero copy result for job result ([#2263](https://github.com/windmill-labs/windmill/issues/2263)) ([22a7da5](https://github.com/windmill-labs/windmill/commit/22a7da58b1d20721892906cba2dee6fbeb1cc1fd))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* 2257 TIME convertion in pg_executor.rs ([#2267](https://github.com/windmill-labs/windmill/issues/2267)) ([3d71253](https://github.com/windmill-labs/windmill/commit/3d71253abdb0dff1670a796d07a53ecd0a98414e))
|
||||
* fix field duplicate in app background settings ([164cdaf](https://github.com/windmill-labs/windmill/commit/164cdaf09464646dee4e70a699222a454eb0d898))
|
||||
* improve bun lockfile resolution ([9103ec4](https://github.com/windmill-labs/windmill/commit/9103ec445db81395a5851202eecb87301d0b4987))
|
||||
* remove result and args from list completed and list queue jobs ([e7e63e1](https://github.com/windmill-labs/windmill/commit/e7e63e111a73e0986050a8fe7fdc18784ba902b0))
|
||||
|
||||
## [1.170.0](https://github.com/windmill-labs/windmill/compare/v1.169.0...v1.170.0) (2023-09-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* display jobs currently waiting for a worker ([3c950c0](https://github.com/windmill-labs/windmill/commit/3c950c03de0bc71974eb29985381adba8c098660))
|
||||
* snowflake schema explorer + refactoring ([#2260](https://github.com/windmill-labs/windmill/issues/2260)) ([5cca583](https://github.com/windmill-labs/windmill/commit/5cca5833e94fc4c8a80e210164da09f2a1ceb677))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* fix get_result for python-client ([fe41f4f](https://github.com/windmill-labs/windmill/commit/fe41f4ff4ce596cf394bd69a0ba48e88db8d2328))
|
||||
|
||||
## [1.169.0](https://github.com/windmill-labs/windmill/compare/v1.168.3...v1.169.0) (2023-09-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* benchmarks graph ([#2244](https://github.com/windmill-labs/windmill/issues/2244)) ([c496602](https://github.com/windmill-labs/windmill/commit/c496602e9e2e0dfecaaffe731e58e551d039d02f))
|
||||
* big query schema explorer ([#2247](https://github.com/windmill-labs/windmill/issues/2247)) ([ec7d923](https://github.com/windmill-labs/windmill/commit/ec7d923cca0f6050855473ababd1bb27d668711b))
|
||||
* flow copilot ([#2219](https://github.com/windmill-labs/windmill/issues/2219)) ([2f3138c](https://github.com/windmill-labs/windmill/commit/2f3138c65d9d3f0161bf3e069c6eec0c32ac3b86))
|
||||
* **frontend:** fix runs page when the row has a parent ([#2255](https://github.com/windmill-labs/windmill/issues/2255)) ([2271263](https://github.com/windmill-labs/windmill/commit/22712632f683fb63ad6d4b475a01c63800a9559d))
|
||||
* introduce container groups ([49c5553](https://github.com/windmill-labs/windmill/commit/49c5553f3b496c2aaf03376689ee0fd42ecbd2bf))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* benchmark svg ([#2249](https://github.com/windmill-labs/windmill/issues/2249)) ([24c5802](https://github.com/windmill-labs/windmill/commit/24c580211572d6447ca502db141e90c5e084d790))
|
||||
* pass TZ from env to runtimes ([75a1490](https://github.com/windmill-labs/windmill/commit/75a149009a5a13230b4d6de6eac8bba0618629d6))
|
||||
|
||||
## [1.168.3](https://github.com/windmill-labs/windmill/compare/v1.168.2...v1.168.3) (2023-09-07)
|
||||
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
|
||||
|
||||
RUN apt-get -y update \
|
||||
&& apt-get install -y \
|
||||
curl nodejs
|
||||
curl nodejs npm
|
||||
|
||||
RUN rustup component add rustfmt
|
||||
|
||||
@@ -180,7 +180,7 @@ RUN chmod 755 /usr/bin/deno
|
||||
|
||||
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
|
||||
|
||||
COPY --from=oven/bun:1.0.2 /usr/local/bin/bun /usr/bin/bun
|
||||
COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun
|
||||
|
||||
# add the docker client to call docker from a worker if enabled
|
||||
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
|
||||
|
||||
26
LICENSE
26
LICENSE
@@ -1,22 +1,16 @@
|
||||
|
||||
Source code in this repository is variously licensed under the Apache License
|
||||
Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file
|
||||
./LICENSE-AGPL)
|
||||
Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
|
||||
|
||||
Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise
|
||||
specified. Every file is under License AGPL unless otherwise specified or
|
||||
belonging to one of the below cases:
|
||||
Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
|
||||
Every file is under License AGPL unless otherwise specified
|
||||
or belonging to one of the below cases:
|
||||
|
||||
The files under backend/ are AGPLv3 Licensed, except any snippets of code under
|
||||
the compile flag "enterprise". Those snippets and files are under a proprietary
|
||||
and commercial license. The files under frontend/ are AGPLv3 Licensed, except
|
||||
any snippets of code that require a positive license check to be activated.
|
||||
Those snippets and files are under a proprietary and commercial license. Private
|
||||
and public forks MUST not include any of the above proprietary and commercial
|
||||
code. Windmill Labs, Inc. provide tools to clean the codebase from those
|
||||
snippets upon demand. The files under python-client/ deno-client/ go-client/ are
|
||||
Apache 2.0 Licensed.
|
||||
The files under backend/ are AGPLv3 Licensed.
|
||||
The files under frontend/ are AGPLv3 Licensed.
|
||||
The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
|
||||
|
||||
The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
|
||||
|
||||
All third party components incorporated into the Windmill Software are licensed
|
||||
under the original license provided by the owner of the applicable component.
|
||||
All third party components incorporated into the Windmill Software are licensed under the
|
||||
original license provided by the owner of the applicable component.
|
||||
|
||||
@@ -211,6 +211,7 @@ compiling from source or using without a postgres super user, see
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/oauth.json -o oauth.json
|
||||
|
||||
docker compose up -d
|
||||
```
|
||||
@@ -376,7 +377,6 @@ it being synced automatically everyday.
|
||||
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails |
|
||||
| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker |
|
||||
| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker |
|
||||
| DEDICATED_WORKER | None | Unique script to run on that worker. Has to be in the form of <workspace>:<script_path> | Worker |
|
||||
| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server |
|
||||
| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete |
|
||||
| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
|
||||
@@ -395,7 +395,7 @@ it being synced automatically everyday.
|
||||
| SMTP_USERNAME | None | username for the smtp server to send invite emails | Server |
|
||||
| SMTP_PASSWORD | None | password for the smtp server to send invite emails | Server |
|
||||
| SMTP_TLS_IMPLICIT | false | https://docs.rs/mail-send/latest/mail_send/struct.SmtpClientBuilder.html#method.implicit_tlsemails | Server |
|
||||
| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | true | If true, only superadmin can create workspaces | Server |
|
||||
| CREATE_WORKSPACE_REQUIRE_SUPERADMIN | false | If true, only superadmin can create workspaces | Server |
|
||||
| GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE | None | Path to a script to run when a root job fails. The script will be run in and from the admins workspace | Server |
|
||||
| WHITELIST_ENVS | None | List of envs variables, separated by a ',' that are whitelisted as being safe to passthrough the workers | Worker |
|
||||
| SAML_METADATA | None | SAML Metadata URL to enable SAML SSO (EE only) | Server |
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "03c7f098ad795d216d58ded0bf4cf6473960377455b5fd7ac3b578a1d36c0cc6"
|
||||
}
|
||||
@@ -1,16 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2 WHERE worker = $3",
|
||||
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1 WHERE worker = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int4",
|
||||
"TextArray",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "47beea5cd6324b53bfb349665fb215280f32b70a617fde87f70ea53ca9ade39f"
|
||||
"hash": "07551a32c49da8c0693dd39c6a63b5b2a596ccc0e52e8918160604a5e133dd32"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM config WHERE name = $1 RETURNING name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "0d407b9e18a6ee7f4be4fb017e7ed278f2070a939ad8a0680cedd0216d53505a"
|
||||
}
|
||||
@@ -37,6 +37,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT script_path FROM completed_job WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "script_path",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "16be0560028361d46bf3b842a5fa07472994d8942c684f5b75339fe71ea23cdd"
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "duration_ms",
|
||||
"type_info": "Int8"
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -67,6 +67,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT tag, count(*) as count FROM queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "tag",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "3e0e5e0076ae0f7771abd64f8c5bb7e003e3f85351c946a012037e0c874acf1a"
|
||||
}
|
||||
@@ -28,6 +28,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -37,11 +37,6 @@
|
||||
"ordinal": 6,
|
||||
"name": "custom_tags",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "worker_group",
|
||||
"type_info": "Varchar"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -57,9 +52,8 @@
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
false
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "240ce8c9b5c7530999642190c6f7915ae2734b90b8c4cd35fe37783b1d4dd0b0"
|
||||
"hash": "4f6b3b472b4b78c0325cf3755f9ef1806d2e82328ceccbeade8cc2333c6dfe47"
|
||||
}
|
||||
@@ -11,7 +11,7 @@
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8"
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "584cb984ea6528baac48c5c437ad2ee3bef92e3fada73dcf519147964c0f4f4a"
|
||||
}
|
||||
@@ -42,6 +42,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
17
backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json
generated
Normal file
17
backend/.sqlx/query-61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf.json
generated
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags) VALUES ($1, $2, $3, $4) ON CONFLICT (worker) DO NOTHING",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "61e6aac871b482b6e36f866b4ec9148a75e1bd130e7614463487e2ba6957dfdf"
|
||||
}
|
||||
@@ -42,6 +42,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $11)\n )\n INSERT INTO queue \n (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n RETURNING id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8",
|
||||
"Varchar",
|
||||
{
|
||||
"Custom": {
|
||||
"name": "job_kind",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"script",
|
||||
"preview",
|
||||
"flow",
|
||||
"dependencies",
|
||||
"flowpreview",
|
||||
"script_hub",
|
||||
"identity",
|
||||
"flowdependencies",
|
||||
"http",
|
||||
"graphql",
|
||||
"postgresql",
|
||||
"noop",
|
||||
"appdependencies"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Custom": {
|
||||
"name": "script_lang",
|
||||
"kind": {
|
||||
"Enum": [
|
||||
"python3",
|
||||
"deno",
|
||||
"go",
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
"snowflake",
|
||||
"graphql",
|
||||
"powershell"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Timestamptz",
|
||||
"Varchar",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COUNT(*) FROM password WHERE login_type != 'password'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "8b8c8a9ada87f6cb2bee23ba5b707af4f1ad8819cca2335031126041a55f9483"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT config FROM config WHERE name = 'server'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "config",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM resource_type WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "8e6c0ebfe557b8955d23ee3b0e11b77154ab0c2d16b75d4d0c51542569c658a6"
|
||||
}
|
||||
22
backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json
generated
Normal file
22
backend/.sqlx/query-967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1.json
generated
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM\n (SELECT duration_ms FROM completed_job WHERE script_path = $1\n ORDER BY started_at\n DESC LIMIT 10) AS t",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "avg_duration_s",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "967ac1b17291ca80d40acc77e3ba01210bfbd0da18ea7daa61d7f3d6aa6bbcc1"
|
||||
}
|
||||
@@ -60,6 +60,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "exists",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "9f16a61d6a9a42f3fd3e30a1e7776503cee1b45eba150c3082eb246ea3f98d47"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM\n (SELECT duration_ms FROM completed_job WHERE script_path = $1\n ORDER BY started_at\n DESC LIMIT 10) AS t",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "avg_duration_s",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "a0ce703def7e976947513029874fb571893c75730e0a8feeb609852423404cf0"
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Jsonb"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "af00c212f509076e37538be52f582ba09e47db50ba93af322649ccddbb05cc49"
|
||||
}
|
||||
@@ -42,6 +42,7 @@
|
||||
"bash",
|
||||
"postgresql",
|
||||
"nativets",
|
||||
"Nativets",
|
||||
"bun",
|
||||
"mysql",
|
||||
"bigquery",
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT config FROM config WHERE name = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "config",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "bbd4136dfcfac26296fb6c52be1a74fd4874f8f74d9f06c018db8867af18e429"
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT * FROM config WHERE name LIKE 'worker__%'",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "config",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "ce9e56ff451bae10af2c396352f5f93f78658e57b79dc5295553cacc328eb2b7"
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT * FROM config WHERE name = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "config",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "d233e07d19e8e339e1378c1bfc5d78d592c00ffb6f42c3d072f56305b40e50f9"
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "d956b7525f83e6d03beadc4bb3ee2798f53d990b01b17bdbc044719d4908e3f4"
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "e87d4e8d6b52a92537b02a02631a393a2f8b17a5678a1f4f9326db86b417e468"
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT hash FROM script WHERE path = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "hash",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "ef6795d93423f98eea82eb18e6332580dc7f7a9e5a67026f8c0b3077f371fc62"
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "duration",
|
||||
"type_info": "Numeric"
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM completed_job WHERE workspace_id = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "database_length!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "f5681df5935ec753f4a6d9ab8b927bbf890f0b4fe2de16dc764b24f24d7bbdca"
|
||||
}
|
||||
516
backend/Cargo.lock
generated
516
backend/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.177.0"
|
||||
version = "1.168.3"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -22,7 +22,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.177.0"
|
||||
version = "1.168.3"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -54,6 +54,7 @@ tokio-metrics.workspace = true
|
||||
rand.workspace = true
|
||||
chrono.workspace = true
|
||||
git-version.workspace = true
|
||||
rsa.workspace = true
|
||||
base64.workspace = true
|
||||
sha2.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
@@ -63,8 +64,6 @@ once_cell.workspace = true
|
||||
prometheus.workspace = true
|
||||
uuid.workspace = true
|
||||
gethostname.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
@@ -97,7 +96,7 @@ tower = "^0"
|
||||
tower-http = { version = "^0", features = ["trace", "cors"] }
|
||||
tower-cookies = "^0"
|
||||
serde = "^1"
|
||||
serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }
|
||||
serde_json = { version = "^1", features = ["preserve_order"] }
|
||||
uuid = { version = "^1", features = ["serde", "v4"] }
|
||||
thiserror = "^1"
|
||||
anyhow = "^1"
|
||||
@@ -113,7 +112,7 @@ mime_guess = "^2"
|
||||
hex = "^0"
|
||||
sql-builder = "^3"
|
||||
argon2 = "^0"
|
||||
quick_cache = "^0"
|
||||
retainer = "^0"
|
||||
rand = "0.8.5"
|
||||
rand_core = { version = "^0", features = ["std"] }
|
||||
magic-crypt = "^3"
|
||||
@@ -156,9 +155,7 @@ sqlx = { version = "^0", features = [
|
||||
"chrono",
|
||||
"postgres",
|
||||
"runtime-tokio-rustls",
|
||||
"bigdecimal"
|
||||
] }
|
||||
bigdecimal = "^0"
|
||||
dotenv = "^0"
|
||||
ulid = { version = "^1", features = ["uuid"] }
|
||||
futures = "^0"
|
||||
@@ -190,7 +187,6 @@ postgres-native-tls = "^0"
|
||||
native-tls = "^0"
|
||||
samael = { version = "0.0.12", features = ["xmlsec"] }
|
||||
gcp_auth = "0.9.0"
|
||||
rust_decimal = { version = "1.31.0", features = ["db-postgres"]}
|
||||
rust_decimal = {version = "1.31.0", features = ["db-postgres"]}
|
||||
jsonwebtoken = "8.3.0"
|
||||
pem = "3.0.1"
|
||||
nix = { version = "0.27.1", features = ["process", "signal"] }
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add down migration script here
|
||||
DROP TABLE worker_group_config;
|
||||
|
||||
ALTER TABLE worker_ping DROP COLUMN worker_group;
|
||||
@@ -1,8 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE TABLE worker_group_config (
|
||||
name VARCHAR(255) PRIMARY KEY,
|
||||
config JSONB DEFAULT '{}'::jsonb
|
||||
);
|
||||
|
||||
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS worker_group VARCHAR(255) NOT NULL DEFAULT 'default';
|
||||
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS dedicated_worker VARCHAR(255);
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1 +0,0 @@
|
||||
-- Add up migration script here
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,5 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE completed_job ALTER COLUMN duration_ms TYPE bigint;
|
||||
ALTER TABLE completed_job ALTER COLUMN email TYPE VARCHAR(255);
|
||||
ALTER TABLE queue ALTER COLUMN email TYPE VARCHAR(255);
|
||||
ALTER TABLE queue ALTER COLUMN canceled_by TYPE VARCHAR(255);
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) SELECT id, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{"g/all": false}' FROM workspace ON CONFLICT DO NOTHING;
|
||||
INSERT INTO resource (workspace_id, path, value, description, resource_type) SELECT id, 'f/app_themes/theme_0', '{"name": "Default Theme", "value": ""}', 'The default app theme', 'app_theme' FROM workspace ON CONFLICT DO NOTHING
|
||||
@@ -1,2 +0,0 @@
|
||||
-- Add down migration script here
|
||||
ALTER TABLE IF exists config RENAME TO worker_group_config ;
|
||||
@@ -1,29 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE IF exists worker_group_config RENAME TO config;
|
||||
UPDATE config SET name = 'worker__' || name;
|
||||
|
||||
CREATE FUNCTION "notify_config_change" ()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify('notify_config_change', NEW.name::text);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL;
|
||||
|
||||
CREATE TRIGGER "notify_config_change"
|
||||
AFTER INSERT OR UPDATE ON "config"
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION "notify_config_change" ();
|
||||
|
||||
CREATE FUNCTION "notify_global_setting_change" ()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify('notify_global_setting_change', NEW.name::text);
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE PLPGSQL;
|
||||
|
||||
CREATE TRIGGER "notify_global_setting_change"
|
||||
AFTER INSERT OR UPDATE ON "global_settings"
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION "notify_global_setting_change" ();
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add up migration script here
|
||||
INSERT INTO config (name, config) VALUES
|
||||
('worker__default', '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun"]}'),
|
||||
('worker__native', '{"worker_tags": ["nativets", "postgresql", "mysql", "graphql", "snowflake"]}') ON CONFLICT DO NOTHING;
|
||||
@@ -35,9 +35,6 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
|
||||
"dateutil" => "python-dateutil",
|
||||
"mailparser" => "mail-parser",
|
||||
"mailparser-reply" => "mail-parser-reply",
|
||||
"gitlab" => "python-gitlab",
|
||||
"smbclient" => "smbprotocol",
|
||||
"playhouse" => "peewee",
|
||||
};
|
||||
|
||||
fn replace_import(x: String) -> String {
|
||||
|
||||
@@ -1,49 +1,41 @@
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_common::error;
|
||||
|
||||
pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
|
||||
use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
|
||||
|
||||
let id = validate_license_key(license_key.clone()).await?;
|
||||
{
|
||||
let mut l = LICENSE_KEY_ID.write().await;
|
||||
*l = id.to_string()
|
||||
}
|
||||
|
||||
{
|
||||
let mut l = LICENSE_KEY.write().await;
|
||||
*l = license_key
|
||||
}
|
||||
{
|
||||
let mut l = LICENSE_KEY_VALID.write().await;
|
||||
*l = true
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
use base64::Engine;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use sha2::Sha256;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn verify_license_key() -> error::Result<()> {
|
||||
use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
|
||||
use windmill_common::error::to_anyhow;
|
||||
pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
|
||||
if let Some(license_key) = license_key {
|
||||
let mut splitted_lk = license_key.split(".");
|
||||
if splitted_lk.clone().count() != 3 {
|
||||
panic!("license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)");
|
||||
}
|
||||
let id = splitted_lk.next().unwrap();
|
||||
let expiry = splitted_lk.next().unwrap();
|
||||
let signature_b64 = splitted_lk.next().unwrap();
|
||||
|
||||
let expiry_nb = LICENSE_KEY
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.split(".")
|
||||
.nth(1)
|
||||
.unwrap_or_else(|| "")
|
||||
.parse::<u64>()
|
||||
.map_err(to_anyhow)?;
|
||||
if expiry_nb < chrono::Utc::now().timestamp() as u64 {
|
||||
tracing::error!(
|
||||
"License key expired: {} < {}",
|
||||
expiry_nb,
|
||||
chrono::Utc::now().timestamp() as u64
|
||||
);
|
||||
let mut l = LICENSE_KEY_VALID.write().await;
|
||||
*l = false;
|
||||
};
|
||||
let expiry_nb = expiry.parse::<u64>()?;
|
||||
if expiry_nb < chrono::Utc::now().timestamp() as u64 {
|
||||
panic!(
|
||||
"License key is expired (timestamp expiry: {expiry_nb}. Now: {}",
|
||||
chrono::Utc::now().timestamp()
|
||||
);
|
||||
}
|
||||
const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
|
||||
let pub_key = rsa::RsaPublicKey::from_public_key_der(
|
||||
&base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
|
||||
)?;
|
||||
let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
|
||||
rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
|
||||
.verify(
|
||||
&format!("{id}{expiry}").as_bytes(),
|
||||
&rsa::pss::Signature::from(signature),
|
||||
)
|
||||
.map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
|
||||
} else {
|
||||
panic!("License key is required for the enterprise edition");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -8,26 +8,19 @@
|
||||
|
||||
use gethostname::gethostname;
|
||||
use git_version::git_version;
|
||||
use rand::Rng;
|
||||
use sqlx::{postgres::PgListener, Pool, Postgres};
|
||||
use monitor::handle_zombie_jobs_periodically;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
fs::{metadata, DirBuilder},
|
||||
join,
|
||||
sync::RwLock,
|
||||
};
|
||||
use windmill_common::{
|
||||
global_settings::{
|
||||
BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, LICENSE_KEY_SETTING, OAUTH_SETTING,
|
||||
REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING,
|
||||
},
|
||||
utils::rd_string,
|
||||
worker::{reload_custom_tags_setting, WORKER_GROUP},
|
||||
DB, METRICS_ADDR,
|
||||
};
|
||||
use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
|
||||
use windmill_common::{global_settings::ENV_SETTINGS, utils::rd_string, METRICS_ADDR};
|
||||
use windmill_worker::{
|
||||
BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
|
||||
DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
|
||||
@@ -35,11 +28,6 @@ use windmill_worker::{
|
||||
PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
|
||||
};
|
||||
|
||||
use crate::monitor::{
|
||||
initial_load, monitor_db, reload_base_url_setting, reload_license_key,
|
||||
reload_retention_period_setting, reload_server_config, reload_worker_config,
|
||||
};
|
||||
|
||||
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
|
||||
const DEFAULT_NUM_WORKERS: usize = 1;
|
||||
const DEFAULT_PORT: u16 = 8000;
|
||||
@@ -138,9 +126,16 @@ Windmill Community Edition {GIT_VERSION}
|
||||
|
||||
display_config(&ENV_SETTINGS);
|
||||
|
||||
let worker_mode = num_workers > 0;
|
||||
|
||||
if server_mode || worker_mode {
|
||||
tracing::info!("Loading OAuth providers...: {:#?}", *OAUTH_CLIENTS);
|
||||
if let Some(ref smtp) = *SMTP_CLIENT {
|
||||
tracing::info!("Smtp client defined. Testing connection...");
|
||||
if let Err(e) = smtp.connect().await {
|
||||
tracing::error!("Failed to connect to smtp server: {}", e);
|
||||
} else {
|
||||
tracing::info!("Smtp client connected.");
|
||||
}
|
||||
}
|
||||
if server_mode || num_workers > 0 {
|
||||
let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
|
||||
|
||||
let port = if server_mode {
|
||||
@@ -149,13 +144,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
port_var.unwrap_or(0)
|
||||
};
|
||||
|
||||
// since it's only on server mode, the port is statically defined
|
||||
let base_internal_url: String = format!("http://localhost:{}", port.to_string());
|
||||
|
||||
initial_load(&db, tx.clone(), worker_mode, server_mode).await;
|
||||
|
||||
monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await;
|
||||
|
||||
if std::env::var("BASE_INTERNAL_URL").is_ok() {
|
||||
tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it.");
|
||||
}
|
||||
@@ -173,11 +161,10 @@ Windmill Community Edition {GIT_VERSION}
|
||||
let workers_f = async {
|
||||
let port = port_rx.await?;
|
||||
let base_internal_url: String = format!("http://localhost:{}", port.to_string());
|
||||
if worker_mode {
|
||||
if num_workers > 0 {
|
||||
run_workers(
|
||||
db.clone(),
|
||||
rx.resubscribe(),
|
||||
tx.clone(),
|
||||
num_workers,
|
||||
base_internal_url.clone(),
|
||||
rsmq.clone(),
|
||||
@@ -189,130 +176,29 @@ Windmill Community Edition {GIT_VERSION}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let rsmq2 = rsmq.clone();
|
||||
let monitor_f = async {
|
||||
let db = db.clone();
|
||||
let tx = tx.clone();
|
||||
let rsmq = rsmq.clone();
|
||||
|
||||
let mut rx = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
let rd_delay = rand::thread_rng().gen_range(0..30);
|
||||
tokio::spawn(async move {
|
||||
//monitor_db is applied at start, no need to apply it twice
|
||||
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
|
||||
|
||||
let mut listener = retry_listen_pg(&db).await;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(30)) => {
|
||||
monitor_db(
|
||||
&db,
|
||||
&base_internal_url,
|
||||
rsmq.clone(),
|
||||
server_mode,
|
||||
)
|
||||
.await;
|
||||
},
|
||||
notification = listener.recv() => {
|
||||
match notification {
|
||||
Ok(n) => {
|
||||
tracing::info!("Received new pg notification: {n:?}");
|
||||
match n.channel() {
|
||||
"notify_config_change" => {
|
||||
tracing::info!("Config change detected");
|
||||
match n.payload() {
|
||||
"server" if server_mode => {
|
||||
tracing::info!("Server config change detected");
|
||||
reload_server_config(&db).await;
|
||||
},
|
||||
a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
|
||||
tracing::info!("Worker config change detected");
|
||||
reload_worker_config(&db, tx.clone(), true).await;
|
||||
},
|
||||
_ => {
|
||||
()
|
||||
}
|
||||
}
|
||||
},
|
||||
"notify_global_setting_change" => {
|
||||
tracing::info!("Global setting change detected");
|
||||
match n.payload() {
|
||||
BASE_URL_SETTING => {
|
||||
tracing::info!("Base URL setting change detected");
|
||||
if let Err(e) = reload_base_url_setting(&db).await {
|
||||
tracing::error!(error = %e, "Could not reload base url setting");
|
||||
}
|
||||
},
|
||||
OAUTH_SETTING => {
|
||||
tracing::info!("OAuth setting change detected");
|
||||
if let Err(e) = reload_base_url_setting(&db).await {
|
||||
tracing::error!(error = %e, "Could not reload oauth setting");
|
||||
}
|
||||
},
|
||||
CUSTOM_TAGS_SETTING => {
|
||||
tracing::info!("Custom tags setting change detected");
|
||||
if let Err(e) = reload_custom_tags_setting(&db).await {
|
||||
tracing::error!(error = %e, "Could not reload custom tags setting");
|
||||
}
|
||||
},
|
||||
LICENSE_KEY_SETTING => {
|
||||
tracing::info!("License Key setting change detected");
|
||||
if let Err(e) = reload_license_key(&db).await {
|
||||
tracing::error!(error = %e, "Could not reload license key setting");
|
||||
}
|
||||
},
|
||||
RETENTION_PERIOD_SECS_SETTING => {
|
||||
tracing::info!("Retention period setting change detected");
|
||||
reload_retention_period_setting(&db).await
|
||||
},
|
||||
REQUEST_SIZE_LIMIT_SETTING => {
|
||||
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
|
||||
// we wait a bit randomly to avoid having all servers shutdown at same time
|
||||
let rd_delay = rand::thread_rng().gen_range(0..4);
|
||||
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
|
||||
if let Err(e) = tx.send(()) {
|
||||
tracing::error!(error = %e, "Could not send killpill to server");
|
||||
}
|
||||
}
|
||||
a @_ => {
|
||||
tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
|
||||
}
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
tracing::warn!("Unknown notification received");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
|
||||
listener = retry_listen_pg(&db).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
},
|
||||
_ = rx.recv() => {
|
||||
println!("received killpill for monitor job");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if server_mode {
|
||||
// since it's only on server mode, the port is statically defined
|
||||
let base_internal_url: String = format!("http://localhost:{}", port.to_string());
|
||||
monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2).await;
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let metrics_f = async {
|
||||
if let Some(_addr) = metrics_addr {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
panic!("Metrics are only available in the Enterprise Edition");
|
||||
match metrics_addr {
|
||||
Some(_addr) => {
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
panic!("Metrics are only available in the Enterprise Edition");
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0).await;
|
||||
#[cfg(feature = "enterprise")]
|
||||
windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
None => Ok(()),
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
|
||||
@@ -322,40 +208,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn listen_pg(db: &DB) -> Option<PgListener> {
|
||||
let mut listener = match PgListener::connect_with(&db).await {
|
||||
Ok(l) => l,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Could not connect to database");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = listener
|
||||
.listen_all(vec!["notify_config_change", "notify_global_setting_change"])
|
||||
.await
|
||||
{
|
||||
tracing::error!(error = %e, "Could not listen to database");
|
||||
return None;
|
||||
}
|
||||
|
||||
return Some(listener);
|
||||
}
|
||||
|
||||
async fn retry_listen_pg(db: &DB) -> PgListener {
|
||||
let mut listener = listen_pg(db).await;
|
||||
loop {
|
||||
if listener.is_none() {
|
||||
tracing::info!("Retrying listening to pg listen in 5 seconds");
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
listener = listen_pg(db).await;
|
||||
} else {
|
||||
tracing::info!("Successfully connected to pg listen");
|
||||
return listener.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn display_config(envs: &[&str]) {
|
||||
tracing::info!(
|
||||
"config: {}",
|
||||
@@ -373,14 +225,40 @@ fn display_config(envs: &[&str]) {
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
db: &Pool<Postgres>,
|
||||
rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let db1 = db.clone();
|
||||
let db2 = db.clone();
|
||||
|
||||
let rx2 = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
tokio::spawn(async move {
|
||||
join!(
|
||||
handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
|
||||
windmill_api::delete_expired_items_perdiodically(&db2, rx2)
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
db: Pool<Postgres>,
|
||||
rx: tokio::sync::broadcast::Receiver<()>,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
num_workers: i32,
|
||||
base_internal_url: String,
|
||||
rsmq: Option<R>,
|
||||
) -> anyhow::Result<()> {
|
||||
#[cfg(feature = "enterprise")]
|
||||
ee::verify_license_key(LICENSE_KEY.clone())?;
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) {
|
||||
panic!("License key is required ONLY for the enterprise edition");
|
||||
}
|
||||
|
||||
let instance_name = gethostname()
|
||||
.to_str()
|
||||
.map(|x| {
|
||||
@@ -442,7 +320,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
let worker_name = format!("wk-{}-{}", &instance_name, rd_string(5));
|
||||
let ip = ip.clone();
|
||||
let rx = rx.resubscribe();
|
||||
let tx = tx.clone();
|
||||
let base_internal_url = base_internal_url.clone();
|
||||
let rsmq2 = rsmq.clone();
|
||||
let sync_barrier = sync_barrier.clone();
|
||||
@@ -456,7 +333,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
num_workers as u32,
|
||||
&ip,
|
||||
rx,
|
||||
tx,
|
||||
&base_internal_url,
|
||||
rsmq2,
|
||||
sync_barrier,
|
||||
|
||||
@@ -1,41 +1,18 @@
|
||||
use std::{collections::HashMap, fmt::Display, ops::Mul, str::FromStr, sync::Arc};
|
||||
use std::time::Duration;
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use serde::de::DeserializeOwned;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{
|
||||
join,
|
||||
sync::{mpsc, RwLock},
|
||||
};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
use windmill_api::{
|
||||
oauth2::{build_oauth_clients, OAuthClient},
|
||||
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT,
|
||||
};
|
||||
use windmill_common::{
|
||||
error,
|
||||
global_settings::{
|
||||
BASE_URL_SETTING, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING,
|
||||
RETENTION_PERIOD_SECS_SETTING,
|
||||
},
|
||||
jobs::{JobKind, QueuedJob},
|
||||
server::load_server_config,
|
||||
users::truncate_token,
|
||||
worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG},
|
||||
BASE_URL, DB, METRICS_ENABLED,
|
||||
METRICS_ENABLED,
|
||||
};
|
||||
use windmill_worker::{
|
||||
create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
|
||||
};
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::ee::verify_license_key;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_api::LICENSE_KEY_VALID;
|
||||
|
||||
use crate::ee::set_license_key;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
|
||||
.ok()
|
||||
@@ -58,419 +35,29 @@ lazy_static::lazy_static! {
|
||||
"Total number of jobs deleted due to their ping timing out in an unrecoverable state."
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
static ref QUEUE_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
|
||||
"queue_count",
|
||||
"Number of jobs in the queue",
|
||||
&["tag"]
|
||||
).unwrap();
|
||||
|
||||
static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
|
||||
|
||||
}
|
||||
|
||||
pub async fn initial_load(
|
||||
db: &Pool<Postgres>,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
worker_mode: bool,
|
||||
server_mode: bool,
|
||||
) {
|
||||
let reload_worker_config_f = async {
|
||||
if worker_mode {
|
||||
reload_worker_config(&db, tx, false).await;
|
||||
}
|
||||
};
|
||||
let reload_custom_tags_f = async {
|
||||
if server_mode {
|
||||
if let Err(e) = reload_custom_tags_setting(db).await {
|
||||
tracing::error!("Error reloading custom tags: {:?}", e)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let reload_base_url_f = async {
|
||||
|
||||
if let Err(e) = reload_base_url_setting(db).await {
|
||||
tracing::error!("Error reloading base url: {:?}", e)
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
let reload_server_config_f = async {
|
||||
if server_mode {
|
||||
reload_server_config(&db).await;
|
||||
}
|
||||
};
|
||||
let reload_retention_period_f = async {
|
||||
if server_mode {
|
||||
reload_retention_period_setting(&db).await;
|
||||
}
|
||||
};
|
||||
|
||||
let reload_request_size_f = async {
|
||||
if server_mode {
|
||||
reload_request_size(&db).await;
|
||||
}
|
||||
};
|
||||
|
||||
let reload_license_key_f = async {
|
||||
if server_mode {
|
||||
#[cfg(feature = "enterprise")]
|
||||
if let Err(e) = reload_license_key(&db).await {
|
||||
tracing::error!("Error reloading license key: {:?}", e)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
join!(
|
||||
reload_worker_config_f,
|
||||
reload_server_config_f,
|
||||
reload_custom_tags_f,
|
||||
reload_request_size_f,
|
||||
reload_base_url_f,
|
||||
reload_retention_period_f,
|
||||
reload_license_key_f
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn delete_expired_items(db: &DB) -> () {
|
||||
let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
|
||||
"DELETE FROM token WHERE expiration <= now()
|
||||
RETURNING concat(substring(token for 10), '*****')",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match tokens_deleted_r {
|
||||
Ok(tokens) => {
|
||||
if tokens.len() > 0 {
|
||||
tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens)
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let pip_resolution_r = sqlx::query_scalar!(
|
||||
"DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match pip_resolution_r {
|
||||
Ok(res) => {
|
||||
if res.len() > 0 {
|
||||
tracing::info!("deleted {} pip_resolution: {:?}", res.len(), res)
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let deleted_cache = sqlx::query_scalar!(
|
||||
"DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_cache {
|
||||
Ok(res) => {
|
||||
if res.len() > 0 {
|
||||
tracing::info!("deleted {} cache resource: {:?}", res.len(), res)
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
|
||||
}
|
||||
|
||||
let job_retention_secs = *JOB_RETENTION_SECS.read().await;
|
||||
if job_retention_secs > 0 {
|
||||
let deleted_jobs = sqlx::query_scalar!(
|
||||
"DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
|
||||
job_retention_secs
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_jobs {
|
||||
Ok(deleted_jobs) => {
|
||||
if deleted_jobs.len() > 0 {
|
||||
tracing::info!(
|
||||
"deleted {} jobs completed JOB_RETENTION_SECS {} ago: {:?}",
|
||||
deleted_jobs.len(),
|
||||
job_retention_secs,
|
||||
deleted_jobs,
|
||||
)
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_retention_period_setting(db: &DB) {
|
||||
if let Err(e) = reload_setting(
|
||||
db,
|
||||
RETENTION_PERIOD_SECS_SETTING,
|
||||
"JOB_RETENTION_SECS",
|
||||
60 * 60 * 24 * 60,
|
||||
JOB_RETENTION_SECS.clone(),
|
||||
|x| x,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error reloading retention period: {:?}", e)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_request_size(db: &DB) {
|
||||
if let Err(e) = reload_setting(
|
||||
db,
|
||||
REQUEST_SIZE_LIMIT_SETTING,
|
||||
"REQUEST_SIZE_LIMIT",
|
||||
DEFAULT_BODY_LIMIT,
|
||||
REQUEST_SIZE_LIMIT.clone(),
|
||||
|x| x.mul(1024 * 1024),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error reloading retention period: {:?}", e)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_license_key(db: &DB) -> error::Result<()> {
|
||||
let q = sqlx::query!(
|
||||
"SELECT value FROM global_settings WHERE name = $1",
|
||||
LICENSE_KEY_SETTING
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let mut value = std::env::var("LICENSE_KEY")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<String>().ok())
|
||||
.unwrap_or(String::new());
|
||||
|
||||
if let Some(q) = q {
|
||||
if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
|
||||
tracing::info!(
|
||||
"Loaded setting LICENSE_KEY from db config: {}",
|
||||
truncate_token(&v)
|
||||
);
|
||||
value = v;
|
||||
} else {
|
||||
tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value);
|
||||
}
|
||||
};
|
||||
|
||||
set_license_key(value).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
|
||||
db: &DB,
|
||||
setting_name: &str,
|
||||
std_env_var: &str,
|
||||
default: T,
|
||||
lock: Arc<RwLock<T>>,
|
||||
transformer: fn(T) -> T,
|
||||
) -> error::Result<()> {
|
||||
let q = sqlx::query!(
|
||||
"SELECT value FROM global_settings WHERE name = $1",
|
||||
setting_name
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let mut value = std::env::var(std_env_var)
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<T>().ok())
|
||||
.unwrap_or(default);
|
||||
|
||||
if let Some(q) = q {
|
||||
if let Ok(v) = serde_json::from_value::<T>(q.value.clone()) {
|
||||
tracing::info!(
|
||||
"Loaded setting {setting_name} from db config: {:#?}",
|
||||
&q.value
|
||||
);
|
||||
value = transformer(v);
|
||||
} else {
|
||||
tracing::error!("Could not parse {setting_name} found: {:#?}", &q.value);
|
||||
}
|
||||
};
|
||||
|
||||
{
|
||||
let mut l = lock.write().await;
|
||||
*l = value;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
pub async fn handle_zombie_jobs_periodically<
|
||||
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
server_mode: bool,
|
||||
) {
|
||||
let zombie_jobs_f = async {
|
||||
if server_mode {
|
||||
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
|
||||
}
|
||||
};
|
||||
let expired_items_f = async {
|
||||
if server_mode {
|
||||
delete_expired_items(&db).await;
|
||||
}
|
||||
};
|
||||
loop {
|
||||
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
|
||||
|
||||
let verify_license_key_f = async {
|
||||
if server_mode {
|
||||
#[cfg(feature = "enterprise")]
|
||||
if let Err(e) = verify_license_key().await {
|
||||
tracing::error!("Error verifying license key: {:?}", e);
|
||||
let mut l = LICENSE_KEY_VALID.write().await;
|
||||
*l = false;
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(30)) => (),
|
||||
_ = rx.recv() => {
|
||||
println!("received killpill for monitor job");
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let expose_queue_metrics_f = async {
|
||||
if *METRICS_ENABLED && server_mode {
|
||||
expose_queue_metrics(&db).await;
|
||||
}
|
||||
};
|
||||
join!(
|
||||
expired_items_f,
|
||||
zombie_jobs_f,
|
||||
expose_queue_metrics_f,
|
||||
verify_license_key_f
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
|
||||
let queue_counts = sqlx::query!(
|
||||
"SELECT tag, count(*) as count FROM queue WHERE
|
||||
scheduled_for <= now() - ('3 seconds')::interval AND running = false
|
||||
GROUP BY tag"
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_else(|| vec![]);
|
||||
for q in queue_counts {
|
||||
let count = q.count.unwrap_or(0);
|
||||
let tag = q.tag;
|
||||
let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
|
||||
metric.set(count as i64);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_server_config(db: &Pool<Postgres>) {
|
||||
let config = load_server_config(&db).await;
|
||||
if let Err(e) = config {
|
||||
tracing::error!("Error reloading server config: {:?}", e)
|
||||
} else {
|
||||
let mut wc = SERVER_CONFIG.write().await;
|
||||
tracing::info!("Reloading server config...");
|
||||
*wc = config.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_worker_config(
|
||||
db: &DB,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
kill_if_change: bool,
|
||||
) {
|
||||
let config = load_worker_config(&db).await;
|
||||
if let Err(e) = config {
|
||||
tracing::error!("Error reloading worker config: {:?}", e)
|
||||
} else {
|
||||
let wc = WORKER_CONFIG.read().await;
|
||||
let config = config.unwrap();
|
||||
if *wc != config {
|
||||
if kill_if_change && (*wc).dedicated_worker != config.dedicated_worker {
|
||||
tracing::info!("Dedicated worker config changed, sending killpill. Expecting to be restarted by supervisor.");
|
||||
let _ = tx.send(());
|
||||
}
|
||||
drop(wc);
|
||||
|
||||
let mut wc = WORKER_CONFIG.write().await;
|
||||
tracing::info!("Reloading worker config...");
|
||||
*wc = config
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
|
||||
let q_base_url = sqlx::query!(
|
||||
"SELECT value FROM global_settings WHERE name = $1",
|
||||
BASE_URL_SETTING
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let base_url = if let Some(q) = q_base_url {
|
||||
if let Ok(v) = serde_json::from_value::<String>(q.value.clone()) {
|
||||
v
|
||||
} else {
|
||||
tracing::error!(
|
||||
"Could not parse base_url setting as a string, found: {:#?}",
|
||||
&q.value
|
||||
);
|
||||
std::env::var("BASE_URL")
|
||||
.ok()
|
||||
.unwrap_or_else(|| "http://localhost".to_string())
|
||||
}
|
||||
} else {
|
||||
std::env::var("BASE_URL")
|
||||
.ok()
|
||||
.unwrap_or_else(|| "http://localhost".to_string())
|
||||
};
|
||||
|
||||
let q_oauth = sqlx::query!(
|
||||
"SELECT value FROM global_settings WHERE name = $1",
|
||||
OAUTH_SETTING
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let oauths = if let Some(q) = q_oauth {
|
||||
if let Ok(v) =
|
||||
serde_json::from_value::<Option<HashMap<String, OAuthClient>>>(q.value.clone())
|
||||
{
|
||||
v
|
||||
} else {
|
||||
tracing::error!(
|
||||
"Could not parse oauth setting as a json, found: {:#?}",
|
||||
&q.value
|
||||
);
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let is_secure = base_url.starts_with("https://");
|
||||
|
||||
{
|
||||
let mut l = OAUTH_CLIENTS.write().await;
|
||||
*l = build_oauth_clients(&base_url, oauths)
|
||||
.map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let mut l = BASE_URL.write().await;
|
||||
*l = base_url
|
||||
}
|
||||
|
||||
{
|
||||
let mut l = IS_SECURE.write().await;
|
||||
*l = is_secure;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
db: &Pool<Postgres>,
|
||||
base_internal_url: &str,
|
||||
|
||||
53
backend/tests/fixtures/base.sql
vendored
53
backend/tests/fixtures/base.sql
vendored
@@ -17,6 +17,59 @@ INSERT INTO workspace_settings (workspace_id) VALUES
|
||||
|
||||
insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'
|
||||
export async function main(fail: boolean = true) {
|
||||
if (fail) {
|
||||
throw new Error("Failed")
|
||||
}
|
||||
|
||||
return "OK"
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/failing_script', -28028598712388162, 'deno', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'
|
||||
export async function main() {
|
||||
return "Error handler";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'
|
||||
export async function main() {
|
||||
return "Recovery handler";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
|
||||
|
||||
INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/failing_flow'
|
||||
);
|
||||
|
||||
GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
|
||||
GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;
|
||||
|
||||
|
||||
44
backend/tests/fixtures/relative_bun.sql
vendored
44
backend/tests/fixtures/relative_bun.sql
vendored
@@ -1,44 +0,0 @@
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
export function main() {
|
||||
return "f/system/same_folder_script";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/same_folder_script', -28028598712388162, 'bun', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
export function main() {
|
||||
return "f/system_relative/different_folder_script";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system_relative/different_folder_script', -28028598712388161, 'bun', '');
|
||||
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
import { main as test1 } from "/f/system/same_folder_script.ts";
|
||||
import { main as test2 } from "../system/same_folder_script.ts";
|
||||
import { main as test3 } from "/f/system_relative/different_folder_script.ts";
|
||||
import { main as test4 } from "./different_folder_script.ts";
|
||||
|
||||
export function main() {
|
||||
return [test1(), test2(), test3(), test4()];
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system_relative/nested_script', -28028598712388160, 'bun', '');
|
||||
44
backend/tests/fixtures/relative_deno.sql
vendored
44
backend/tests/fixtures/relative_deno.sql
vendored
@@ -1,44 +0,0 @@
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
export function main() {
|
||||
return "f/system/same_folder_script";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/same_folder_script', -28028598712388162, 'deno', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
export function main() {
|
||||
return "f/system_relative/different_folder_script";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system_relative/different_folder_script', -28028598712388161, 'deno', '');
|
||||
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
import { main as test1 } from "/f/system/same_folder_script.ts";
|
||||
import { main as test2 } from "../system/same_folder_script.ts";
|
||||
import { main as test3 } from "/f/system_relative/different_folder_script.ts";
|
||||
import { main as test4 } from "./different_folder_script.ts";
|
||||
|
||||
export function main() {
|
||||
return [test1(), test2(), test3(), test4()];
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system_relative/nested_script', -28028598712388160, 'deno', '');
|
||||
41
backend/tests/fixtures/relative_python.sql
vendored
41
backend/tests/fixtures/relative_python.sql
vendored
@@ -1,41 +0,0 @@
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
def main():
|
||||
return "f/system/same_folder_script"
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/same_folder_script', -28028598712388162, 'python3', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
def main():
|
||||
return "f/system_relative/different_folder_script"
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system_relative/different_folder_script', -28028598712388161, 'python3', '');
|
||||
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'test-user',
|
||||
'
|
||||
from f.system.same_folder_script import main as test1
|
||||
from ..system.same_folder_script import main as test2
|
||||
from f.system_relative.different_folder_script import main as test3
|
||||
from .different_folder_script import main as test4
|
||||
|
||||
def main():
|
||||
return [test1(), test2(), test3(), test4()]
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system_relative/nested_script', -28028598712388160, 'python3', '');
|
||||
52
backend/tests/fixtures/schedule.sql
vendored
52
backend/tests/fixtures/schedule.sql
vendored
@@ -1,52 +0,0 @@
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'
|
||||
export async function main(fail: boolean = true) {
|
||||
if (fail) {
|
||||
throw new Error("Failed")
|
||||
}
|
||||
|
||||
return "OK"
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/failing_script', -28028598712388162, 'deno', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'
|
||||
export async function main() {
|
||||
return "Error handler";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
|
||||
|
||||
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'
|
||||
export async function main() {
|
||||
return "Recovery handler";
|
||||
}
|
||||
',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
|
||||
|
||||
INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
|
||||
'test-workspace',
|
||||
'system',
|
||||
'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
|
||||
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
|
||||
'',
|
||||
'',
|
||||
'f/system/failing_flow'
|
||||
);
|
||||
@@ -1,81 +1,26 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use chrono::Timelike;
|
||||
use futures::StreamExt;
|
||||
|
||||
use futures::{stream, Stream};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use tokio::time::{timeout, Duration};
|
||||
|
||||
use windmill_api_client::types::{
|
||||
CreateFlowBody, RawScript
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
|
||||
use tokio::{
|
||||
sync::RwLock,
|
||||
time::{timeout, Duration},
|
||||
};
|
||||
use windmill_api::jobs::{CompletedJob, Job};
|
||||
use windmill_api_client::types::{
|
||||
CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
|
||||
};
|
||||
|
||||
use sqlx::query;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs};
|
||||
|
||||
use windmill_api_client::types::{NewScript, NewScriptLanguage};
|
||||
|
||||
use windmill_common::worker::WORKER_CONFIG;
|
||||
use windmill_common::{
|
||||
flow_status::{FlowStatus, FlowStatusModule},
|
||||
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
|
||||
jobs::{JobPayload, RawCode, JobKind},
|
||||
scripts::{ScriptLang, ScriptHash}
|
||||
jobs::{JobPayload, RawCode},
|
||||
scripts::ScriptLang,
|
||||
};
|
||||
use windmill_queue::PushIsolationLevel;
|
||||
use serde::Serialize;
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize)]
|
||||
pub struct CompletedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub started_at: chrono::DateTime<chrono::Utc>,
|
||||
pub duration_ms: i64,
|
||||
pub success: bool,
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
pub result: Option<serde_json::Value>,
|
||||
pub logs: Option<String>,
|
||||
pub deleted: bool,
|
||||
pub raw_code: Option<String>,
|
||||
pub canceled: bool,
|
||||
pub canceled_by: Option<String>,
|
||||
pub canceled_reason: Option<String>,
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
pub is_skipped: bool,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
pub mem_peak: Option<i32>,
|
||||
pub tag: String,
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
pub language: Option<ScriptLang>,
|
||||
pub job_kind: JobKind,
|
||||
|
||||
}
|
||||
|
||||
impl CompletedJob {
|
||||
pub fn json_result(&self) -> Option<serde_json::Value> {
|
||||
self.result.clone()
|
||||
}
|
||||
}
|
||||
use windmill_queue::{get_queued_job, PushIsolationLevel};
|
||||
|
||||
async fn initialize_tracing() {
|
||||
use std::sync::Once;
|
||||
@@ -109,6 +54,37 @@ fn next_worker_name() -> String {
|
||||
format!("{id}/{thread_name}")
|
||||
}
|
||||
|
||||
pub async fn get_job_by_id<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
id: Uuid,
|
||||
) -> windmill_common::error::Result<(Option<Job>, Transaction<'c, Postgres>)> {
|
||||
let cjob_option = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
let job_option = match cjob_option {
|
||||
Some(job) => Some(Job::CompletedJob(job)),
|
||||
None => get_queued_job(id, w_id, &mut tx).await?.map(Job::QueuedJob),
|
||||
};
|
||||
if job_option.is_some() {
|
||||
Ok((job_option, tx))
|
||||
} else {
|
||||
// check if a job had been moved in-between queries
|
||||
let cjob_option = sqlx::query_as::<_, CompletedJob>(
|
||||
"SELECT * FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await?;
|
||||
Ok((cjob_option.map(Job::CompletedJob), tx))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ApiServer {
|
||||
pub addr: std::net::SocketAddr,
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
@@ -139,22 +115,21 @@ impl ApiServer {
|
||||
}
|
||||
|
||||
async fn close(self) -> anyhow::Result<()> {
|
||||
println!("closing api server");
|
||||
let Self { tx, task, .. } = self;
|
||||
drop(tx);
|
||||
task.await.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
// async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
|
||||
// tracing::info!(
|
||||
// "{:#?}",
|
||||
// get_job_by_id(db.begin().await?, "test-workspace", id)
|
||||
// .await?
|
||||
// .0
|
||||
// );
|
||||
// Ok(())
|
||||
// }
|
||||
async fn _print_job(id: Uuid, db: &Pool<Postgres>) -> Result<(), anyhow::Error> {
|
||||
tracing::info!(
|
||||
"{:#?}",
|
||||
get_job_by_id(db.begin().await?, "test-workspace", id)
|
||||
.await?
|
||||
.0
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_module(cjob: &CompletedJob, id: &str) -> Option<FlowStatusModule> {
|
||||
cjob.flow_status.clone().and_then(|fs| {
|
||||
@@ -324,7 +299,7 @@ mod suspend_resume {
|
||||
|
||||
server.close().await.unwrap();
|
||||
|
||||
let result = completed_job(flow, &db).await.json_result().unwrap();
|
||||
let result = completed_job(flow, &db).await.result.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
json!({
|
||||
@@ -362,7 +337,7 @@ mod suspend_resume {
|
||||
.arg("port", json!(port))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
server.close().await.unwrap();
|
||||
@@ -425,7 +400,7 @@ mod suspend_resume {
|
||||
|
||||
server.close().await.unwrap();
|
||||
|
||||
let result = completed_job(flow, &db).await.json_result().unwrap();
|
||||
let result = completed_job(flow, &db).await.result.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
json!( {"error": {"name": "Canceled", "reason": "approval request disapproved", "message": "Job canceled: approval request disapproved by unknown", "canceler": "unknown"}}),
|
||||
@@ -580,7 +555,7 @@ def main(last, port):
|
||||
.arg("port", json!(server.addr.port()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(server.close().await, attempts);
|
||||
@@ -609,7 +584,7 @@ def main(last, port):
|
||||
.arg("port", json!(server.addr.port()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(server.close().await, attempts);
|
||||
@@ -651,7 +626,7 @@ def main(last, port):
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await;
|
||||
|
||||
let result = job.json_result().unwrap();
|
||||
let result = job.result.unwrap();
|
||||
assert_eq!(server.close().await, attempts);
|
||||
assert!(result["error"]
|
||||
.as_object()
|
||||
@@ -715,7 +690,7 @@ def main(error, port):
|
||||
.arg("port", json!(server.addr.port()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await;
|
||||
let result = cjob.json_result().clone().unwrap();
|
||||
let result = cjob.result.clone().unwrap();
|
||||
let failed_module = get_module(&cjob, "a").unwrap();
|
||||
match failed_module {
|
||||
FlowStatusModule::Failure { .. } => {}
|
||||
@@ -773,7 +748,7 @@ async fn test_iteration(db: Pool<Postgres>) {
|
||||
.arg("items", json!([]))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!([]));
|
||||
|
||||
@@ -782,7 +757,7 @@ async fn test_iteration(db: Pool<Postgres>) {
|
||||
.arg("items", json!((0..257).collect::<Vec<_>>()))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert!(matches!(result, serde_json::Value::Array(_)));
|
||||
assert!(result[2]["error"]
|
||||
@@ -830,7 +805,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
|
||||
.arg("items", json!([]))
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!([]));
|
||||
|
||||
@@ -840,7 +815,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await;
|
||||
// println!("{:#?}", job);
|
||||
let result = job.json_result().unwrap();
|
||||
let result = job.result.unwrap();
|
||||
assert!(matches!(result, serde_json::Value::Array(_)));
|
||||
assert!(result[2]["error"]
|
||||
.as_object()
|
||||
@@ -880,7 +855,7 @@ impl RunJob {
|
||||
args,
|
||||
/* user */ "test-user",
|
||||
/* email */ "test@windmill.dev",
|
||||
/* permissioned_as */ "u/test-user".to_string(),
|
||||
/* permissioned_as */ "u/admin".to_string(),
|
||||
/* scheduled_for_o */ None,
|
||||
/* schedule_path */ None,
|
||||
/* parent_job */ None,
|
||||
@@ -906,8 +881,7 @@ impl RunJob {
|
||||
let uuid = self.push(db).await;
|
||||
let listener = listen_for_completed_jobs(db).await;
|
||||
in_test_worker(db, listener.find(&uuid), port).await;
|
||||
let r = completed_job(uuid, db).await;
|
||||
r
|
||||
completed_job(uuid, db).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -942,12 +916,13 @@ async fn in_test_worker<Fut: std::future::Future>(
|
||||
};
|
||||
|
||||
/* ensure the worker quits before we return */
|
||||
quit.send(()).expect("send");
|
||||
drop(quit);
|
||||
|
||||
let _: () = worker
|
||||
.await
|
||||
.expect("worker timed out")
|
||||
.expect("worker panicked");
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
@@ -971,13 +946,8 @@ fn spawn_test_worker(
|
||||
let worker_name: String = next_worker_name();
|
||||
let ip: &str = Default::default();
|
||||
|
||||
let tx2 = tx.clone();
|
||||
let future = async move {
|
||||
let base_internal_url = format!("http://localhost:{}", port);
|
||||
{
|
||||
let mut wc = WORKER_CONFIG.write().await;
|
||||
(*wc).worker_tags = windmill_common::worker::DEFAULT_TAGS.clone();
|
||||
}
|
||||
windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
|
||||
&db,
|
||||
worker_instance,
|
||||
@@ -986,7 +956,6 @@ fn spawn_test_worker(
|
||||
1,
|
||||
ip,
|
||||
rx,
|
||||
tx2,
|
||||
&base_internal_url,
|
||||
None,
|
||||
Arc::new(RwLock::new(None)),
|
||||
@@ -1026,8 +995,8 @@ async fn listen_for_uuid_on(
|
||||
}
|
||||
|
||||
async fn completed_job(uuid: Uuid, db: &Pool<Postgres>) -> CompletedJob {
|
||||
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1").bind(uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.unwrap()
|
||||
@@ -1137,7 +1106,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
println!("deno flow iteration: {}", i);
|
||||
let job = run_job_in_new_worker_until_complete(&db, job.clone(), port).await;
|
||||
// println!("job: {:#?}", job.flow_status);
|
||||
let result = job.json_result().unwrap();
|
||||
let result = job.result.unwrap();
|
||||
assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {}", i);
|
||||
}
|
||||
}
|
||||
@@ -1173,7 +1142,7 @@ async fn test_identity(db: Pool<Postgres>) {
|
||||
let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None })
|
||||
.run_until_complete(&db, server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!(42));
|
||||
}
|
||||
@@ -1362,7 +1331,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job.clone(), server.addr.port())
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
@@ -1417,7 +1386,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) {
|
||||
let job = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, job.clone(), port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(result, serde_json::json!([[42]]));
|
||||
}
|
||||
@@ -1462,7 +1431,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
|
||||
.arg("n", json!(123))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(json!("last step saw 123"), result);
|
||||
|
||||
@@ -1471,7 +1440,7 @@ async fn test_stop_after_if(db: Pool<Postgres>) {
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
|
||||
let result = cjob.json_result().unwrap();
|
||||
let result = cjob.result.unwrap();
|
||||
assert_eq!(json!(-123), result);
|
||||
}
|
||||
|
||||
@@ -1520,7 +1489,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
|
||||
.arg("n", json!(123))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(json!("last step saw [123]"), result);
|
||||
|
||||
@@ -1529,7 +1498,7 @@ async fn test_stop_after_if_nested(db: Pool<Postgres>) {
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
|
||||
let result = cjob.json_result().unwrap();
|
||||
let result = cjob.result.unwrap();
|
||||
assert_eq!(json!([-123]), result);
|
||||
}
|
||||
|
||||
@@ -1583,7 +1552,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
|
||||
port,
|
||||
)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([2, 4, 6]), "iteration: {i}");
|
||||
@@ -1618,7 +1587,7 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
|
||||
port,
|
||||
)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("Hello"), "iteration: {i}");
|
||||
@@ -1655,7 +1624,7 @@ func main(derp string) (string, error) {
|
||||
.arg("derp", json!("world"))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("hello world"));
|
||||
@@ -1685,7 +1654,8 @@ echo "hello $msg"
|
||||
.arg("msg", json!("world"))
|
||||
.run_until_complete(&db, port)
|
||||
.await;
|
||||
assert_eq!(job.json_result(), Some(json!("hello world")));
|
||||
|
||||
assert_eq!(job.result, Some(json!("hello world")));
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
@@ -1712,7 +1682,7 @@ def main():
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("hello world"));
|
||||
@@ -1745,7 +1715,7 @@ def main():
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(3));
|
||||
@@ -1777,7 +1747,7 @@ def main():
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!("test-workspace"));
|
||||
@@ -1833,7 +1803,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(0));
|
||||
@@ -1873,7 +1843,7 @@ async fn test_invalid_first_step(db: Pool<Postgres>) {
|
||||
let job = run_job_in_new_worker_until_complete(&db, flow, port).await;
|
||||
|
||||
assert_eq!(
|
||||
job.json_result().unwrap(),
|
||||
job.result.unwrap(),
|
||||
serde_json::json!( {"error": {"name": "InternalErr", "message": "Expected an array value, found: {}"}})
|
||||
);
|
||||
}
|
||||
@@ -1914,7 +1884,7 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([]));
|
||||
@@ -1969,7 +1939,7 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(9));
|
||||
@@ -2037,7 +2007,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([1, 2]));
|
||||
@@ -2073,7 +2043,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([1, 3]));
|
||||
@@ -2111,7 +2081,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
|
||||
@@ -2148,7 +2118,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
|
||||
@@ -2195,7 +2165,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2232,7 +2202,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
@@ -2296,7 +2266,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!([1, 2, 3]));
|
||||
@@ -2353,7 +2323,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) {
|
||||
let flow = JobPayload::RawFlow { value: flow, path: None };
|
||||
let result = run_job_in_new_worker_until_complete(&db, flow, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
println!("{:#?}", result);
|
||||
@@ -2418,7 +2388,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(0))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert!(result["from failure module"]["error"]
|
||||
@@ -2434,7 +2404,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(1))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert!(result["from failure module"]["error"]
|
||||
@@ -2450,7 +2420,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(2))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
|
||||
assert!(result["from failure module"]["error"]
|
||||
@@ -2466,7 +2436,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
.arg("n", json!(3))
|
||||
.run_until_complete(&db, port)
|
||||
.await
|
||||
.json_result()
|
||||
.result
|
||||
.unwrap();
|
||||
assert_eq!(json!({ "l": [0, 1, 2] }), result);
|
||||
}
|
||||
@@ -2616,9 +2586,7 @@ async fn test_rust_client(db: Pool<Postgres>) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[sqlx::test(fixtures("base", "schedule"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await;
|
||||
@@ -2674,7 +2642,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2733,7 +2702,8 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2751,8 +2721,7 @@ async fn test_script_schedule_handlers(db: Pool<Postgres>) {
|
||||
}
|
||||
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
#[sqlx::test(fixtures("base", "schedule"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await;
|
||||
@@ -2809,7 +2778,8 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2869,7 +2839,8 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
|
||||
let uuid = uuid.unwrap().unwrap();
|
||||
|
||||
let completed_job =
|
||||
query!("SELECT script_path FROM completed_job WHERE id = $1", uuid)
|
||||
sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
|
||||
.bind(uuid)
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -2885,203 +2856,3 @@ async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
|
||||
async fn run_deployed_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await;
|
||||
let port = server.addr.port();
|
||||
let client = windmill_api_client::create_client(
|
||||
&format!("http://localhost:{port}"),
|
||||
"SECRET_TOKEN".to_string(),
|
||||
);
|
||||
|
||||
client.create_script(
|
||||
"test-workspace",
|
||||
&NewScript {
|
||||
language: NewScriptLanguage::from_str(language.as_str()).unwrap(),
|
||||
content: script_content,
|
||||
path: "f/system/test_import".to_string(),
|
||||
concurrent_limit: vec![],
|
||||
concurrency_time_window_s: vec![],
|
||||
cache_ttl: None,
|
||||
dedicated_worker: None,
|
||||
description: "".to_string(),
|
||||
draft_only: None,
|
||||
envs: vec![],
|
||||
is_template: None,
|
||||
kind: None,
|
||||
parent_hash: None,
|
||||
lock: vec![],
|
||||
summary: "".to_string(),
|
||||
tag: None,
|
||||
schema: std::collections::HashMap::new(),
|
||||
},
|
||||
).await.unwrap();
|
||||
|
||||
let mut completed = listen_for_completed_jobs(&db).await;
|
||||
let db2 = db.clone();
|
||||
in_test_worker(&db, async move {
|
||||
completed.next().await; // deployed script
|
||||
|
||||
let script =
|
||||
query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string())
|
||||
.fetch_one(&db2)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let job = RunJob::from(JobPayload::ScriptHash {
|
||||
path: "f/system/test_import".to_string(),
|
||||
hash: ScriptHash(script.hash),
|
||||
concurrent_limit: None,
|
||||
concurrency_time_window_s: None,
|
||||
cache_ttl: None,
|
||||
dedicated_worker: None,
|
||||
language,
|
||||
}).push(&db2).await;
|
||||
|
||||
|
||||
completed.next().await; // completed job
|
||||
|
||||
|
||||
let result = completed_job(job, &db2).await.json_result().unwrap();
|
||||
|
||||
|
||||
assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
|
||||
|
||||
}, port).await;
|
||||
}
|
||||
|
||||
|
||||
async fn run_preview_relative_imports(db: &Pool<Postgres>, script_content: String, language: ScriptLang) {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await;
|
||||
let port = server.addr.port();
|
||||
|
||||
let mut completed = listen_for_completed_jobs(&db).await;
|
||||
let db2 = db.clone();
|
||||
in_test_worker(&db, async move {
|
||||
let job = RunJob::from(JobPayload::Code(RawCode {
|
||||
content: script_content,
|
||||
path: Some("f/system/test_import".to_string()),
|
||||
language,
|
||||
lock: None,
|
||||
concurrent_limit: None,
|
||||
concurrency_time_window_s: None,
|
||||
cache_ttl: None,
|
||||
})).push(&db2).await;
|
||||
|
||||
|
||||
completed.next().await; // completed job
|
||||
|
||||
|
||||
let result = completed_job(job, &db2).await.json_result().unwrap();
|
||||
|
||||
|
||||
assert_eq!(result, serde_json::json!(["f/system/same_folder_script", "f/system/same_folder_script", "f/system_relative/different_folder_script", "f/system_relative/different_folder_script"]));
|
||||
|
||||
}, port).await;
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base", "relative_bun"))]
|
||||
async fn test_relative_imports_bun(db: Pool<Postgres>) {
|
||||
let content = r#"
|
||||
import { main as test1 } from "/f/system/same_folder_script.ts";
|
||||
import { main as test2 } from "./same_folder_script.ts";
|
||||
import { main as test3 } from "/f/system_relative/different_folder_script.ts";
|
||||
import { main as test4 } from "../system_relative/different_folder_script.ts";
|
||||
|
||||
export async function main() {
|
||||
return [test1(), test2(), test3(), test4()];
|
||||
}
|
||||
"#.to_string();
|
||||
|
||||
run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
|
||||
run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base", "relative_bun"))]
|
||||
async fn test_nested_imports_bun(db: Pool<Postgres>) {
|
||||
|
||||
let content = r#"
|
||||
import { main as test } from "/f/system_relative/nested_script.ts";
|
||||
|
||||
export async function main() {
|
||||
return test();
|
||||
}
|
||||
"#.to_string();
|
||||
|
||||
run_deployed_relative_imports(&db, content.clone(), ScriptLang::Bun).await;
|
||||
run_preview_relative_imports(&db, content, ScriptLang::Bun).await;
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base", "relative_deno"))]
|
||||
async fn test_relative_imports_deno(db: Pool<Postgres>) {
|
||||
let content = r#"
|
||||
import { main as test1 } from "/f/system/same_folder_script.ts";
|
||||
import { main as test2 } from "./same_folder_script.ts";
|
||||
import { main as test3 } from "/f/system_relative/different_folder_script.ts";
|
||||
import { main as test4 } from "../system_relative/different_folder_script.ts";
|
||||
|
||||
export async function main() {
|
||||
return [test1(), test2(), test3(), test4()];
|
||||
}
|
||||
"#.to_string();
|
||||
|
||||
run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
|
||||
run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
|
||||
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base", "relative_deno"))]
|
||||
async fn test_nested_imports_deno(db: Pool<Postgres>) {
|
||||
|
||||
let content = r#"
|
||||
import { main as test } from "/f/system_relative/nested_script.ts";
|
||||
|
||||
export async function main() {
|
||||
return test();
|
||||
}
|
||||
"#.to_string();
|
||||
|
||||
run_deployed_relative_imports(&db, content.clone(), ScriptLang::Deno).await;
|
||||
run_preview_relative_imports(&db, content, ScriptLang::Deno).await;
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base", "relative_python"))]
|
||||
async fn test_relative_imports_python(db: Pool<Postgres>) {
|
||||
let content = r#"
|
||||
from f.system.same_folder_script import main as test1
|
||||
from .same_folder_script import main as test2
|
||||
from f.system_relative.different_folder_script import main as test3
|
||||
from ..system_relative.different_folder_script import main as test4
|
||||
|
||||
def main():
|
||||
return [test1(), test2(), test3(), test4()]
|
||||
"#.to_string();
|
||||
|
||||
run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
|
||||
run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
|
||||
|
||||
}
|
||||
|
||||
|
||||
#[sqlx::test(fixtures("base", "relative_python"))]
|
||||
async fn test_nested_imports_python(db: Pool<Postgres>) {
|
||||
|
||||
let content = r#"
|
||||
|
||||
from f.system_relative.nested_script import main as test
|
||||
|
||||
def main():
|
||||
return test()
|
||||
"#.to_string();
|
||||
|
||||
run_deployed_relative_imports(&db, content.clone(), ScriptLang::Python3).await;
|
||||
run_preview_relative_imports(&db, content, ScriptLang::Python3).await;
|
||||
}
|
||||
@@ -53,7 +53,7 @@ cron.workspace = true
|
||||
mime_guess.workspace = true
|
||||
rust-embed.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
quick_cache.workspace = true
|
||||
retainer.workspace = true
|
||||
rand.workspace = true
|
||||
time.workspace = true
|
||||
magic-crypt.workspace = true
|
||||
@@ -74,4 +74,3 @@ bytes.workspace = true
|
||||
mail-send.workspace = true
|
||||
samael = { workspace = true, optional = true }
|
||||
async-recursion.workspace = true
|
||||
rsa.workspace = true
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.177.0
|
||||
version: 1.168.3
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -575,81 +575,6 @@ paths:
|
||||
application/json:
|
||||
schema: {}
|
||||
|
||||
/settings/test_smtp:
|
||||
post:
|
||||
summary: test smtp
|
||||
operationId: testSmtp
|
||||
tags:
|
||||
- setting
|
||||
requestBody:
|
||||
description: test smtp payload
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
to:
|
||||
type: string
|
||||
smtp:
|
||||
type: object
|
||||
properties:
|
||||
host:
|
||||
type: string
|
||||
username:
|
||||
type: string
|
||||
password:
|
||||
type: string
|
||||
port:
|
||||
type: integer
|
||||
from:
|
||||
type: string
|
||||
tls_implicit:
|
||||
type: boolean
|
||||
required:
|
||||
- host
|
||||
- username
|
||||
- password
|
||||
- port
|
||||
- from
|
||||
- tls_implicit
|
||||
required:
|
||||
- to
|
||||
- smtp
|
||||
responses:
|
||||
"200":
|
||||
description: status
|
||||
content:
|
||||
text/plain::
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/settings/test_license_key:
|
||||
post:
|
||||
summary: test license key
|
||||
operationId: testLicenseKey
|
||||
tags:
|
||||
- setting
|
||||
requestBody:
|
||||
description: test license key
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
license_key:
|
||||
type: string
|
||||
required:
|
||||
- license_key
|
||||
responses:
|
||||
"200":
|
||||
description: status
|
||||
content:
|
||||
text/plain::
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/users/email:
|
||||
get:
|
||||
summary: get current user email (if logged in)
|
||||
@@ -2461,46 +2386,6 @@ paths:
|
||||
- content
|
||||
- language
|
||||
|
||||
/scripts/hub/query:
|
||||
get:
|
||||
summary: query hub scripts by similarity
|
||||
operationId: queryHubScripts
|
||||
tags:
|
||||
- script
|
||||
parameters:
|
||||
- name: text
|
||||
description: query text
|
||||
in: query
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: kind
|
||||
description: query scripts kind
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
- name: limit
|
||||
description: query limit
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: number
|
||||
responses:
|
||||
"200":
|
||||
description: script details
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
|
||||
/w/{workspace}/scripts/list:
|
||||
get:
|
||||
summary: list all available scripts
|
||||
@@ -3594,22 +3479,6 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/AppWithLastVersion"
|
||||
|
||||
/w/{workspace}/apps_u/public_resource/{path}:
|
||||
get:
|
||||
summary: get public resource
|
||||
operationId: get public resource
|
||||
tags:
|
||||
- app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/Path"
|
||||
responses:
|
||||
"200":
|
||||
description: resource value
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
|
||||
/w/{workspace}/apps/secret_of/{path}:
|
||||
get:
|
||||
summary: get public secret of app
|
||||
@@ -4045,27 +3914,6 @@ paths:
|
||||
required:
|
||||
- database_length
|
||||
|
||||
/w/{workspace}/jobs/completed/count:
|
||||
get:
|
||||
summary: get completed count
|
||||
operationId: getCompletedCount
|
||||
tags:
|
||||
- job
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
responses:
|
||||
"200":
|
||||
description: completed count
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
database_length:
|
||||
type: integer
|
||||
required:
|
||||
- database_length
|
||||
|
||||
/w/{workspace}/jobs/queue/cancel_all:
|
||||
post:
|
||||
summary: cancel all jobs
|
||||
@@ -4143,7 +3991,6 @@ paths:
|
||||
- $ref: "#/components/parameters/StartedBefore"
|
||||
- $ref: "#/components/parameters/StartedAfter"
|
||||
- $ref: "#/components/parameters/CreatedOrStartedBefore"
|
||||
- $ref: "#/components/parameters/Running"
|
||||
- $ref: "#/components/parameters/CreatedOrStartedAfter"
|
||||
- $ref: "#/components/parameters/JobKinds"
|
||||
- $ref: "#/components/parameters/ArgsFilter"
|
||||
@@ -5344,80 +5191,6 @@ paths:
|
||||
items:
|
||||
$ref: "#/components/schemas/WorkerPing"
|
||||
|
||||
/configs/list_worker_groups:
|
||||
get:
|
||||
summary: list worker groups
|
||||
operationId: listWorkerGroups
|
||||
tags:
|
||||
- config
|
||||
responses:
|
||||
"200":
|
||||
description: a list of worker group configs
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
config: {}
|
||||
required:
|
||||
- name
|
||||
- config
|
||||
|
||||
/configs/get/{name}:
|
||||
get:
|
||||
summary: get config
|
||||
operationId: get config
|
||||
tags:
|
||||
- config
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/Name"
|
||||
responses:
|
||||
"200":
|
||||
description: a config
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
|
||||
/configs/update/{name}:
|
||||
post:
|
||||
summary: Update config
|
||||
operationId: updateConfig
|
||||
tags:
|
||||
- config
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/Name"
|
||||
requestBody:
|
||||
description: worker group
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: {}
|
||||
responses:
|
||||
"200":
|
||||
description: Update a worker group
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
delete:
|
||||
summary: Delete Config
|
||||
operationId: deleteConfig
|
||||
tags:
|
||||
- config
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/Name"
|
||||
responses:
|
||||
"200":
|
||||
description: Delete config
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/acls/get/{kind}/{path}:
|
||||
get:
|
||||
summary: get granular acls
|
||||
@@ -7338,8 +7111,6 @@ components:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
worker_group:
|
||||
type: string
|
||||
required:
|
||||
- worker
|
||||
- worker_instance
|
||||
@@ -7347,7 +7118,6 @@ components:
|
||||
- started_at
|
||||
- ip
|
||||
- jobs_executed
|
||||
- worker_group
|
||||
|
||||
UserWorkspaceList:
|
||||
type: object
|
||||
|
||||
@@ -57,7 +57,6 @@ pub fn unauthed_service() -> Router {
|
||||
Router::new()
|
||||
.route("/execute_component/*path", post(execute_component))
|
||||
.route("/public_app/:secret", get(get_public_app_by_secret))
|
||||
.route("/public_resource/*path", get(get_public_resource))
|
||||
}
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
@@ -342,27 +341,6 @@ async fn get_public_app_by_secret(
|
||||
Ok(Json(app))
|
||||
}
|
||||
|
||||
async fn get_public_resource(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<Option<serde_json::Value>> {
|
||||
let path = path.to_path();
|
||||
if !path.starts_with("f/app_themes/") {
|
||||
return Err(Error::BadRequest(
|
||||
"Only app themes are public resources".to_string(),
|
||||
));
|
||||
}
|
||||
let res = sqlx::query_scalar!(
|
||||
"SELECT value from resource WHERE path = $1 AND workspace_id = $2",
|
||||
path.to_owned(),
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.flatten();
|
||||
Ok(Json(res))
|
||||
}
|
||||
|
||||
async fn get_secret_id(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -530,7 +508,6 @@ pub async fn get_hub_app_by_id(
|
||||
&format!("https://hub.windmill.dev/apps/{id}/json"),
|
||||
&email,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
routing::{get, post, put},
|
||||
Router,
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
use serde::Deserialize;
|
||||
@@ -21,7 +21,7 @@ use windmill_common::{
|
||||
|
||||
use crate::{
|
||||
db::{ApiAuthed, DB},
|
||||
jobs::{add_include_headers, add_raw_string, JsonOrForm},
|
||||
jobs::add_include_headers,
|
||||
};
|
||||
|
||||
const KEEP_LAST: i64 = 8;
|
||||
@@ -95,12 +95,11 @@ pub async fn update_payload(
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<IncludeHeaderQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args, raw_string): JsonOrForm,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> Result<StatusCode> {
|
||||
let mut tx = db.begin().await?;
|
||||
let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
|
||||
let args = add_raw_string(raw_string, args);
|
||||
|
||||
let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE capture
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2022
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path},
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{self},
|
||||
DB,
|
||||
};
|
||||
|
||||
use crate::{db::ApiAuthed, utils::require_super_admin};
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list_worker_groups", get(list_worker_groups))
|
||||
.route("/update/:name", post(update_config).delete(delete_config))
|
||||
.route("/get/:name", get(get_config))
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, FromRow)]
|
||||
struct Config {
|
||||
name: String,
|
||||
config: serde_json::Value,
|
||||
}
|
||||
|
||||
async fn list_worker_groups(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
) -> error::JsonResult<Vec<Config>> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
|
||||
let rows = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
async fn get_config(
|
||||
authed: ApiAuthed,
|
||||
Path(name): Path<String>,
|
||||
Extension(db): Extension<DB>,
|
||||
) -> error::JsonResult<Option<serde_json::Value>> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
|
||||
let config = sqlx::query_as!(Config, "SELECT * FROM config WHERE name = $1", name)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.map(|c| c.config);
|
||||
|
||||
Ok(Json(config))
|
||||
}
|
||||
|
||||
async fn update_config(
|
||||
Path(name): Path<String>,
|
||||
Extension(db): Extension<DB>,
|
||||
authed: ApiAuthed,
|
||||
Json(config): Json<serde_json::Value>,
|
||||
) -> error::Result<String> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if name.starts_with("worker__") {
|
||||
return Err(error::Error::BadRequest(
|
||||
"Worker groups configurable from UI available only in the enterprise version"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = $2",
|
||||
&name,
|
||||
config
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
Ok(format!("Updated config {name}"))
|
||||
}
|
||||
|
||||
async fn delete_config(
|
||||
Path(name): Path<String>,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
authed: ApiAuthed,
|
||||
) -> error::Result<String> {
|
||||
let tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let deleted = sqlx::query!("DELETE FROM config WHERE name = $1 RETURNING name", name)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
|
||||
if deleted.len() == 0 {
|
||||
return Err(error::Error::NotFound(format!(
|
||||
"Config {name} not found",
|
||||
name = name
|
||||
)));
|
||||
}
|
||||
Ok(format!("Deleted config {name}"))
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
use anyhow::anyhow;
|
||||
use base64::Engine;
|
||||
use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
|
||||
use sha2::Sha256;
|
||||
|
||||
pub async fn validate_license_key(license_key: String) -> anyhow::Result<String> {
|
||||
let mut splitted_lk = license_key.split(".");
|
||||
if splitted_lk.clone().count() != 3 {
|
||||
return Err(anyhow!(
|
||||
"license_key can be splitted with 2 . (<client id>.<expiry>.<signature>)"
|
||||
));
|
||||
}
|
||||
|
||||
let id = splitted_lk.next().unwrap();
|
||||
let expiry = splitted_lk.next().unwrap();
|
||||
let signature_b64 = splitted_lk.next().unwrap();
|
||||
|
||||
const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB";
|
||||
let pub_key = rsa::RsaPublicKey::from_public_key_der(
|
||||
&base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?,
|
||||
)?;
|
||||
let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?;
|
||||
rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
|
||||
.verify(
|
||||
&format!("{id}{expiry}").as_bytes(),
|
||||
&rsa::pss::Signature::from(signature),
|
||||
)
|
||||
.map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
|
||||
|
||||
let expiry_nb = expiry.parse::<u64>()?;
|
||||
if expiry_nb < chrono::Utc::now().timestamp() as u64 {
|
||||
tracing::error!(
|
||||
"License key expired: {} < {}",
|
||||
expiry_nb,
|
||||
chrono::Utc::now().timestamp() as u64
|
||||
);
|
||||
return Err(anyhow!("License key expired".to_string()));
|
||||
};
|
||||
|
||||
Ok(id.to_string())
|
||||
}
|
||||
@@ -160,7 +160,6 @@ pub async fn get_hub_flow_by_id(
|
||||
&format!("https://hub.windmill.dev/flows/{id}/json"),
|
||||
&email,
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.json()
|
||||
|
||||
@@ -15,7 +15,6 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
use windmill_common::{db::UserDB, users::username_to_permissioned_as};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
@@ -24,6 +23,7 @@ use windmill_common::{
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{query_scalar, FromRow, Postgres, Transaction};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -212,7 +212,6 @@ async fn _check_nb_of_groups(db: &DB) -> Result<()> {
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
async fn create_group(
|
||||
authed: ApiAuthed,
|
||||
Extension(_db): Extension<DB>,
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
use crate::{db::ApiAuthed, jobs::CompletedJob};
|
||||
use axum::{
|
||||
extract::{Path, Query},
|
||||
routing::{get, post},
|
||||
@@ -15,7 +15,7 @@ use axum::{
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sqlx::{types::Uuid, FromRow};
|
||||
use sqlx::types::Uuid;
|
||||
use std::{
|
||||
fmt::{Display, Formatter},
|
||||
vec,
|
||||
@@ -102,15 +102,6 @@ pub struct Input {
|
||||
success: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, FromRow)]
|
||||
pub struct CompletedJobMini {
|
||||
id: Uuid,
|
||||
created_at: chrono::DateTime<chrono::Utc>,
|
||||
args: Option<serde_json::Value>,
|
||||
created_by: String,
|
||||
success: bool,
|
||||
}
|
||||
|
||||
async fn get_input_history(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -123,13 +114,13 @@ async fn get_input_history(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let sql = &format!(
|
||||
"select id, created_at, created_by, args, success from completed_job \
|
||||
"select * from completed_job \
|
||||
where {} = $1 and job_kind = $2 and workspace_id = $3 \
|
||||
order by created_at desc limit $4 offset $5",
|
||||
r.runnable_type.column_name()
|
||||
);
|
||||
|
||||
let query = sqlx::query_as::<_, CompletedJobMini>(sql);
|
||||
let query = sqlx::query_as::<_, CompletedJob>(sql);
|
||||
|
||||
let query = match r.runnable_type {
|
||||
RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,8 +11,9 @@ use crate::oauth2::AllClients;
|
||||
use crate::saml::{SamlSsoLogin, ServiceProviderExt};
|
||||
use crate::scim::has_scim_token;
|
||||
use crate::tracing_init::MyOnFailure;
|
||||
use crate::workers::ALL_TAGS;
|
||||
use crate::{
|
||||
oauth2::SlackVerifier,
|
||||
oauth2::{build_oauth_clients, SlackVerifier},
|
||||
tracing_init::{MyMakeSpan, MyOnResponse},
|
||||
users::OptAuthed,
|
||||
webhook_util::WebhookShared,
|
||||
@@ -24,10 +25,9 @@ use axum::{middleware::from_extractor, routing::get, Extension, Router};
|
||||
use db::DB;
|
||||
use git_version::git_version;
|
||||
use hyper::{http, Method};
|
||||
use mail_send::SmtpClientBuilder;
|
||||
use reqwest::Client;
|
||||
use std::collections::HashMap;
|
||||
use std::{net::SocketAddr, sync::Arc};
|
||||
use tokio::sync::RwLock;
|
||||
use tower::ServiceBuilder;
|
||||
use tower_cookies::CookieManagerLayer;
|
||||
use tower_http::{
|
||||
@@ -36,18 +36,14 @@ use tower_http::{
|
||||
};
|
||||
use windmill_common::db::UserDB;
|
||||
use windmill_common::utils::rd_string;
|
||||
use windmill_common::worker::ALL_TAGS;
|
||||
use windmill_common::BASE_URL;
|
||||
|
||||
use windmill_common::error::AppError;
|
||||
|
||||
mod apps;
|
||||
mod audit;
|
||||
mod capture;
|
||||
mod configs;
|
||||
mod db;
|
||||
mod drafts;
|
||||
pub mod ee;
|
||||
mod favorite;
|
||||
mod flows;
|
||||
mod folders;
|
||||
@@ -55,7 +51,7 @@ mod granular_acls;
|
||||
mod groups;
|
||||
mod inputs;
|
||||
pub mod jobs;
|
||||
pub mod oauth2;
|
||||
mod oauth2;
|
||||
mod openai;
|
||||
mod raw_apps;
|
||||
mod resources;
|
||||
@@ -76,11 +72,16 @@ mod workspaces;
|
||||
pub const GIT_VERSION: &str =
|
||||
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
|
||||
|
||||
pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB
|
||||
pub use users::delete_expired_items_perdiodically;
|
||||
|
||||
pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
|
||||
|
||||
pub static ref REQUEST_SIZE_LIMIT: Arc<RwLock<usize>> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT));
|
||||
pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<usize>().ok())
|
||||
.unwrap_or(DEFAULT_BODY_LIMIT);
|
||||
|
||||
|
||||
pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
|
||||
@@ -89,22 +90,59 @@ lazy_static::lazy_static! {
|
||||
.ok()
|
||||
.map(|x| SlackVerifier::new(x).unwrap());
|
||||
|
||||
pub static ref IS_SECURE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
|
||||
static ref IS_SECURE: bool = BASE_URL.starts_with("https://");
|
||||
|
||||
pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
|
||||
.user_agent("windmill/beta")
|
||||
.danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())
|
||||
.build().unwrap();
|
||||
|
||||
pub static ref OAUTH_CLIENTS: Arc<RwLock<AllClients>> = Arc::new(RwLock::new(AllClients {
|
||||
logins: HashMap::new(),
|
||||
connects: HashMap::new(),
|
||||
slack: None
|
||||
}));
|
||||
pub static ref OAUTH_CLIENTS: AllClients = build_oauth_clients(&BASE_URL)
|
||||
.map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
|
||||
.unwrap();
|
||||
|
||||
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
|
||||
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
|
||||
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
|
||||
pub static ref SMTP_CLIENT: Option<SmtpClientBuilder<String>> = {
|
||||
let smtp = parse_smtp();
|
||||
if let Some(smtp) = smtp {
|
||||
match smtp {
|
||||
Ok(smtp) => Some(smtp),
|
||||
Err(e) => {
|
||||
tracing::error!("SMTP is not configured correctly, emails will not be sent: {}", e);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tracing::warn!("SMTP is not configured, emails will not be sent");
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
pub static ref SMTP_FROM: String = std::env::var("SMTP_FROM").unwrap_or_else(|_| "noreply@getwindmill.com".to_string());
|
||||
|
||||
pub static ref LICENSE_KEY: Option<String> = std::env::var("LICENSE_KEY").ok();
|
||||
}
|
||||
|
||||
pub fn parse_smtp() -> Option<windmill_common::error::Result<SmtpClientBuilder<String>>> {
|
||||
let username = std::env::var("SMTP_USERNAME").ok();
|
||||
let port = std::env::var("SMTP_PORT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse().ok())
|
||||
.unwrap_or(587);
|
||||
let password = std::env::var("SMTP_PASSWORD").ok();
|
||||
let host = std::env::var("SMTP_HOST").ok();
|
||||
let tls_implicit = std::env::var("SMTP_TLS_IMPLICIT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse().ok())
|
||||
.unwrap_or(false);
|
||||
|
||||
if username.is_some() && password.is_some() && host.is_some() {
|
||||
let smtp = SmtpClientBuilder::new(host.unwrap(), port)
|
||||
.implicit_tls(tls_implicit)
|
||||
.credentials((username.unwrap(), password.unwrap()));
|
||||
Some(Ok(smtp))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run_server(
|
||||
@@ -115,7 +153,7 @@ pub async fn run_server(
|
||||
port_tx: tokio::sync::oneshot::Sender<u16>,
|
||||
) -> anyhow::Result<()> {
|
||||
if let Some(mut rsmq) = rsmq.clone() {
|
||||
for tag in ALL_TAGS.read().await.iter() {
|
||||
for tag in ALL_TAGS.clone() {
|
||||
let r =
|
||||
rsmq_async::RsmqConnection::create_queue(&mut rsmq, &tag, None, None, None).await;
|
||||
if let Err(e) = r {
|
||||
@@ -147,9 +185,7 @@ pub async fn run_server(
|
||||
.layer(Extension(auth_cache.clone()))
|
||||
.layer(CookieManagerLayer::new())
|
||||
.layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
|
||||
.layer(DefaultBodyLimit::max(
|
||||
REQUEST_SIZE_LIMIT.read().await.clone(),
|
||||
));
|
||||
.layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT));
|
||||
|
||||
let cors = CorsLayer::new()
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
@@ -203,7 +239,6 @@ pub async fn run_server(
|
||||
.nest("/settings", settings::global_service())
|
||||
.nest("/jobs", jobs::global_root_service())
|
||||
.nest("/workers", workers::global_service())
|
||||
.nest("/configs", configs::global_service())
|
||||
.nest("/scripts", scripts::global_service())
|
||||
.nest("/groups", groups::global_service())
|
||||
.nest("/flows", flows::global_service())
|
||||
@@ -252,16 +287,15 @@ pub async fn run_server(
|
||||
|
||||
let instance_name = rd_string(5);
|
||||
|
||||
tracing::info!(addr = %addr.to_string(), instance = %instance_name, "server started listening");
|
||||
let server = axum::Server::bind(&addr).serve(app.into_make_service());
|
||||
|
||||
let port = server.local_addr().port();
|
||||
tracing::info!(
|
||||
instance = %instance_name,
|
||||
"server started on port={} and addr={}",
|
||||
port,
|
||||
server.local_addr().ip()
|
||||
);
|
||||
|
||||
port_tx
|
||||
.send(server.local_addr().port())
|
||||
.expect("Failed to send port");
|
||||
@@ -271,6 +305,8 @@ pub async fn run_server(
|
||||
println!("Graceful shutdown of server");
|
||||
});
|
||||
|
||||
tokio::spawn(async move { auth_cache.monitor().await });
|
||||
|
||||
server.await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -318,7 +354,13 @@ async fn ee_license() -> &'static str {
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn ee_license() -> String {
|
||||
LICENSE_KEY_ID.read().await.clone()
|
||||
LICENSE_KEY
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.split(".")
|
||||
.next()
|
||||
.unwrap()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
async fn openapi() -> &'static str {
|
||||
|
||||
@@ -81,7 +81,7 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/connect_slack_callback", post(connect_slack_callback))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug)]
|
||||
pub struct ClientWithScopes {
|
||||
client: OClient,
|
||||
scopes: Vec<String>,
|
||||
@@ -120,10 +120,7 @@ pub struct AllClients {
|
||||
pub slack: Option<OClient>,
|
||||
}
|
||||
|
||||
pub fn build_oauth_clients(
|
||||
base_url: &str,
|
||||
oauths_from_config: Option<HashMap<String, OAuthClient>>,
|
||||
) -> anyhow::Result<AllClients> {
|
||||
pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
|
||||
let connect_configs = serde_json::from_str::<HashMap<String, OAuthConfig>>(include_str!(
|
||||
"../../oauth_connect.json"
|
||||
))?;
|
||||
@@ -131,36 +128,26 @@ pub fn build_oauth_clients(
|
||||
"../../oauth_login.json"
|
||||
))?;
|
||||
|
||||
let oauths = if let Some(oauths) = oauths_from_config {
|
||||
oauths
|
||||
let path = "./oauth.json";
|
||||
let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
|
||||
str::from_utf8(
|
||||
&base64::engine::general_purpose::STANDARD
|
||||
.decode(e)
|
||||
.map_err(to_anyhow)?,
|
||||
)?
|
||||
.to_string()
|
||||
} else if std::path::Path::new(path).exists() {
|
||||
fs::read_to_string(path).map_err(to_anyhow)?
|
||||
} else {
|
||||
let path = "./oauth.json";
|
||||
let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
|
||||
str::from_utf8(
|
||||
&base64::engine::general_purpose::STANDARD
|
||||
.decode(e)
|
||||
.map_err(to_anyhow)?,
|
||||
)?
|
||||
.to_string()
|
||||
} else if std::path::Path::new(path).exists() {
|
||||
fs::read_to_string(path).map_err(to_anyhow)?
|
||||
} else {
|
||||
tracing::warn!("oauth.json not found, no OAuth clients loaded");
|
||||
return Ok(AllClients {
|
||||
logins: HashMap::new(),
|
||||
connects: HashMap::new(),
|
||||
slack: None,
|
||||
});
|
||||
};
|
||||
tracing::warn!("oauth.json not found, no OAuth clients loaded");
|
||||
return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
|
||||
};
|
||||
|
||||
if content.is_empty() {
|
||||
tracing::warn!("oauth.json is empty, no OAuth clients loaded");
|
||||
return Ok(AllClients {
|
||||
logins: HashMap::new(),
|
||||
connects: HashMap::new(),
|
||||
slack: None,
|
||||
});
|
||||
};
|
||||
if content.is_empty() {
|
||||
tracing::warn!("oauth.json is empty, no OAuth clients loaded");
|
||||
return Ok(AllClients { logins: HashMap::new(), connects: HashMap::new(), slack: None });
|
||||
};
|
||||
let oauths: HashMap<String, OAuthClient> =
|
||||
match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
|
||||
Ok(clients) => clients,
|
||||
Err(e) => {
|
||||
@@ -169,8 +156,7 @@ pub fn build_oauth_clients(
|
||||
}
|
||||
}
|
||||
.into_iter()
|
||||
.collect()
|
||||
};
|
||||
.collect();
|
||||
|
||||
tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", "));
|
||||
|
||||
@@ -182,7 +168,7 @@ pub fn build_oauth_clients(
|
||||
.as_ref()
|
||||
.map(|c| (x.0.clone(), (x.1, c.clone())))
|
||||
}))
|
||||
.filter_map(|(k, (client_params, config))| {
|
||||
.map(|(k, (client_params, config))| {
|
||||
let named_client = build_basic_client(
|
||||
k.clone(),
|
||||
config.clone(),
|
||||
@@ -191,25 +177,17 @@ pub fn build_oauth_clients(
|
||||
base_url,
|
||||
None,
|
||||
);
|
||||
named_client
|
||||
.map(|named_client| {
|
||||
(
|
||||
named_client.0,
|
||||
ClientWithScopes {
|
||||
client: named_client.1,
|
||||
scopes: config.scopes.unwrap_or(vec![]),
|
||||
extra_params: config.extra_params,
|
||||
extra_params_callback: config.extra_params_callback,
|
||||
allowed_domains: client_params.allowed_domains.clone(),
|
||||
userinfo_url: config.userinfo_url,
|
||||
},
|
||||
)
|
||||
})
|
||||
.map_err(|e| {
|
||||
tracing::error!("Error building oauth client {k}: {e}");
|
||||
e
|
||||
})
|
||||
.ok()
|
||||
(
|
||||
named_client.0,
|
||||
ClientWithScopes {
|
||||
client: named_client.1,
|
||||
scopes: config.scopes.unwrap_or(vec![]),
|
||||
extra_params: config.extra_params,
|
||||
extra_params_callback: config.extra_params_callback,
|
||||
allowed_domains: client_params.allowed_domains.clone(),
|
||||
userinfo_url: config.userinfo_url,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -221,7 +199,7 @@ pub fn build_oauth_clients(
|
||||
.as_ref()
|
||||
.map(|c| (x.0.clone(), (x.1, c.clone())))
|
||||
}))
|
||||
.filter_map(|(k, (client_params, config))| {
|
||||
.map(|(k, (client_params, config))| {
|
||||
let named_client = build_basic_client(
|
||||
k.clone(),
|
||||
config.clone(),
|
||||
@@ -234,61 +212,43 @@ pub fn build_oauth_clients(
|
||||
None
|
||||
},
|
||||
);
|
||||
named_client
|
||||
.map(|named_client| {
|
||||
(
|
||||
named_client.0,
|
||||
ClientWithScopes {
|
||||
client: named_client.1,
|
||||
scopes: config.scopes.unwrap_or(vec![]),
|
||||
extra_params: config.extra_params,
|
||||
extra_params_callback: config.extra_params_callback,
|
||||
allowed_domains: None,
|
||||
userinfo_url: None,
|
||||
},
|
||||
)
|
||||
})
|
||||
.map_err(|e| {
|
||||
tracing::error!("Error building oauth client {k}: {e}");
|
||||
e
|
||||
})
|
||||
.ok()
|
||||
(
|
||||
named_client.0,
|
||||
ClientWithScopes {
|
||||
client: named_client.1,
|
||||
scopes: config.scopes.unwrap_or(vec![]),
|
||||
extra_params: config.extra_params,
|
||||
extra_params_callback: config.extra_params_callback,
|
||||
allowed_domains: None,
|
||||
userinfo_url: None,
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let slack = oauths
|
||||
.get("slack")
|
||||
.map(|v| {
|
||||
build_basic_client(
|
||||
"slack".to_string(),
|
||||
OAuthConfig {
|
||||
auth_url: "https://slack.com/oauth/authorize".to_string(),
|
||||
token_url: "https://slack.com/api/oauth.access".to_string(),
|
||||
userinfo_url: None,
|
||||
scopes: None,
|
||||
extra_params: None,
|
||||
extra_params_callback: None,
|
||||
req_body_auth: None,
|
||||
},
|
||||
v.clone(),
|
||||
false,
|
||||
base_url,
|
||||
Some(format!("{base_url}/oauth/callback_slack")),
|
||||
)
|
||||
.map(|x| x.1)
|
||||
.map_err(|e| {
|
||||
tracing::error!("Error building oauth slack client: {e}");
|
||||
e
|
||||
})
|
||||
.ok()
|
||||
})
|
||||
.flatten();
|
||||
let all_clients = AllClients { logins, connects, slack };
|
||||
tracing::info!("Final oauth config: {all_clients:#?}");
|
||||
Ok(all_clients)
|
||||
let slack = oauths.get("slack").map(|v| {
|
||||
build_basic_client(
|
||||
"slack".to_string(),
|
||||
OAuthConfig {
|
||||
auth_url: "https://slack.com/oauth/authorize".to_string(),
|
||||
token_url: "https://slack.com/api/oauth.access".to_string(),
|
||||
userinfo_url: None,
|
||||
scopes: None,
|
||||
extra_params: None,
|
||||
extra_params_callback: None,
|
||||
req_body_auth: None,
|
||||
},
|
||||
v.clone(),
|
||||
false,
|
||||
base_url,
|
||||
Some(format!("{base_url}/oauth/callback_slack")),
|
||||
)
|
||||
.1
|
||||
});
|
||||
|
||||
Ok(AllClients { logins, connects, slack })
|
||||
}
|
||||
|
||||
use anyhow::anyhow;
|
||||
pub fn build_basic_client(
|
||||
name: String,
|
||||
config: OAuthConfig,
|
||||
@@ -296,11 +256,9 @@ pub fn build_basic_client(
|
||||
login: bool,
|
||||
base_url: &str,
|
||||
override_callback: Option<String>,
|
||||
) -> error::Result<(String, OClient)> {
|
||||
let auth_url = Url::parse(&config.auth_url)
|
||||
.map_err(|e| anyhow!("Invalid authorization endpoint URL: {e}"))?;
|
||||
let token_url =
|
||||
Url::parse(&config.token_url).map_err(|e| anyhow!("Invalid token endpoint URL: {e}"))?;
|
||||
) -> (String, OClient) {
|
||||
let auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL");
|
||||
let token_url = Url::parse(&config.token_url).expect("Invalid token endpoint URL");
|
||||
|
||||
let redirect_url = if login {
|
||||
format!("{base_url}/user/login_callback/{name}")
|
||||
@@ -315,12 +273,9 @@ pub fn build_basic_client(
|
||||
client.set_auth_type(AuthType::RequestBody);
|
||||
}
|
||||
client.set_client_secret(client_params.secret.clone());
|
||||
client.set_redirect_url(
|
||||
Url::parse(&redirect_url).map_err(|e| anyhow!("Invalid redirect URL: {e}"))?,
|
||||
);
|
||||
|
||||
client.set_redirect_url(Url::parse(&redirect_url).expect("Invalid redirect URL"));
|
||||
// Set up the config for the Github OAuth2 process.
|
||||
Ok((name.to_string(), client))
|
||||
(name.to_string(), client)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
@@ -359,7 +314,7 @@ async fn connect(
|
||||
cookies: Cookies,
|
||||
) -> error::Result<Redirect> {
|
||||
let mut query = query.clone();
|
||||
let connects = &OAUTH_CLIENTS.read().await.connects;
|
||||
let connects = &OAUTH_CLIENTS.connects;
|
||||
let scopes = query
|
||||
.get("scopes")
|
||||
.map(|x| x.split('+').map(|x| x.to_owned()).collect());
|
||||
@@ -375,7 +330,7 @@ async fn connect(
|
||||
cookies,
|
||||
scopes,
|
||||
extra_params,
|
||||
IS_SECURE.read().await.clone(),
|
||||
*IS_SECURE,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -386,7 +341,6 @@ struct CreateAccount {
|
||||
refresh_token: Option<String>,
|
||||
expires_in: i64,
|
||||
}
|
||||
|
||||
async fn create_account(
|
||||
authed: ApiAuthed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -451,8 +405,6 @@ struct Logins {
|
||||
async fn list_logins(Extension(sso): Extension<Arc<SamlSsoLogin>>) -> error::JsonResult<Logins> {
|
||||
Ok(Json(Logins {
|
||||
oauth: OAUTH_CLIENTS
|
||||
.read()
|
||||
.await
|
||||
.logins
|
||||
.keys()
|
||||
.map(|x| x.to_owned())
|
||||
@@ -468,7 +420,7 @@ struct ScopesAndParams {
|
||||
}
|
||||
async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>> {
|
||||
Ok(Json(
|
||||
(&OAUTH_CLIENTS.read().await.connects)
|
||||
(&OAUTH_CLIENTS.connects)
|
||||
.into_iter()
|
||||
.map(|(k, v)| {
|
||||
(
|
||||
@@ -485,8 +437,6 @@ async fn list_connects() -> error::JsonResult<HashMap<String, ScopesAndParams>>
|
||||
|
||||
async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
|
||||
let mut client = OAUTH_CLIENTS
|
||||
.read()
|
||||
.await
|
||||
.slack
|
||||
.as_ref()
|
||||
.ok_or_else(|| {
|
||||
@@ -502,7 +452,7 @@ async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
|
||||
client.add_scope("commands");
|
||||
let url = client.authorize_url(&state);
|
||||
|
||||
set_cookie(&state, cookies, IS_SECURE.read().await.clone());
|
||||
set_cookie(&state, cookies, *IS_SECURE);
|
||||
Ok(Redirect::to(url.as_str()))
|
||||
}
|
||||
|
||||
@@ -545,15 +495,8 @@ async fn disconnect_slack(
|
||||
}
|
||||
|
||||
async fn login(Path(client_name): Path<String>, cookies: Cookies) -> error::Result<Redirect> {
|
||||
let clients = &OAUTH_CLIENTS.read().await.logins;
|
||||
oauth_redirect(
|
||||
clients,
|
||||
client_name,
|
||||
cookies,
|
||||
None,
|
||||
None,
|
||||
IS_SECURE.read().await.clone(),
|
||||
)
|
||||
let clients = &OAUTH_CLIENTS.logins;
|
||||
oauth_redirect(clients, client_name, cookies, None, None, *IS_SECURE)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -588,8 +531,6 @@ pub async fn _refresh_token<'c>(
|
||||
.await?;
|
||||
let account = not_found_if_none(account, "Account", &id.to_string())?;
|
||||
let client = (&OAUTH_CLIENTS
|
||||
.read()
|
||||
.await
|
||||
.connects
|
||||
.get(&account.client)
|
||||
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
|
||||
@@ -680,10 +621,11 @@ async fn connect_callback(
|
||||
Path(client_name): Path<String>,
|
||||
Json(callback): Json<OAuthCallback>,
|
||||
) -> error::JsonResult<TokenResponse> {
|
||||
let connects = &OAUTH_CLIENTS.read().await.connects;
|
||||
let client_w_scopes = connects
|
||||
let client_w_scopes = OAUTH_CLIENTS
|
||||
.connects
|
||||
.get(&client_name)
|
||||
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
|
||||
|
||||
let client = client_w_scopes.client.to_owned();
|
||||
let extra_params = client_w_scopes.extra_params_callback.clone();
|
||||
let token_response =
|
||||
@@ -701,8 +643,6 @@ async fn connect_slack_callback(
|
||||
Json(callback): Json<OAuthCallback>,
|
||||
) -> error::Result<String> {
|
||||
let client = OAUTH_CLIENTS
|
||||
.read()
|
||||
.await
|
||||
.slack
|
||||
.as_ref()
|
||||
.ok_or_else(|| {
|
||||
@@ -917,7 +857,7 @@ async fn slack_command(
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let url = BASE_URL.read().await.clone();
|
||||
let url = BASE_URL.to_owned();
|
||||
tx.commit().await?;
|
||||
return Ok(format!(
|
||||
"Job launched. See details at {url}/run/{uuid}?workspace={}",
|
||||
@@ -939,13 +879,10 @@ async fn login_callback(
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Json(callback): Json<OAuthCallback>,
|
||||
) -> error::Result<String> {
|
||||
let client_w_config = {
|
||||
let clients = OAUTH_CLIENTS.read().await.logins.clone();
|
||||
clients
|
||||
.get(&client_name)
|
||||
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?
|
||||
.clone()
|
||||
};
|
||||
let client_w_config = &OAUTH_CLIENTS
|
||||
.logins
|
||||
.get(&client_name)
|
||||
.ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?;
|
||||
let client = client_w_config.client.to_owned();
|
||||
let token_res =
|
||||
exchange_code::<TokenResponse>(callback, &cookies, client, &HTTP_CLIENT, None).await;
|
||||
|
||||
@@ -71,7 +71,7 @@ pub async fn build_sp_extension() -> anyhow::Result<(ServiceProviderExt, SamlSso
|
||||
..ContactPerson::default()
|
||||
})
|
||||
.idp_metadata(idp_metadata)
|
||||
.acs_url(format!("{}/api/saml/acs", BASE_URL.read().await.clone()))
|
||||
.acs_url(format!("{}/api/saml/acs", *BASE_URL))
|
||||
.build()?;
|
||||
|
||||
tracing::info!("SAML Configured, sso login link at: {:?}", url);
|
||||
|
||||
@@ -97,21 +97,6 @@ async fn create_schedule(
|
||||
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if ns.on_recovery.is_some() {
|
||||
return Err(Error::BadRequest(
|
||||
"on_recovery is only available in enterprise version".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
if ns.on_failure_times.is_some() && ns.on_failure_times.unwrap() > 1 {
|
||||
return Err(Error::BadRequest(
|
||||
"on_failure with a number of times > 1 is only available in enterprise version"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
|
||||
check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
|
||||
check_flow_conflict(
|
||||
|
||||
@@ -19,7 +19,7 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use sql_builder::prelude::*;
|
||||
use sql_builder::SqlBuilder;
|
||||
@@ -41,8 +41,7 @@ use windmill_common::{
|
||||
},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{
|
||||
list_elems_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
|
||||
Pagination, StripPath,
|
||||
list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
|
||||
@@ -81,7 +80,6 @@ pub fn global_service() -> Router {
|
||||
.route("/hub/list", get(list_hub_scripts))
|
||||
.route("/hub/get/*path", get(get_hub_script_by_path))
|
||||
.route("/hub/get_full/*path", get(get_full_hub_script_by_path))
|
||||
.route("/hub/query", get(query_hub_scripts))
|
||||
}
|
||||
|
||||
pub fn global_unauthed_service() -> Router {
|
||||
@@ -211,28 +209,6 @@ async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serd
|
||||
Ok(Json(asks))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct HubScriptsQuery {
|
||||
text: String,
|
||||
kind: Option<String>,
|
||||
limit: Option<i64>,
|
||||
}
|
||||
async fn query_hub_scripts(
|
||||
ApiAuthed { email, .. }: ApiAuthed,
|
||||
Query(query): Query<HubScriptsQuery>,
|
||||
) -> JsonResult<serde_json::Value> {
|
||||
let asks = query_elems_from_hub(
|
||||
&HTTP_CLIENT,
|
||||
"https://hub.windmill.dev/scripts/query",
|
||||
&email,
|
||||
&query.text,
|
||||
&query.kind,
|
||||
&query.limit,
|
||||
)
|
||||
.await?;
|
||||
Ok(Json(asks))
|
||||
}
|
||||
|
||||
fn hash_script(ns: &NewScript) -> i64 {
|
||||
let mut dh = DefaultHasher::new();
|
||||
ns.hash(&mut dh);
|
||||
|
||||
@@ -6,11 +6,8 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::{
|
||||
db::{ApiAuthed, DB},
|
||||
ee::validate_license_key,
|
||||
utils::require_super_admin,
|
||||
};
|
||||
|
||||
@@ -20,74 +17,18 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
|
||||
use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder};
|
||||
use serde::Deserialize;
|
||||
use tokio::time::timeout;
|
||||
use windmill_common::{
|
||||
error::{self, to_anyhow, JsonResult},
|
||||
error::{self, JsonResult},
|
||||
global_settings::ENV_SETTINGS,
|
||||
server::Smtp,
|
||||
};
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/envs", get(get_local_settings))
|
||||
.route("/local", get(get_local_settings))
|
||||
.route(
|
||||
"/global/:key",
|
||||
post(set_global_setting).get(get_global_setting),
|
||||
)
|
||||
.route("/test_smtp", post(test_email))
|
||||
.route("/test_license_key", post(test_license_key))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct TestEmail {
|
||||
pub to: String,
|
||||
pub smtp: Smtp,
|
||||
}
|
||||
|
||||
pub async fn test_email(
|
||||
Extension(db): Extension<DB>,
|
||||
authed: ApiAuthed,
|
||||
Json(test_email): Json<TestEmail>,
|
||||
) -> error::Result<String> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
let smtp = test_email.smtp;
|
||||
let to = test_email.to;
|
||||
let client = SmtpClientBuilder::new(smtp.host, smtp.port)
|
||||
.implicit_tls(smtp.tls_implicit)
|
||||
.credentials((smtp.username, smtp.password));
|
||||
let message = MessageBuilder::new()
|
||||
.from(("Windmill", smtp.from.as_str()))
|
||||
.to(to.clone())
|
||||
.subject("Test email from Windmill")
|
||||
.text_body("Test email content");
|
||||
let dur = Duration::from_secs(3);
|
||||
timeout(dur, client.connect())
|
||||
.await
|
||||
.map_err(to_anyhow)?
|
||||
.map_err(to_anyhow)?
|
||||
.send(message)
|
||||
.await
|
||||
.map_err(to_anyhow)?;
|
||||
tracing::info!("Sent test email to {to}");
|
||||
Ok("Sent test email".to_string())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct TestKey {
|
||||
pub license_key: String,
|
||||
}
|
||||
|
||||
|
||||
pub async fn test_license_key(
|
||||
Extension(db): Extension<DB>,
|
||||
authed: ApiAuthed,
|
||||
Json(TestKey { license_key }): Json<TestKey>,
|
||||
) -> error::Result<String> {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
validate_license_key(license_key).await?;
|
||||
Ok("Sent test email".to_string())
|
||||
}
|
||||
|
||||
pub async fn get_local_settings(
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::{
|
||||
utils::require_super_admin,
|
||||
webhook_util::{InstanceEvent, WebhookShared},
|
||||
workspaces::invite_user_to_all_auto_invite_worspaces,
|
||||
BASE_URL, COOKIE_DOMAIN, IS_SECURE,
|
||||
BASE_URL, COOKIE_DOMAIN, IS_SECURE, SMTP_CLIENT, SMTP_FROM,
|
||||
};
|
||||
use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
|
||||
use axum::{
|
||||
@@ -32,25 +32,24 @@ use axum::{
|
||||
use hyper::{header::LOCATION, StatusCode};
|
||||
use lazy_static::lazy_static;
|
||||
use mail_send::mail_builder::MessageBuilder;
|
||||
use mail_send::SmtpClientBuilder;
|
||||
use quick_cache::sync::Cache;
|
||||
use rand::rngs::OsRng;
|
||||
use regex::Regex;
|
||||
use retainer::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use time::OffsetDateTime;
|
||||
use tower_cookies::{Cookie, Cookies};
|
||||
use tracing::{Instrument, Span};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::users::truncate_token;
|
||||
use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG};
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::{self, to_anyhow, Error, JsonResult, Result},
|
||||
users::SUPERADMIN_SECRET_EMAIL,
|
||||
utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
const TTL_TOKEN_CACHE_S: u64 = 60; // 60s
|
||||
pub const TTL_TOKEN_DB_H: u32 = 72;
|
||||
|
||||
const COOKIE_NAME: &str = "token";
|
||||
@@ -103,24 +102,19 @@ pub fn make_unauthed_service() -> Router {
|
||||
.route("/logout", get(logout))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ExpiringAuthCache {
|
||||
pub authed: ApiAuthed,
|
||||
pub expiry: Option<chrono::DateTime<chrono::Utc>>,
|
||||
}
|
||||
pub struct AuthCache {
|
||||
cache: Cache<(String, String), ExpiringAuthCache>,
|
||||
cache: Cache<(String, String), ApiAuthed>,
|
||||
db: DB,
|
||||
superadmin_secret: Option<String>,
|
||||
}
|
||||
|
||||
impl AuthCache {
|
||||
pub fn new(db: DB, superadmin_secret: Option<String>) -> Self {
|
||||
AuthCache { cache: Cache::new(300), db, superadmin_secret }
|
||||
AuthCache { cache: Cache::new(), db, superadmin_secret }
|
||||
}
|
||||
|
||||
pub async fn invalidate(&self, w_id: &str, token: String) {
|
||||
self.cache.remove(&(w_id.to_string(), token));
|
||||
self.cache.remove(&(w_id.to_string(), token)).await;
|
||||
}
|
||||
|
||||
pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
|
||||
@@ -128,17 +122,13 @@ impl AuthCache {
|
||||
w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
|
||||
token.to_string(),
|
||||
);
|
||||
let s = self.cache.get(&key).map(|c| c.to_owned());
|
||||
let s = self.cache.get(&key).await.map(|c| c.to_owned());
|
||||
match s {
|
||||
Some(ExpiringAuthCache { authed, expiry })
|
||||
if expiry.is_none() || expiry.unwrap() > chrono::Utc::now() =>
|
||||
{
|
||||
Some(authed)
|
||||
}
|
||||
_ => {
|
||||
let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>, Option<chrono::DateTime<chrono::Utc>>)>(
|
||||
a @ Some(_) => a,
|
||||
None => {
|
||||
let user_o = sqlx::query_as::<_, (Option<String>, Option<String>, bool, Option<Vec<String>>)>(
|
||||
"UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \
|
||||
OR expiration IS NULL) RETURNING owner, email, super_admin, scopes, expiration",
|
||||
OR expiration IS NULL) RETURNING owner, email, super_admin, scopes",
|
||||
)
|
||||
.bind(token)
|
||||
.fetch_optional(&self.db)
|
||||
@@ -149,7 +139,7 @@ impl AuthCache {
|
||||
if let Some(user) = user_o {
|
||||
let authed_o = {
|
||||
match user {
|
||||
(Some(owner), Some(email), super_admin, _, _) if w_id.is_some() => {
|
||||
(Some(owner), Some(email), super_admin, _) if w_id.is_some() => {
|
||||
if let Some((prefix, name)) = owner.split_once('/') {
|
||||
if prefix == "u" {
|
||||
let (is_admin, is_operator) = if super_admin {
|
||||
@@ -228,7 +218,7 @@ impl AuthCache {
|
||||
})
|
||||
}
|
||||
}
|
||||
(_, Some(email), super_admin, scopes, _) => {
|
||||
(_, Some(email), super_admin, scopes) => {
|
||||
if w_id.is_some() {
|
||||
let row_o = sqlx::query_as::<_, (String, bool, bool)>(
|
||||
"SELECT username, is_admin, operator FROM usr where email = $1 AND \
|
||||
@@ -298,10 +288,9 @@ impl AuthCache {
|
||||
}
|
||||
};
|
||||
if let Some(authed) = authed_o.as_ref() {
|
||||
self.cache.insert(
|
||||
key,
|
||||
ExpiringAuthCache { authed: authed.clone(), expiry: user.4 },
|
||||
);
|
||||
self.cache
|
||||
.insert(key, authed.clone(), Duration::from_secs(TTL_TOKEN_CACHE_S))
|
||||
.await;
|
||||
}
|
||||
authed_o
|
||||
} else if self
|
||||
@@ -325,6 +314,10 @@ impl AuthCache {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn monitor(&self) {
|
||||
self.cache.monitor(20, 0.25, Duration::from_secs(10)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn extract_token<S: Send + Sync>(parts: &mut Parts, state: &S) -> Option<String> {
|
||||
@@ -719,7 +712,7 @@ async fn list_users(
|
||||
SELECT usr.*, usage.*
|
||||
FROM usr
|
||||
, LATERAL (
|
||||
SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions
|
||||
SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0) executions
|
||||
FROM completed_job
|
||||
WHERE workspace_id = $1
|
||||
AND job_kind NOT IN ('flow', 'flowpreview')
|
||||
@@ -1226,7 +1219,13 @@ async fn decline_invite(
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap();
|
||||
pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
|
||||
|
||||
pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<u32>().ok())
|
||||
.unwrap_or(60 * 60 * 24 * 60); // 60 days
|
||||
|
||||
}
|
||||
|
||||
async fn accept_invite(
|
||||
@@ -1237,7 +1236,7 @@ async fn accept_invite(
|
||||
) -> Result<(StatusCode, String)> {
|
||||
if !VALID_USERNAME.is_match(&nu.username) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Usermame can only contain alphanumeric characters and underscores and must start with a letter"
|
||||
"Usermame can only contain alphanumeric characters and underscores"
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -1320,7 +1319,7 @@ async fn add_user_to_workspace<'c>(
|
||||
|
||||
if !VALID_USERNAME.is_match(username) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Usermame can only contain alphanumeric characters and underscores and must start with a letter"
|
||||
"Usermame can only contain alphanumeric characters and underscores"
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -1571,9 +1570,7 @@ async fn create_user(
|
||||
Log in and change your password: {}/user/login?email={}&password={}&rd=%2F%23user-settings
|
||||
|
||||
You can then join or create a workspace. Happy building!",
|
||||
BASE_URL.read().await.clone(),
|
||||
&nu.email,
|
||||
&nu.password
|
||||
*BASE_URL, &nu.email, &nu.password
|
||||
),
|
||||
&nu.email,
|
||||
);
|
||||
@@ -1593,17 +1590,13 @@ pub fn send_email_if_possible(subject: &str, content: &str, to: &str) {
|
||||
}
|
||||
|
||||
pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: &str) -> Result<()> {
|
||||
if let Some(smtp) = SERVER_CONFIG.read().await.smtp.clone() {
|
||||
let client = SmtpClientBuilder::new(smtp.host, smtp.port)
|
||||
.implicit_tls(smtp.tls_implicit)
|
||||
.credentials((smtp.username, smtp.password));
|
||||
if let Some(ref smtp) = *SMTP_CLIENT {
|
||||
let message = MessageBuilder::new()
|
||||
.from(("Windmill", smtp.from.as_str()))
|
||||
.from(("Windmill", SMTP_FROM.as_str()))
|
||||
.to(to)
|
||||
.subject(subject)
|
||||
.text_body(content);
|
||||
client
|
||||
.connect()
|
||||
smtp.connect()
|
||||
.await
|
||||
.map_err(to_anyhow)?
|
||||
.send(message)
|
||||
@@ -1949,7 +1942,7 @@ pub async fn create_session_token<'c>(
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
let mut cookie = Cookie::new(COOKIE_NAME, token.clone());
|
||||
cookie.set_secure(IS_SECURE.read().await.clone());
|
||||
cookie.set_secure(*IS_SECURE);
|
||||
cookie.set_same_site(Some(cookie::SameSite::Lax));
|
||||
cookie.set_http_only(true);
|
||||
cookie.set_path(COOKIE_PATH);
|
||||
@@ -2247,20 +2240,6 @@ pub struct LoginUserInfo {
|
||||
pub displayName: Option<String>,
|
||||
}
|
||||
|
||||
async fn _check_nb_of_user(db: &DB) -> Result<()> {
|
||||
let nb_groups =
|
||||
sqlx::query_scalar!("SELECT COUNT(*) FROM password WHERE login_type != 'password'",)
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
if nb_groups.unwrap_or(0) >= 50 {
|
||||
return Err(Error::BadRequest(
|
||||
"You have reached the maximum number of oauth users accounts (50) without an enterprise license"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub async fn login_externally(
|
||||
db: DB,
|
||||
email: &String,
|
||||
@@ -2313,10 +2292,6 @@ pub async fn login_externally(
|
||||
if (name.is_none() || name == Some(String::new())) && user.is_some() {
|
||||
name = user.clone().unwrap().displayName;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
_check_nb_of_user(&db).await?;
|
||||
|
||||
sqlx::query(&format!(
|
||||
"INSERT INTO password (email, name, company, login_type, verified) VALUES ($1, \
|
||||
$2, $3, '{}', true)",
|
||||
@@ -2365,3 +2340,78 @@ pub async fn login_externally(
|
||||
tx.commit().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_expired_items_perdiodically(
|
||||
db: &DB,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> () {
|
||||
loop {
|
||||
let tokens_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
|
||||
"DELETE FROM token WHERE expiration <= now()
|
||||
RETURNING concat(substring(token for 10), '*****')",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match tokens_deleted_r {
|
||||
Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let pip_resolution_r = sqlx::query_scalar!(
|
||||
"DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match pip_resolution_r {
|
||||
Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let deleted_cache = sqlx::query_scalar!(
|
||||
"DELETE FROM resource WHERE resource_type = 'cache' AND to_timestamp((value->>'expire')::int) < now() RETURNING path",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_cache {
|
||||
Ok(res) => tracing::debug!("deleted {} cache resource: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()),
|
||||
}
|
||||
|
||||
if *JOB_RETENTION_SECS > 0 {
|
||||
let deleted_jobs = sqlx::query_scalar!(
|
||||
"DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
|
||||
*JOB_RETENTION_SECS as i64
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_jobs {
|
||||
Ok(deleted_jobs) => {
|
||||
tracing::info!(
|
||||
"deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
|
||||
deleted_jobs.len(),
|
||||
deleted_jobs
|
||||
)
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(600)) => (),
|
||||
_ = rx. recv() => {
|
||||
println!("received killpill for delete expired tokens");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate_token(token: &str) -> String {
|
||||
let mut s = token[..10].to_owned();
|
||||
s.push_str("*****");
|
||||
s
|
||||
}
|
||||
|
||||
@@ -68,7 +68,6 @@ async fn list_contextual_variables(
|
||||
Some("u/user/triggering_flow_path".to_string()),
|
||||
Some("c".to_string()),
|
||||
)
|
||||
.await
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use quick_cache::sync::Cache;
|
||||
use serde::Serialize;
|
||||
use tokio::{select, sync::mpsc};
|
||||
use tokio::{select, sync::mpsc, time::interval};
|
||||
use windmill_common::METRICS_ENABLED;
|
||||
|
||||
use crate::db::DB;
|
||||
@@ -79,7 +78,8 @@ impl WebhookShared {
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap();
|
||||
let cache = Cache::new(100);
|
||||
let cache = retainer::Cache::new();
|
||||
let mut cache_purge_interval = interval(Duration::from_secs(30));
|
||||
|
||||
loop {
|
||||
select! {
|
||||
@@ -87,7 +87,7 @@ impl WebhookShared {
|
||||
_ = shutdown_rx.recv() => break,
|
||||
r = rx.recv() => match r {
|
||||
Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
|
||||
let webhook_opt = match cache.get(&workspace_id) {
|
||||
let url_guard = match cache.get(&workspace_id).await {
|
||||
Some(guard) => {
|
||||
guard
|
||||
},
|
||||
@@ -104,14 +104,16 @@ impl WebhookShared {
|
||||
tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}");
|
||||
continue;
|
||||
};
|
||||
cache.insert(workspace_id, webook_opt.clone());
|
||||
webook_opt
|
||||
cache.insert(workspace_id.clone(), webook_opt, Duration::from_secs(30)).await;
|
||||
cache.get(&workspace_id).await.unwrap()
|
||||
}
|
||||
};
|
||||
if let Some(url) = webhook_opt {
|
||||
let webook_opt = url_guard.value();
|
||||
if let Some(url) = webook_opt {
|
||||
let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
|
||||
let _ = client.post(url).json(&message).send().await;
|
||||
timer.map(|x| x.stop_and_record());
|
||||
drop(url_guard);
|
||||
}
|
||||
},
|
||||
Some(WebhookPayload::InstanceEvent(event)) => {
|
||||
@@ -123,6 +125,10 @@ impl WebhookShared {
|
||||
},
|
||||
None => break,
|
||||
},
|
||||
_ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
|
||||
tracing::trace!("Purging Webhook Cache");
|
||||
cache.purge(10, 0.50).await;
|
||||
},
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -12,23 +12,71 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
use windmill_common::{
|
||||
db::UserDB,
|
||||
error::JsonResult,
|
||||
utils::{paginate, Pagination},
|
||||
worker::ALL_TAGS,
|
||||
};
|
||||
|
||||
use std::collections::HashMap;
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::sync::atomic::Ordering;
|
||||
#[cfg(feature = "benchmark")]
|
||||
use windmill_queue::IDLE_WORKERS;
|
||||
|
||||
use crate::db::ApiAuthed;
|
||||
|
||||
#[cfg(not(feature = "benchmark"))]
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_worker_pings))
|
||||
.route("/custom_tags", get(get_custom_tags))
|
||||
}
|
||||
|
||||
#[cfg(feature = "benchmark")]
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/toggle", get(toggle))
|
||||
.route("/list", get(list_worker_pings))
|
||||
.route("/custom_tags", get(get_custom_tags))
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
pub static ref CUSTOM_TAGS_PER_WORKSPACE: (Vec<String>, HashMap<String, Vec<String>>) = process_custom_tags(std::env::var("CUSTOM_TAGS")
|
||||
.ok());
|
||||
|
||||
pub static ref ALL_TAGS: Vec<String> = [CUSTOM_TAGS_PER_WORKSPACE.0.clone(), CUSTOM_TAGS_PER_WORKSPACE.1.keys().map(|x| x.to_string()).collect_vec()].concat();
|
||||
|
||||
}
|
||||
|
||||
fn process_custom_tags(o: Option<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
|
||||
let regex = Regex::new(r"^(\w+)\(((?:\w+)\+?)+\)$").unwrap();
|
||||
if let Some(s) = o {
|
||||
let mut global = vec![];
|
||||
let mut specific: HashMap<String, Vec<String>> = HashMap::new();
|
||||
for e in s.split(",") {
|
||||
if let Some(cap) = regex.captures(e) {
|
||||
let tag = cap.get(1).unwrap().as_str().to_string();
|
||||
let workspaces = cap.get(2).unwrap().as_str().split("+");
|
||||
specific.insert(tag, workspaces.map(|x| x.to_string()).collect_vec());
|
||||
} else {
|
||||
global.push(e.to_string());
|
||||
}
|
||||
}
|
||||
(global, specific)
|
||||
} else {
|
||||
(vec![], HashMap::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize)]
|
||||
struct WorkerPing {
|
||||
worker: String,
|
||||
@@ -38,7 +86,6 @@ struct WorkerPing {
|
||||
ip: String,
|
||||
jobs_executed: i32,
|
||||
custom_tags: Option<Vec<String>>,
|
||||
worker_group: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -57,7 +104,7 @@ async fn list_worker_pings(
|
||||
|
||||
let rows = sqlx::query_as!(
|
||||
WorkerPing,
|
||||
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags, worker_group FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, custom_tags FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
per_page as i64,
|
||||
offset as i64
|
||||
)
|
||||
@@ -67,6 +114,12 @@ async fn list_worker_pings(
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
async fn get_custom_tags() -> Json<Vec<String>> {
|
||||
Json(ALL_TAGS.read().await.clone().into())
|
||||
#[cfg(feature = "benchmark")]
|
||||
async fn toggle(Query(query): Query<EnableWorkerQuery>) -> JsonResult<bool> {
|
||||
IDLE_WORKERS.store(query.disable, Ordering::Relaxed);
|
||||
Ok(Json(IDLE_WORKERS.load(Ordering::Relaxed)))
|
||||
}
|
||||
|
||||
async fn get_custom_tags() -> Json<Vec<String>> {
|
||||
Json(ALL_TAGS.clone())
|
||||
}
|
||||
|
||||
@@ -74,19 +74,13 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/edit_error_handler", post(edit_error_handler));
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
{
|
||||
if std::env::var("STRIPE_KEY").is_err() {
|
||||
return router;
|
||||
} else {
|
||||
tracing::info!("stripe enabled");
|
||||
|
||||
return router
|
||||
.route("/checkout", get(stripe_checkout))
|
||||
.route("/billing_portal", get(stripe_portal));
|
||||
}
|
||||
}
|
||||
tracing::info!("stripe enabled");
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
let router = router
|
||||
.route("/checkout", get(stripe_checkout))
|
||||
.route("/billing_portal", get(stripe_portal));
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
router
|
||||
}
|
||||
pub fn global_service() -> Router {
|
||||
@@ -284,9 +278,8 @@ async fn stripe_checkout(
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
|
||||
let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
|
||||
let base_url = BASE_URL.read().await.clone();
|
||||
let success_rd = format!("{}/workspace_settings/checkout?success=true", base_url);
|
||||
let failure_rd = format!("{}/workspace_settings/checkout?success=false", base_url);
|
||||
let success_rd = format!("{}/workspace_settings/checkout?success=true", *BASE_URL);
|
||||
let failure_rd = format!("{}/workspace_settings/checkout?success=false", *BASE_URL);
|
||||
let checkout_session = {
|
||||
let mut params = stripe::CreateCheckoutSession::new(&failure_rd, &success_rd);
|
||||
params.mode = Some(stripe::CheckoutSessionMode::Subscription);
|
||||
@@ -333,7 +326,7 @@ async fn stripe_portal(
|
||||
.await?
|
||||
.ok_or_else(|| Error::InternalErr(format!("no customer id for workspace {}", w_id)))?;
|
||||
let client = stripe::Client::new(std::env::var("STRIPE_KEY").expect("STRIPE_KEY"));
|
||||
let success_rd = format!("{}/workspace_settings?tab=premium", BASE_URL.read().await.clone());
|
||||
let success_rd = format!("{}/workspace_settings?tab=premium", *BASE_URL);
|
||||
let portal_session = {
|
||||
let customer_id = CustomerId::from_str(&customer_id).unwrap();
|
||||
let mut params = stripe::CreateBillingPortalSession::new(customer_id);
|
||||
@@ -846,24 +839,10 @@ async fn check_name_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str)
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
|
||||
pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN").is_ok_and(|x| x.parse::<bool>().unwrap_or(true));
|
||||
pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN").is_ok_and(|x| x.parse::<bool>().unwrap_or(false));
|
||||
|
||||
}
|
||||
|
||||
async fn _check_nb_of_workspaces(db: &DB) -> Result<()> {
|
||||
let nb_workspaces = sqlx::query_scalar!("SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false",)
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
if nb_workspaces.unwrap_or(0) >= 3 {
|
||||
return Err(Error::BadRequest(
|
||||
"You have reached the maximum number of workspaces (3 outside of default group 'admins') without an enterprise license. Archive/delete another workspace to create a new one"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
async fn create_workspace(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
@@ -873,10 +852,6 @@ async fn create_workspace(
|
||||
if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN {
|
||||
require_super_admin(&db, &authed.email).await?;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
_check_nb_of_workspaces(&db).await?;
|
||||
|
||||
let mut tx: Transaction<'_, Postgres> = db.begin().await?;
|
||||
|
||||
check_name_conflict(&mut tx, &nw.id).await?;
|
||||
@@ -951,21 +926,6 @@ async fn create_workspace(
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}') ON CONFLICT DO NOTHING",
|
||||
nw.id,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO resource (workspace_id, path, value, description, resource_type) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme') ON CONFLICT DO NOTHING",
|
||||
nw.id,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed.username,
|
||||
@@ -1110,10 +1070,6 @@ async fn delete_workspace(
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!("DELETE FROM resource_type WHERE workspace_id = $1", &w_id)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM workspace_invite WHERE workspace_id = $1",
|
||||
&w_id
|
||||
@@ -1222,7 +1178,7 @@ async fn invite_user(
|
||||
"You have been granted access to Windmill's workspace {w_id}
|
||||
|
||||
If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
|
||||
BASE_URL.read().await.clone()
|
||||
*BASE_URL
|
||||
),
|
||||
&nu.email,
|
||||
);
|
||||
@@ -1251,7 +1207,7 @@ async fn add_user(
|
||||
let mut tx = db.begin().await?;
|
||||
if !VALID_USERNAME.is_match(&nu.username) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Usermame can only contain alphanumeric characters and underscores and must start with a letter"
|
||||
"Usermame can only contain alphanumeric characters and underscores"
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -1286,7 +1242,7 @@ async fn add_user(
|
||||
"You have been granted access to Windmill's workspace {w_id} by {email}
|
||||
|
||||
If you do not have an account on {}, login with SSO or ask an admin to create an account for you.",
|
||||
BASE_URL.read().await.clone()
|
||||
*BASE_URL
|
||||
),
|
||||
&nu.email,
|
||||
);
|
||||
|
||||
@@ -44,5 +44,3 @@ reqwest = { workspace = true, optional = true }
|
||||
tracing-subscriber = { workspace = true, optional = true }
|
||||
lazy_static.workspace = true
|
||||
tracing-flame = { version = "^0", optional = true }
|
||||
itertools.workspace = true
|
||||
regex.workspace = true
|
||||
|
||||
@@ -1,10 +1,4 @@
|
||||
pub const WORKER_S3_BUCKET_SYNC: &str = "worker_s3_bucket_sync";
|
||||
pub const CUSTOM_TAGS_SETTING: &str = "custom_tags";
|
||||
pub const BASE_URL_SETTING: &str = "base_url";
|
||||
pub const OAUTH_SETTING: &str = "oauths";
|
||||
pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
|
||||
pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
|
||||
pub const LICENSE_KEY_SETTING: &str = "license_key";
|
||||
|
||||
pub const ENV_SETTINGS: [&str; 54] = [
|
||||
"DISABLE_NSJAIL",
|
||||
@@ -47,6 +41,8 @@ pub const ENV_SETTINGS: [&str; 54] = [
|
||||
"INSTANCE_EVENTS_WEBHOOK",
|
||||
"CLOUD_HOSTED",
|
||||
"GLOBAL_CACHE_INTERVAL",
|
||||
"WORKER_TAGS",
|
||||
"CUSTOM_TAGS",
|
||||
"JOB_RETENTION_SECS",
|
||||
"WAIT_RESULT_FAST_POLL_DURATION_SECS",
|
||||
"WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
|
||||
@@ -59,6 +55,4 @@ pub const ENV_SETTINGS: [&str; 54] = [
|
||||
"SMTP_TLS_IMPLICIT",
|
||||
"CREATE_WORKSPACE_REQUIRE_SUPERADMIN",
|
||||
"GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE",
|
||||
"MAX_WAIT_FOR_SIGTERM",
|
||||
"WORKER_GROUP",
|
||||
];
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{net::SocketAddr, sync::Arc};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use error::Error;
|
||||
use scripts::ScriptLang;
|
||||
@@ -24,11 +24,9 @@ pub mod more_serde;
|
||||
pub mod oauth2;
|
||||
pub mod schedule;
|
||||
pub mod scripts;
|
||||
pub mod server;
|
||||
pub mod users;
|
||||
pub mod utils;
|
||||
pub mod variables;
|
||||
pub mod worker;
|
||||
|
||||
#[cfg(feature = "tracing_init")]
|
||||
pub mod tracing_init;
|
||||
@@ -48,7 +46,7 @@ lazy_static::lazy_static! {
|
||||
.flatten()
|
||||
.flatten();
|
||||
pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
|
||||
pub static ref BASE_URL: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
|
||||
pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
|
||||
pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
}
|
||||
|
||||
@@ -79,16 +77,12 @@ pub async fn shutdown_signal(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
use tokio::sync::RwLock;
|
||||
#[cfg(feature = "prometheus")]
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
#[cfg(feature = "prometheus")]
|
||||
pub async fn serve_metrics(
|
||||
addr: SocketAddr,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
ready_worker_endpoint: bool,
|
||||
) -> JoinHandle<()> {
|
||||
) -> Result<(), hyper::Error> {
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use axum::{routing::get, Router};
|
||||
@@ -110,18 +104,13 @@ pub async fn serve_metrics(
|
||||
router
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = axum::Server::bind(&addr)
|
||||
.serve(router.into_make_service())
|
||||
.with_graceful_shutdown(async {
|
||||
rx.recv().await.ok();
|
||||
println!("Graceful shutdown of metrics");
|
||||
})
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error serving metrics: {}", e);
|
||||
}
|
||||
})
|
||||
axum::Server::bind(&addr)
|
||||
.serve(router.into_make_service())
|
||||
.with_graceful_shutdown(async {
|
||||
rx.recv().await.ok();
|
||||
println!("Graceful shutdown of metrics");
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn metrics() -> Result<String, Error> {
|
||||
|
||||
@@ -161,7 +161,6 @@ pub struct Script {
|
||||
pub concurrent_limit: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub concurrency_time_window_s: Option<i32>,
|
||||
pub dedicated_worker: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -275,7 +274,6 @@ pub async fn get_hub_script_by_path(
|
||||
&format!("https://hub.windmill.dev/raw/{path}.ts"),
|
||||
email,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.text()
|
||||
@@ -305,7 +303,6 @@ pub async fn get_full_hub_script_by_path(
|
||||
&format!("https://hub.windmill.dev/raw2/{path}"),
|
||||
email,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.json::<HubScript>()
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{error, DB};
|
||||
|
||||
#[derive(Serialize, Deserialize, PartialEq, Clone, Debug)]
|
||||
pub struct Smtp {
|
||||
pub host: String,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub port: u16,
|
||||
pub from: String,
|
||||
pub tls_implicit: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, PartialEq)]
|
||||
pub struct ServerConfigOpt {
|
||||
pub smtp_host: Option<String>,
|
||||
pub smtp_username: Option<String>,
|
||||
pub smtp_password: Option<String>,
|
||||
pub smtp_port: Option<u16>,
|
||||
pub smtp_from: Option<String>,
|
||||
pub smtp_tls_implicit: Option<bool>,
|
||||
pub timeout_wait_result: Option<u64>,
|
||||
}
|
||||
|
||||
pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
|
||||
let config: ServerConfigOpt =
|
||||
sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.flatten()
|
||||
.map(|x| serde_json::from_value(x).ok())
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
|
||||
let config_smtp = if let (Some(host), Some(username), Some(password)) =
|
||||
(config.smtp_host, config.smtp_username, config.smtp_password)
|
||||
{
|
||||
Some(Smtp {
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
tls_implicit: config.smtp_tls_implicit.unwrap_or(false),
|
||||
port: config.smtp_port.unwrap_or(587),
|
||||
from: config
|
||||
.smtp_from
|
||||
.unwrap_or_else(|| "noreply@getwindmill.com".to_string()),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let smtp = config_smtp.or(
|
||||
if let (Some(host), Some(username), Some(password)) = (
|
||||
std::env::var("SMTP_HOST").ok(),
|
||||
std::env::var("SMTP_USERNAME").ok(),
|
||||
std::env::var("SMTP_PASSWORD").ok(),
|
||||
) {
|
||||
Some(Smtp {
|
||||
host,
|
||||
username,
|
||||
password,
|
||||
tls_implicit: std::env::var("SMTP_TLS_IMPLICIT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse().ok())
|
||||
.unwrap_or(false),
|
||||
port: std::env::var("SMTP_PORT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse().ok())
|
||||
.unwrap_or(587),
|
||||
from: std::env::var("SMTP_FROM")
|
||||
.unwrap_or_else(|_| "noreply@getwindmill.com".to_string()),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
},
|
||||
);
|
||||
if smtp.is_none() {
|
||||
tracing::warn!("SMTP not configured");
|
||||
}
|
||||
|
||||
Ok(ServerConfig {
|
||||
smtp,
|
||||
timeout_wait_result: config
|
||||
.timeout_wait_result
|
||||
.ok_or(
|
||||
std::env::var("TIMEOUT_WAIT_RESULT")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<u64>().ok()),
|
||||
)
|
||||
.unwrap_or(600),
|
||||
})
|
||||
}
|
||||
|
||||
impl Default for ServerConfigOpt {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
smtp_from: None,
|
||||
smtp_host: None,
|
||||
smtp_password: None,
|
||||
smtp_port: None,
|
||||
smtp_tls_implicit: None,
|
||||
smtp_username: None,
|
||||
timeout_wait_result: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Clone, Debug)]
|
||||
pub struct ServerConfig {
|
||||
pub smtp: Option<Smtp>,
|
||||
pub timeout_wait_result: u64,
|
||||
}
|
||||
@@ -15,9 +15,3 @@ pub fn username_to_permissioned_as(user: &str) -> String {
|
||||
format!("u/{}", user)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truncate_token(token: &str) -> String {
|
||||
let mut s = token[..10].to_owned();
|
||||
s.push_str("*****");
|
||||
s
|
||||
}
|
||||
|
||||
@@ -79,32 +79,7 @@ pub async fn list_elems_from_hub(
|
||||
url: &str,
|
||||
email: &str,
|
||||
) -> Result<serde_json::Value> {
|
||||
let rows = http_get_from_hub(http_client, url, email, false, None)
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await
|
||||
.map_err(crate::error::to_anyhow)?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
#[cfg(feature = "reqwest")]
|
||||
pub async fn query_elems_from_hub(
|
||||
http_client: &reqwest::Client,
|
||||
url: &str,
|
||||
email: &str,
|
||||
query_text: &str,
|
||||
query_kind: &Option<String>,
|
||||
query_limit: &Option<i64>,
|
||||
) -> Result<serde_json::Value> {
|
||||
let mut query_params = vec![("text", query_text)];
|
||||
if let Some(query_kind) = query_kind {
|
||||
query_params.push(("kind", query_kind.as_str()));
|
||||
}
|
||||
let query_limit = query_limit.unwrap_or(0).to_string();
|
||||
if query_limit.parse::<i64>().unwrap() > 0 {
|
||||
query_params.push(("limit", query_limit.as_str()));
|
||||
}
|
||||
let rows = http_get_from_hub(http_client, url, email, false, Some(query_params))
|
||||
let rows = http_get_from_hub(http_client, url, email, false)
|
||||
.await?
|
||||
.json::<serde_json::Value>()
|
||||
.await
|
||||
@@ -118,9 +93,8 @@ pub async fn http_get_from_hub(
|
||||
url: &str,
|
||||
email: &str,
|
||||
plain: bool,
|
||||
query_params: Option<Vec<(&str, &str)>>,
|
||||
) -> Result<reqwest::Response> {
|
||||
let mut request = http_client
|
||||
let response = http_client
|
||||
.get(url)
|
||||
.header(
|
||||
"Accept",
|
||||
@@ -130,15 +104,10 @@ pub async fn http_get_from_hub(
|
||||
"application/json"
|
||||
},
|
||||
)
|
||||
.header("X-email", email);
|
||||
|
||||
if let Some(query_params) = query_params {
|
||||
for (key, value) in query_params {
|
||||
request = request.query(&[(key, value)]);
|
||||
}
|
||||
}
|
||||
|
||||
let response = request.send().await.map_err(crate::error::to_anyhow)?;
|
||||
.header("X-email", email)
|
||||
.send()
|
||||
.await
|
||||
.map_err(crate::error::to_anyhow)?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ pub struct CreateVariable {
|
||||
pub is_oauth: Option<bool>,
|
||||
}
|
||||
|
||||
pub async fn get_reserved_variables(
|
||||
pub fn get_reserved_variables(
|
||||
w_id: &str,
|
||||
token: &str,
|
||||
email: &str,
|
||||
@@ -163,7 +163,7 @@ pub async fn get_reserved_variables(
|
||||
},
|
||||
ContextualVariable {
|
||||
name: "WM_BASE_URL".to_string(),
|
||||
value: BASE_URL.read().await.clone(),
|
||||
value: BASE_URL.clone(),
|
||||
description: "base url of this instance".to_string(),
|
||||
},
|
||||
ContextualVariable {
|
||||
|
||||
@@ -1,206 +0,0 @@
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
|
||||
|
||||
pub static ref DEFAULT_TAGS : Vec<String> = vec![
|
||||
"deno".to_string(),
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"powershell".to_string(),
|
||||
"nativets".to_string(),
|
||||
"mysql".to_string(),
|
||||
"graphql".to_string(),
|
||||
"bun".to_string(),
|
||||
"postgresql".to_string(),
|
||||
"bigquery".to_string(),
|
||||
"snowflake".to_string(),
|
||||
"graphql".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
"other".to_string()];
|
||||
|
||||
|
||||
pub static ref WORKER_CONFIG: Arc<RwLock<WorkerConfig>> = Arc::new(RwLock::new(WorkerConfig {
|
||||
worker_tags: Default::default(),
|
||||
dedicated_worker: Default::default(),
|
||||
}));
|
||||
|
||||
pub static ref SERVER_CONFIG: Arc<RwLock<ServerConfig>> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 }));
|
||||
|
||||
|
||||
|
||||
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
|
||||
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
|
||||
pub static ref CUSTOM_TAGS_PER_WORKSPACE: Arc<RwLock<(Vec<String>, HashMap<String, Vec<String>>)>> = Arc::new(RwLock::new((vec![], HashMap::new())));
|
||||
|
||||
pub static ref ALL_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(vec![]));
|
||||
|
||||
static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^(\w+)\(((?:\w+)\+?)+\)$").unwrap();
|
||||
|
||||
}
|
||||
|
||||
pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {
|
||||
let q = sqlx::query!(
|
||||
"SELECT value FROM global_settings WHERE name = $1",
|
||||
CUSTOM_TAGS_SETTING
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let tags = if let Some(q) = q {
|
||||
if let Ok(v) = serde_json::from_value::<Vec<String>>(q.value.clone()) {
|
||||
v
|
||||
} else {
|
||||
tracing::error!(
|
||||
"Could not parse custom tags setting as vec of strings, found: {:#?}",
|
||||
&q.value
|
||||
);
|
||||
vec![]
|
||||
}
|
||||
} else {
|
||||
CUSTOM_TAGS.clone()
|
||||
};
|
||||
|
||||
let custom_tags = process_custom_tags(tags);
|
||||
|
||||
tracing::info!(
|
||||
"Loaded setting custom tags, common: {:?}, per-workspace: {:?}",
|
||||
custom_tags.0,
|
||||
custom_tags.1,
|
||||
);
|
||||
|
||||
{
|
||||
let mut l = CUSTOM_TAGS_PER_WORKSPACE.write().await;
|
||||
*l = custom_tags.clone()
|
||||
}
|
||||
{
|
||||
let mut l = ALL_TAGS.write().await;
|
||||
*l = [
|
||||
custom_tags.0.clone(),
|
||||
custom_tags.1.keys().map(|x| x.to_string()).collect_vec(),
|
||||
]
|
||||
.concat();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn process_custom_tags(tags: Vec<String>) -> (Vec<String>, HashMap<String, Vec<String>>) {
|
||||
let mut global = vec![];
|
||||
let mut specific: HashMap<String, Vec<String>> = HashMap::new();
|
||||
for e in tags {
|
||||
if let Some(cap) = CUSTOM_TAG_REGEX.captures(&e) {
|
||||
let tag = cap.get(1).unwrap().as_str().to_string();
|
||||
let workspaces = cap.get(2).unwrap().as_str().split("+");
|
||||
specific.insert(tag, workspaces.map(|x| x.to_string()).collect_vec());
|
||||
} else {
|
||||
global.push(e.to_string());
|
||||
}
|
||||
}
|
||||
(global, specific)
|
||||
}
|
||||
|
||||
pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
|
||||
let (tags, dw) = {
|
||||
let wc = WORKER_CONFIG.read().await.clone();
|
||||
(
|
||||
wc.worker_tags,
|
||||
wc.dedicated_worker
|
||||
.as_ref()
|
||||
.map(|x| format!("{}:{}", x.workspace_id, x.path)),
|
||||
)
|
||||
};
|
||||
sqlx::query!(
|
||||
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
|
||||
worker_instance,
|
||||
worker_name,
|
||||
ip,
|
||||
tags.as_slice(),
|
||||
*WORKER_GROUP,
|
||||
dw
|
||||
)
|
||||
.execute(db)
|
||||
.await
|
||||
.expect("insert worker_ping initial value");
|
||||
}
|
||||
|
||||
pub async fn load_worker_config(db: &DB) -> error::Result<WorkerConfig> {
|
||||
let mut config: WorkerConfigOpt = sqlx::query_scalar!(
|
||||
"SELECT config FROM config WHERE name = $1",
|
||||
format!("worker__{}", *WORKER_GROUP)
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.flatten()
|
||||
.map(|x| serde_json::from_value(x).ok())
|
||||
.flatten()
|
||||
.unwrap_or_default();
|
||||
if config.dedicated_worker.is_none() {
|
||||
config.dedicated_worker = std::env::var("DEDICATED_WORKER").ok();
|
||||
}
|
||||
let dedicated_worker = config.dedicated_worker.map(|x| {
|
||||
let splitted = x.split(':').to_owned().collect_vec();
|
||||
if splitted.len() != 2 {
|
||||
panic!("DEDICATED_WORKER setting should be in the form of <workspace>:<script_path>")
|
||||
} else {
|
||||
let workspace = splitted[0];
|
||||
let script_path = splitted[1];
|
||||
WorkspacedPath { workspace_id: workspace.to_string(), path: script_path.to_string() }
|
||||
}
|
||||
});
|
||||
Ok(WorkerConfig {
|
||||
worker_tags: config
|
||||
.worker_tags
|
||||
.or_else(|| {
|
||||
if let Some(ref dedicated_worker) = dedicated_worker.as_ref() {
|
||||
Some(vec![format!(
|
||||
"{}:{}",
|
||||
dedicated_worker.workspace_id, dedicated_worker.path
|
||||
)])
|
||||
} else {
|
||||
std::env::var("WORKER_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect())
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| DEFAULT_TAGS.clone()),
|
||||
dedicated_worker,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Debug)]
|
||||
pub struct WorkspacedPath {
|
||||
pub workspace_id: String,
|
||||
pub path: String,
|
||||
}
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct WorkerConfigOpt {
|
||||
pub worker_tags: Option<Vec<String>>,
|
||||
pub dedicated_worker: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for WorkerConfigOpt {
|
||||
fn default() -> Self {
|
||||
Self { worker_tags: Default::default(), dedicated_worker: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Debug, Clone)]
|
||||
pub struct WorkerConfig {
|
||||
pub worker_tags: Vec<String>,
|
||||
pub dedicated_worker: Option<WorkspacedPath>,
|
||||
}
|
||||
@@ -37,4 +37,3 @@ tokio.workspace = true
|
||||
futures-core.workspace = true
|
||||
itertools.workspace = true
|
||||
async-recursion.workspace = true
|
||||
bigdecimal.workspace = true
|
||||
@@ -6,18 +6,19 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{collections::HashMap, vec};
|
||||
use std::{collections::HashMap, sync::atomic::AtomicBool, vec};
|
||||
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::Context;
|
||||
use async_recursion::async_recursion;
|
||||
use bigdecimal::ToPrimitive;
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use itertools::Itertools;
|
||||
use reqwest::Client;
|
||||
use rsmq_async::RsmqConnection;
|
||||
use serde_json::json;
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
#[cfg(feature = "benchmark")]
|
||||
use std::time::Instant;
|
||||
use tracing::{instrument, Instrument};
|
||||
use ulid::Ulid;
|
||||
use uuid::Uuid;
|
||||
@@ -36,13 +37,9 @@ use windmill_common::{
|
||||
schedule::{schedule_to_user, Schedule},
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
users::{username_to_permissioned_as, SUPERADMIN_SECRET_EMAIL},
|
||||
worker::WORKER_CONFIG,
|
||||
DB, METRICS_ENABLED,
|
||||
};
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_common::worker::CLOUD_HOSTED;
|
||||
|
||||
use crate::{
|
||||
schedule::{get_schedule_opt, push_scheduled_job},
|
||||
QueueTransaction,
|
||||
@@ -69,7 +66,62 @@ lazy_static::lazy_static! {
|
||||
"Total number of jobs pulled from the queue."
|
||||
)
|
||||
.unwrap();
|
||||
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
|
||||
|
||||
pub static ref DEFAULT_TAGS : Vec<String> = vec![
|
||||
"deno".to_string(),
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"powershell".to_string(),
|
||||
"nativets".to_string(),
|
||||
"mysql".to_string(),
|
||||
"graphql".to_string(),
|
||||
"bun".to_string(),
|
||||
"postgresql".to_string(),
|
||||
"bigquery".to_string(),
|
||||
"snowflake".to_string(),
|
||||
"graphql".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
"other".to_string()];
|
||||
|
||||
pub static ref DEDICATED_WORKER: Option<(String, String)> = std::env::var("DEDICATED_WORKER")
|
||||
.ok()
|
||||
.map(|x| {
|
||||
let splitted = x.split(':').to_owned().collect_vec();
|
||||
if splitted.len() != 2 {
|
||||
panic!("DEDICATED_WORKER should be in the form of <workspace>:<script_path>")
|
||||
} else {
|
||||
let workspace = splitted[0];
|
||||
let script_path = splitted[1];
|
||||
(workspace.to_string(), script_path.to_string())
|
||||
}
|
||||
});
|
||||
|
||||
pub static ref ACCEPTED_TAGS: Vec<String> = {
|
||||
let worker_tags = std::env::var("WORKER_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect())
|
||||
.unwrap_or_else(|| DEFAULT_TAGS.clone());
|
||||
if let Some(ref dedicated_worker) = DEDICATED_WORKER.as_ref() {
|
||||
vec![format!("{}:{}", dedicated_worker.0, dedicated_worker.1)]
|
||||
} else {
|
||||
worker_tags
|
||||
}
|
||||
};
|
||||
|
||||
pub static ref IS_WORKER_TAGS_DEFINED: bool = std::env::var("WORKER_TAGS").ok().is_some();
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// When compiled in 'benchmark' mode, this flags is exposed via the /workers/toggle endpoint
|
||||
// and make it possible to disable to current active workers (such that they don't pull any)
|
||||
// jobs from the queue
|
||||
pub static ref IDLE_WORKERS: AtomicBool = AtomicBool::new(false);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
@@ -232,8 +284,6 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.map(|x| x.to_i64())
|
||||
.flatten()
|
||||
} else {
|
||||
tracing::warn!("Could not parse flow status");
|
||||
None
|
||||
@@ -250,7 +300,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
.flatten();
|
||||
let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
|
||||
let job_id = queued_job.id.clone();
|
||||
let _duration: i64 = sqlx::query_scalar!(
|
||||
let _duration = sqlx::query_scalar!(
|
||||
"INSERT INTO completed_job AS cj
|
||||
( workspace_id
|
||||
, id
|
||||
@@ -375,7 +425,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
if !is_flow && _duration > 1000 {
|
||||
let additional_usage = _duration / 1000;
|
||||
let w_id = &queued_job.workspace_id;
|
||||
let premium_workspace = *windmill_common::worker::CLOUD_HOSTED
|
||||
let premium_workspace = *CLOUD_HOSTED
|
||||
&& sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
@@ -386,7 +436,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $3",
|
||||
if premium_workspace { w_id } else { &queued_job.email },
|
||||
premium_workspace,
|
||||
additional_usage as i32)
|
||||
additional_usage)
|
||||
.execute(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("updating usage: {e}")));
|
||||
@@ -1022,8 +1072,8 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
|
||||
|
||||
let job_uuid: Uuid = pulled_job.id;
|
||||
let min_started_at: Option<DateTime<Utc>> = script_path_live_stats.min_started_at;
|
||||
let avg_script_duration: Option<i64> = sqlx::query_scalar!(
|
||||
"SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS BIGINT) AS avg_duration_s FROM
|
||||
let avg_script_duration: Option<i32> = sqlx::query_scalar!(
|
||||
"SELECT CAST(ROUND(AVG(duration_ms) / 1000, 0) AS INT) AS avg_duration_s FROM
|
||||
(SELECT duration_ms FROM completed_job WHERE script_path = $1
|
||||
ORDER BY started_at
|
||||
DESC LIMIT 10) AS t",
|
||||
@@ -1077,7 +1127,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
|
||||
, started_at = null
|
||||
, scheduled_for = '{estimated_next_schedule_timestamp}'
|
||||
, logs = CASE WHEN logs IS NULL OR logs = '' THEN '{job_log_event}'::text WHEN logs LIKE '%{job_log_event}' THEN logs ELSE concat(logs, '{job_log_line_break}{job_log_event}'::text) END
|
||||
WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false AND scheduled_for <= now())
|
||||
WHERE (id = '{job_uuid}') OR (script_path = '{job_script_path}' AND running = false)
|
||||
RETURNING *"
|
||||
))
|
||||
.fetch_all(&mut tx)
|
||||
@@ -1103,7 +1153,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
// TODO: REDIS: Race conditions / replace last_ping
|
||||
|
||||
// TODO: shuffle this list to have fairness
|
||||
let mut all_tags = WORKER_CONFIG.read().await.worker_tags.clone();
|
||||
let mut all_tags = ACCEPTED_TAGS.clone();
|
||||
|
||||
let mut msg: Option<_> = None;
|
||||
let mut tag = None;
|
||||
@@ -1160,9 +1210,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
* suspend_until is non-null
|
||||
* and suspend = 0 when the resume messages are received
|
||||
* or suspend_until <= now() if it has timed out */
|
||||
let config = WORKER_CONFIG.read().await.clone();
|
||||
let tags = config.worker_tags.clone();
|
||||
drop(config);
|
||||
|
||||
let r = if suspend_first {
|
||||
sqlx::query_as::<_, QueuedJob>("UPDATE queue
|
||||
SET running = true
|
||||
@@ -1178,7 +1226,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING *")
|
||||
.bind(tags)
|
||||
.bind(ACCEPTED_TAGS.as_slice())
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
} else {
|
||||
@@ -1189,8 +1237,6 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
// #[cfg(feature = "benchmark")]
|
||||
// let instant = Instant::now();
|
||||
|
||||
let tags = WORKER_CONFIG.read().await.worker_tags.clone();
|
||||
|
||||
let r = sqlx::query_as::<_, QueuedJob>(
|
||||
"UPDATE queue
|
||||
SET running = true
|
||||
@@ -1207,9 +1253,10 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
|
||||
)
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(tags)
|
||||
.bind(ACCEPTED_TAGS.as_slice())
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
// #[cfg(feature = "benchmark")]
|
||||
// println!("pull query: {:?}", instant.elapsed());
|
||||
|
||||
@@ -1306,35 +1353,15 @@ pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
w_id,
|
||||
job_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await;
|
||||
|
||||
if let Err(job_removed) = job_removed {
|
||||
tracing::error!(
|
||||
"Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"
|
||||
);
|
||||
} else {
|
||||
let job_removed = job_removed.unwrap().flatten().unwrap_or(0);
|
||||
if job_removed != 1 {
|
||||
tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling");
|
||||
}
|
||||
}
|
||||
|
||||
tracing::debug!("Job {job_id} deleted");
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
|
||||
.unwrap_or(0)
|
||||
== 1;
|
||||
tracing::debug!("Job {job_id} deleted: {job_removed}");
|
||||
Ok(tx)
|
||||
}
|
||||
|
||||
pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result<bool> {
|
||||
Ok(sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM completed_job WHERE id = $1 AND workspace_id = $2)",
|
||||
id,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await?
|
||||
.unwrap_or(false))
|
||||
}
|
||||
|
||||
pub async fn get_queued_job<'c>(
|
||||
id: Uuid,
|
||||
w_id: &str,
|
||||
@@ -1744,11 +1771,11 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
tag = None;
|
||||
}
|
||||
let default = || {
|
||||
if job_kind == JobKind::Flow
|
||||
|| job_kind == JobKind::FlowPreview
|
||||
|| job_kind == JobKind::Identity
|
||||
{
|
||||
if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
|
||||
"flow".to_string()
|
||||
} else if job_kind == JobKind::Identity {
|
||||
// identity is a light script, nativets is too
|
||||
"nativets".to_string()
|
||||
} else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
|
||||
"dependency".to_string()
|
||||
} else {
|
||||
|
||||
@@ -71,7 +71,6 @@ jsonwebtoken = { workspace = true, optional = true }
|
||||
sha2 = { workspace = true, optional = true }
|
||||
pem = { workspace = true, optional = true }
|
||||
urlencoding.workspace = true
|
||||
nix.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
deno_fetch.workspace = true
|
||||
|
||||
@@ -24,25 +24,15 @@ const p = {
|
||||
loader: "tsx",
|
||||
};
|
||||
});
|
||||
const cdir = resolve("./");
|
||||
const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos
|
||||
const filter = new RegExp(
|
||||
`^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.ts).*\\.ts$`
|
||||
);
|
||||
build.onResolve({ filter }, (args) => {
|
||||
build.onResolve({ filter: /(?!\.\/main\.ts)\..*\.ts$/ }, (args) => {
|
||||
const cdir = resolve("./");
|
||||
const file_path =
|
||||
args.importer == "./main.ts" || args.importer == resolve("./main.ts")
|
||||
? current_path
|
||||
: args.importer.replace(cdir + "/", "");
|
||||
|
||||
const isRelative = !args.path.startsWith("/");
|
||||
|
||||
const url = isRelative
|
||||
? `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${file_path}/../${args.path}`
|
||||
: `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${args.path}`;
|
||||
const file = isRelative
|
||||
? resolve("./" + current_path + "/../" + args.path + ".url")
|
||||
: resolve("./" + args.path + ".url");
|
||||
const url = `${base_internal_url}/api/w/${w_id}/scripts/raw/p/${file_path}/../${args.path}`;
|
||||
const file = resolve("./" + current_path + "/../" + args.path + ".url");
|
||||
mkdirSync(dirname(file), { recursive: true });
|
||||
writeFileSync(file, url);
|
||||
return {
|
||||
|
||||
@@ -7,8 +7,6 @@ const bo = await Bun.build({
|
||||
|
||||
const fs = require("fs/promises");
|
||||
|
||||
const captureVersion = /(^\@?[^\@]+)(?:\@(.+))?/;
|
||||
|
||||
if (!bo.success) {
|
||||
bo.logs.forEach((l) => console.log(l));
|
||||
process.exit(1);
|
||||
@@ -19,7 +17,7 @@ if (!bo.success) {
|
||||
const { intersect } = require("semver-intersect");
|
||||
const dependencies: Record<string, string[]> = {};
|
||||
for (const i of imports) {
|
||||
let [_, name, version] = i.path.match(captureVersion) ?? [];
|
||||
let [name, version] = i.path.split("@");
|
||||
if (version == undefined) {
|
||||
if (dependencies[name] == undefined) {
|
||||
dependencies[name] = [];
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user