Compare commits
5 Commits
v1.92.1
...
rf/flowFix
| Author | SHA1 | Date | |
|---|---|---|---|
|
1ea08f1e8f | ||
|
|
d302c92766 | ||
|
|
128622117d | ||
|
|
150ce7f731 | ||
|
|
fae406ee8f |
7
.env
7
.env
@@ -1,7 +1,2 @@
|
||||
DB_PASSWORD=changeme
|
||||
|
||||
# this is the url that your instance is publicly exposed to
|
||||
WM_BASE_URL=http://localhost
|
||||
|
||||
# this is the addres caddy will listen on
|
||||
CADDY_REVERSE_PROXY=":80"
|
||||
WM_BASE_URL=localhost
|
||||
|
||||
2
.github/change-versions.sh
vendored
2
.github/change-versions.sh
vendored
@@ -4,7 +4,7 @@ VERSION=$1
|
||||
echo "Updating versions to: $VERSION"
|
||||
|
||||
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
|
||||
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
|
||||
sed -i -e "/^const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
|
||||
sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
|
||||
sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
|
||||
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
|
||||
|
||||
2
.github/workflows/backend-test.yml
vendored
2
.github/workflows/backend-test.yml
vendored
@@ -40,4 +40,4 @@ jobs:
|
||||
backend -> target
|
||||
- name: cargo test
|
||||
timeout-minutes: 10
|
||||
run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
|
||||
run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture
|
||||
|
||||
20
.github/workflows/deploy_to_windmill.yml
vendored
Normal file
20
.github/workflows/deploy_to_windmill.yml
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
name: Deploy to windmill.dev
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "community/**"
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Deploy to windmill.dev
|
||||
uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
|
||||
with:
|
||||
dry_run: false
|
||||
input_dir: community
|
||||
windmill_workspace: starter
|
||||
windmill_token: ${{ secrets.WINDMILL_API_TOKEN }}
|
||||
101
.github/workflows/docker-image.yml
vendored
101
.github/workflows/docker-image.yml
vendored
@@ -18,7 +18,7 @@ permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
packages: write
|
||||
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -50,6 +50,7 @@ jobs:
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
|
||||
|
||||
- name: Build and push publicly
|
||||
uses: depot/build-push-action@v1
|
||||
with:
|
||||
@@ -62,7 +63,7 @@ jobs:
|
||||
labels: |
|
||||
${{ steps.meta-public.outputs.labels }}
|
||||
org.opencontainers.image.licenses=AGPLv3
|
||||
|
||||
|
||||
build_ee:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
@@ -108,61 +109,40 @@ jobs:
|
||||
${{ steps.meta-ee-public.outputs.labels }}
|
||||
org.opencontainers.image.licenses=Windmill-Enterprise-License
|
||||
|
||||
# disabled until we make it 100% reliable and add more meaningful tests
|
||||
# playwright:
|
||||
# runs-on: [self-hosted, new]
|
||||
# needs: [build]
|
||||
# services:
|
||||
# postgres:
|
||||
# image: postgres
|
||||
# env:
|
||||
# POSTGRES_DB: windmill
|
||||
# POSTGRES_USER: admin
|
||||
# POSTGRES_PASSWORD: changeme
|
||||
# ports:
|
||||
# - 5432:5432
|
||||
# options: >-
|
||||
# --health-cmd pg_isready
|
||||
# --health-interval 10s
|
||||
# --health-timeout 5s
|
||||
# --health-retries 5
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: "Docker"
|
||||
# run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
|
||||
# id: docker-container
|
||||
# - uses: actions/setup-node@v3
|
||||
# with:
|
||||
# node-version: 16
|
||||
# - name: "Playwright run"
|
||||
# timeout-minutes: 2
|
||||
# run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
|
||||
# - name: "Clean up"
|
||||
# run: docker kill ${{ steps.docker-container.outputs.id }}
|
||||
# if: always()
|
||||
|
||||
deploy_s3:
|
||||
needs: [build_ee]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
playwright:
|
||||
runs-on: [self-hosted, new]
|
||||
needs: [build]
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_DB: windmill
|
||||
POSTGRES_USER: admin
|
||||
POSTGRES_PASSWORD: changeme
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: "Docker"
|
||||
run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
|
||||
id: docker-container
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 18
|
||||
- uses: shrink/actions-docker-extract@v2
|
||||
id: extract
|
||||
with:
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
|
||||
path: "/static_frontend/."
|
||||
node-version: 16
|
||||
- name: "Playwright run"
|
||||
timeout-minutes: 2
|
||||
run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
|
||||
- name: "Clean up"
|
||||
run: docker kill ${{ steps.docker-container.outputs.id }}
|
||||
if: always()
|
||||
|
||||
|
||||
- uses: reggionick/s3-deploy@v3
|
||||
with:
|
||||
folder: ${{ steps.extract.outputs.destination }}
|
||||
bucket: windmill-frontend
|
||||
bucket-region: us-east-1
|
||||
publish_privately_heavy:
|
||||
needs: [build_ee]
|
||||
runs-on: [self-hosted, new]
|
||||
@@ -211,12 +191,8 @@ jobs:
|
||||
tags: |
|
||||
${{ steps.meta-heavy.outputs.tags }}
|
||||
labels: ${{ steps.meta-heavy.outputs.labels }}
|
||||
cache-from:
|
||||
type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
|
||||
}}-heavy:buildcache
|
||||
cache-to:
|
||||
type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
|
||||
}}-heavy:buildcache,mode=max
|
||||
cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache
|
||||
cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache,mode=max
|
||||
|
||||
publish_privately_helm:
|
||||
runs-on: [self-hosted, new]
|
||||
@@ -228,6 +204,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
|
||||
- name: Login to registry
|
||||
uses: docker/login-action@v2
|
||||
@@ -243,7 +220,7 @@ jobs:
|
||||
registry: ${{ env.ECR_REGISTRY }}
|
||||
username: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
|
||||
|
||||
- name: Build and push privately
|
||||
uses: docker/build-push-action@v4
|
||||
if: github.event_name != 'pull_request'
|
||||
@@ -253,9 +230,5 @@ jobs:
|
||||
file: ./docker/DockerfileHelm
|
||||
tags: |
|
||||
${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:helm
|
||||
cache-from:
|
||||
type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
|
||||
}}-helm:buildcache
|
||||
cache-to:
|
||||
type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
|
||||
}}-helm:buildcache,mode=max
|
||||
cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache
|
||||
cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache,mode=max
|
||||
|
||||
2
.github/workflows/frontend-check.yml
vendored
2
.github/workflows/frontend-check.yml
vendored
@@ -4,8 +4,6 @@ on:
|
||||
types: [opened,synchronize,reopened,closed]
|
||||
paths:
|
||||
- "frontend/**"
|
||||
merge_group:
|
||||
|
||||
jobs:
|
||||
npm_check:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
2
.github/workflows/go_on_release.yml
vendored
2
.github/workflows/go_on_release.yml
vendored
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
- uses: actions/setup-go@v3
|
||||
- name: generate_go
|
||||
run: |
|
||||
go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.11.0
|
||||
|
||||
510
CHANGELOG.md
510
CHANGELOG.md
@@ -1,516 +1,6 @@
|
||||
# Changelog
|
||||
|
||||
|
||||
## [1.92.1](https://github.com/windmill-labs/windmill/compare/v1.92.0...v1.92.1) (2023-05-02)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **go-client:** fix go-client gen ([df333d9](https://github.com/windmill-labs/windmill/commit/df333d9739f601714f7a0124f47422dfb113d320))
|
||||
|
||||
## [1.92.0](https://github.com/windmill-labs/windmill/compare/v1.91.0...v1.92.0) (2023-05-02)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add labels as table action ([64065c1](https://github.com/windmill-labs/windmill/commit/64065c17f305fb4c7c078c7fa4935d5423da8f66))
|
||||
* **frontend:** add labels as table action ([2ab1714](https://github.com/windmill-labs/windmill/commit/2ab1714dfa7cffc46b4b6aa40dabdd92c5a6270f))
|
||||
* **frontend:** allow running eval in every field ([62acbb5](https://github.com/windmill-labs/windmill/commit/62acbb5ab3a0727b306e25a80b74ac8216619501))
|
||||
* **frontend:** background script can run script and flows ([#1515](https://github.com/windmill-labs/windmill/issues/1515)) ([607c803](https://github.com/windmill-labs/windmill/commit/607c803be91921b53f329a2c2c3c129ce53d6c0c))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** fix small ui issues ([#1513](https://github.com/windmill-labs/windmill/issues/1513)) ([f6ff8ca](https://github.com/windmill-labs/windmill/commit/f6ff8ca232f5725f86a36379956da2731def2580))
|
||||
|
||||
## [1.91.0](https://github.com/windmill-labs/windmill/compare/v1.90.0...v1.91.0) (2023-05-01)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add drafts for apps ([f7374c8](https://github.com/windmill-labs/windmill/commit/f7374c8204f85b4371e61f34dcd4b66857c0f8ab))
|
||||
* introduce backend persisted draft systems for scripts ([88e37fe](https://github.com/windmill-labs/windmill/commit/88e37fe0bed58f396690622e925d5e078c60140c))
|
||||
* introduce draft for flows ([a196642](https://github.com/windmill-labs/windmill/commit/a1966427e893dc8a58c8f2862ded752884843813))
|
||||
|
||||
## [1.90.0](https://github.com/windmill-labs/windmill/compare/v1.89.0...v1.90.0) (2023-04-28)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** add EXIT_AFTER_NO_JOB_FOR_SECS for ephemeral workers ([de9abd1](https://github.com/windmill-labs/windmill/commit/de9abd129db13dcdf0e69e2c1e2d3aa558fb783a))
|
||||
* **backend:** add JOB_RETENTION_SECS to delete completed jobs completed after expiration period ([0b7bad3](https://github.com/windmill-labs/windmill/commit/0b7bad3816e61841ef4765db7881227274c20b23))
|
||||
* **backend:** expose tag in the job ([#1486](https://github.com/windmill-labs/windmill/issues/1486)) ([324d4f5](https://github.com/windmill-labs/windmill/commit/324d4f5e9e89e6de600e882f23bf545c0b1dc539))
|
||||
* **frontend:** adapt style ([#1488](https://github.com/windmill-labs/windmill/issues/1488)) ([41a24ec](https://github.com/windmill-labs/windmill/commit/41a24ecd36d9cc537cbd1dd0cd1de6f689be1b8c))
|
||||
* **frontend:** add an eval input component for flow ([#1494](https://github.com/windmill-labs/windmill/issues/1494)) ([2815f1e](https://github.com/windmill-labs/windmill/commit/2815f1ec71177bb6e89d0d62a8df89030d37b1fc))
|
||||
* **frontend:** Add new integration icons ([#1479](https://github.com/windmill-labs/windmill/issues/1479)) ([7adacd4](https://github.com/windmill-labs/windmill/commit/7adacd4c9f03d17b47abc515bc391e348a7e6ec1))
|
||||
* **frontend:** refactor inline script ([#1480](https://github.com/windmill-labs/windmill/issues/1480)) ([05c837e](https://github.com/windmill-labs/windmill/commit/05c837e64f61bfb22ae1f80263deb1c879030985))
|
||||
* **frontend:** Schedules run now ([#1475](https://github.com/windmill-labs/windmill/issues/1475)) ([47f0f35](https://github.com/windmill-labs/windmill/commit/47f0f35236e02958f6bc00b5652e06e25eabeaf5))
|
||||
* **frontend:** Small style fix ([#1473](https://github.com/windmill-labs/windmill/issues/1473)) ([7ad496a](https://github.com/windmill-labs/windmill/commit/7ad496ad3f746ffd782856a35c1456999792fa94))
|
||||
* **frontend:** Support TS union type with a select field ([#1457](https://github.com/windmill-labs/windmill/issues/1457)) ([8b76324](https://github.com/windmill-labs/windmill/commit/8b763249cb1360c122cc81d50a1a95d1ad3ddd5b))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** Allow 0 as select default value ([#1474](https://github.com/windmill-labs/windmill/issues/1474)) ([d8529ff](https://github.com/windmill-labs/windmill/commit/d8529ff3ed6168de60bb24626a3e36ab4beae15c))
|
||||
* **frontend:** close the modal before deleting a form modal ([#1484](https://github.com/windmill-labs/windmill/issues/1484)) ([430c733](https://github.com/windmill-labs/windmill/commit/430c73399b7e1524ee81d0ce8d7d2eaf16117f9a))
|
||||
* **frontend:** fix apply connection ([#1487](https://github.com/windmill-labs/windmill/issues/1487)) ([cf59cc0](https://github.com/windmill-labs/windmill/commit/cf59cc04efb853fe06a23042115128689b5d26ee))
|
||||
* **frontend:** Fix frontend script ([#1476](https://github.com/windmill-labs/windmill/issues/1476)) ([b60a7f6](https://github.com/windmill-labs/windmill/commit/b60a7f63d04b1fc851479060e77613c77d20198a))
|
||||
* **frontend:** fix recomputa all ([#1491](https://github.com/windmill-labs/windmill/issues/1491)) ([fb05a09](https://github.com/windmill-labs/windmill/commit/fb05a09955f937000fbba0826c03c52e65aa146e))
|
||||
* **frontend:** Flow editor design updates ([#1477](https://github.com/windmill-labs/windmill/issues/1477)) ([50d814c](https://github.com/windmill-labs/windmill/commit/50d814c3dc55841b16cd1df8ae86d021de4e880c))
|
||||
* **frontend:** Minor app editor updates ([#1458](https://github.com/windmill-labs/windmill/issues/1458)) ([8fd10b1](https://github.com/windmill-labs/windmill/commit/8fd10b1f5813b3f4200a980de4a85e7d701660a7))
|
||||
* **frontend:** register applyConnection as a callback to remove unnecessary reactivit ([#1485](https://github.com/windmill-labs/windmill/issues/1485)) ([d915f6b](https://github.com/windmill-labs/windmill/commit/d915f6b004ea7cc27a3c55b3504df902f5db1aef))
|
||||
* **frontend:** reset ui job loading state when submitting preview job triggers error ([#1483](https://github.com/windmill-labs/windmill/issues/1483)) ([6f8616f](https://github.com/windmill-labs/windmill/commit/6f8616fb273b1f4a1489878a617d239f30ecb1c0))
|
||||
* **frontend:** Update CLI login request styling ([#1454](https://github.com/windmill-labs/windmill/issues/1454)) ([c77393c](https://github.com/windmill-labs/windmill/commit/c77393c15444868308b44a72bee89e49fc23d80f))
|
||||
* **frontend:** Update direct exports ([#1456](https://github.com/windmill-labs/windmill/issues/1456)) ([4a2af13](https://github.com/windmill-labs/windmill/commit/4a2af1359ee29001236591f685cacfa9df6715df))
|
||||
|
||||
## [1.89.0](https://github.com/windmill-labs/windmill/compare/v1.88.1...v1.89.0) (2023-04-23)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** global cache refactor for pip using tar for each dependency ([#1443](https://github.com/windmill-labs/windmill/issues/1443)) ([369dd0d](https://github.com/windmill-labs/windmill/commit/369dd0dac61e5430856ed9abf7129bbad3b75860))
|
||||
* **backend:** only run fully deployed scripts ([3d031c7](https://github.com/windmill-labs/windmill/commit/3d031c701705459f418b11d2ca83e71943e4079b))
|
||||
* **backend:** worker groups ([#1452](https://github.com/windmill-labs/windmill/issues/1452)) ([722783f](https://github.com/windmill-labs/windmill/commit/722783f7f630e3123ffd2605deb21a915188bd20))
|
||||
* **backend:** workers are instantly ready and sync with global cache in background ([670ba51](https://github.com/windmill-labs/windmill/commit/670ba51d9bd9a9a0b07a2ed064c316234bb5819d))
|
||||
* **ee:** sync cache in background ([c919827](https://github.com/windmill-labs/windmill/commit/c919827cf8eb9437b6d9bd57b3d3ad883a66de3b))
|
||||
* **ee:** sync cache in background ([0e77e37](https://github.com/windmill-labs/windmill/commit/0e77e37fbddbb517d5e1b1f07a27a40b63371439))
|
||||
* **frontend:** Add documentation links ([#1399](https://github.com/windmill-labs/windmill/issues/1399)) ([36acbf7](https://github.com/windmill-labs/windmill/commit/36acbf793b6714dffe4dbb0e2501b9438f034858))
|
||||
* **frontend:** Add seconds input ([#1445](https://github.com/windmill-labs/windmill/issues/1445)) ([30bf7ad](https://github.com/windmill-labs/windmill/commit/30bf7ad3e9785420b4dcd814ce3c7a444d23cc9f))
|
||||
* **frontend:** add toast actions ([#1411](https://github.com/windmill-labs/windmill/issues/1411)) ([d173232](https://github.com/windmill-labs/windmill/commit/d17323286a05aa0b1680ef94d7058d2c8902782f))
|
||||
* **frontend:** reorder array items in app editor ([#1426](https://github.com/windmill-labs/windmill/issues/1426)) ([3615fb2](https://github.com/windmill-labs/windmill/commit/3615fb26fb91d64626d82ace6e6275e424ece832))
|
||||
* **frontend:** support showing metadata on script add via query param ([#1438](https://github.com/windmill-labs/windmill/issues/1438)) ([3c98452](https://github.com/windmill-labs/windmill/commit/3c98452f50913ef639eba96996f3a6c80508bd63))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** avoid potential conflict between pull from tar and background sync ([d76e907](https://github.com/windmill-labs/windmill/commit/d76e90757e209263da7f79fa85052969e7efd63d))
|
||||
* **backend:** global cache synco only start if all piptars have been downloaded ([5f8a730](https://github.com/windmill-labs/windmill/commit/5f8a730fdfbb9e3d518555f7272a3bb297725f28))
|
||||
* **frontend:** App color picker overflow issue ([#1449](https://github.com/windmill-labs/windmill/issues/1449)) ([32903d2](https://github.com/windmill-labs/windmill/commit/32903d2839a082d53168bb9177fc88f6ab0ec482))
|
||||
* **frontend:** fix copy content button width ([#1428](https://github.com/windmill-labs/windmill/issues/1428)) ([d96d4a5](https://github.com/windmill-labs/windmill/commit/d96d4a524edebea65bc602194c8f11f5d69e920a))
|
||||
* **frontend:** Minor update of app default codes ([#1440](https://github.com/windmill-labs/windmill/issues/1440)) ([fe75aa1](https://github.com/windmill-labs/windmill/commit/fe75aa18f2f27745db35329aa60938694640a8c6))
|
||||
* **frontend:** Update app default codes ([#1432](https://github.com/windmill-labs/windmill/issues/1432)) ([c8acfbc](https://github.com/windmill-labs/windmill/commit/c8acfbc1ff0f6c23e5a2229ca83a3b09eec826c3))
|
||||
* **frontend:** Update app mobile preview width ([#1431](https://github.com/windmill-labs/windmill/issues/1431)) ([1764613](https://github.com/windmill-labs/windmill/commit/17646130bcf8cf646a4ccdfa39f9a8791876a137))
|
||||
* **frontend:** Update flow tooltip z-indexes ([#1433](https://github.com/windmill-labs/windmill/issues/1433)) ([17cb8fc](https://github.com/windmill-labs/windmill/commit/17cb8fc3fa0b39a9750e61ca2731a15bbda690ec))
|
||||
* **frontend:** Update flow viewer styling ([#1441](https://github.com/windmill-labs/windmill/issues/1441)) ([46a29b5](https://github.com/windmill-labs/windmill/commit/46a29b5d27b8d9f7ea38c1063fc081ed5933db5d))
|
||||
|
||||
## [1.88.1](https://github.com/windmill-labs/windmill/compare/v1.88.0...v1.88.1) (2023-04-18)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** fix hub list ([1144329](https://github.com/windmill-labs/windmill/commit/1144329972fb61e2df62873ca1e485c88fabc478))
|
||||
|
||||
## [1.88.0](https://github.com/windmill-labs/windmill/compare/v1.87.0...v1.88.0) (2023-04-17)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** install python scripts on save ([cb7e686](https://github.com/windmill-labs/windmill/commit/cb7e686dd95397d5b37edd5aac50b6d1429c4a71))
|
||||
* **frontend:** Add runs preview popup ([#1405](https://github.com/windmill-labs/windmill/issues/1405)) ([4ab023f](https://github.com/windmill-labs/windmill/commit/4ab023f95085958ab1ad01dc249d308c7ebf423e))
|
||||
* **frontend:** cancellable inline script editor run ([e828d26](https://github.com/windmill-labs/windmill/commit/e828d2673e62e95e5e1235eeca8107ac7cfb7e45))
|
||||
* **frontend:** Remove gap when button label is empty ([#1402](https://github.com/windmill-labs/windmill/issues/1402)) ([568f59e](https://github.com/windmill-labs/windmill/commit/568f59eefb104047b8ef063f273fe238075d6407))
|
||||
* **frontend:** Unify main lists ([#1406](https://github.com/windmill-labs/windmill/issues/1406)) ([48bbbd0](https://github.com/windmill-labs/windmill/commit/48bbbd0e872a12ed1c562a6d14967a2a0f7c4735))
|
||||
* **frontend:** Update airtable instructions ([#1403](https://github.com/windmill-labs/windmill/issues/1403)) ([7dc7ece](https://github.com/windmill-labs/windmill/commit/7dc7ecef55b465fc096f71fc9de5c8b543136ff7))
|
||||
* inputs library on run page ([92a2934](https://github.com/windmill-labs/windmill/commit/92a293488e8e58350229931ab69f7924d58474be))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** deno uses --no-check ([a5499c2](https://github.com/windmill-labs/windmill/commit/a5499c26f3ebd8b07541a7e0cbf33a7008a8f476))
|
||||
* **backend:** do not fail on schedule not existing anymore ([a5f6d73](https://github.com/windmill-labs/windmill/commit/a5f6d73f7d53d7af9d285a85460509763263c508))
|
||||
* **frontend:** Fix app file uploads ([#1408](https://github.com/windmill-labs/windmill/issues/1408)) ([ac489ac](https://github.com/windmill-labs/windmill/commit/ac489ac2da0fbf01f5e2877612c14cfaf1ef79c2))
|
||||
* **frontend:** fix buttons width ([#1407](https://github.com/windmill-labs/windmill/issues/1407)) ([75a0482](https://github.com/windmill-labs/windmill/commit/75a0482ef046dd7e30f6d6039dbc66880182dc5e))
|
||||
* **frontend:** fix enum sync ([#1410](https://github.com/windmill-labs/windmill/issues/1410)) ([98060ce](https://github.com/windmill-labs/windmill/commit/98060ce55d5efa59a8989cf9357935976d57650b))
|
||||
* **frontend:** Handle scheduled runs in preview ([#1413](https://github.com/windmill-labs/windmill/issues/1413)) ([accdc1a](https://github.com/windmill-labs/windmill/commit/accdc1ac59ce9611f66567222a73995d3c0a3f9d))
|
||||
* **frontend:** Keep selected tab during renaming ([#1409](https://github.com/windmill-labs/windmill/issues/1409)) ([82cd048](https://github.com/windmill-labs/windmill/commit/82cd048ef4d08f31660f6f31a96940676a28996c))
|
||||
* **frontend:** Queued-running jobs preview ([#1414](https://github.com/windmill-labs/windmill/issues/1414)) ([b2a40a0](https://github.com/windmill-labs/windmill/commit/b2a40a05805344c1c34f2ba917b4cdd52dfffc3f))
|
||||
* **frontend:** Remove output when deleting a component ([#1397](https://github.com/windmill-labs/windmill/issues/1397)) ([6aa1008](https://github.com/windmill-labs/windmill/commit/6aa100893352870d5a99fdd56d7f1425a221a273))
|
||||
|
||||
## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)
|
||||
|
||||
|
||||
### ⚠ BREAKING CHANGES
|
||||
|
||||
* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380))
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** add instance events webhook ([f2d3c82](https://github.com/windmill-labs/windmill/commit/f2d3c8208b6daa49f304f355752145de47138a3c))
|
||||
* **backend:** extend cached resolution for go ([dac61d1](https://github.com/windmill-labs/windmill/commit/dac61d1c982576d7589e16ab01c8cc8bad6e1686))
|
||||
* **backend:** Redis based queue ([#1324](https://github.com/windmill-labs/windmill/issues/1324)) ([d45e6c9](https://github.com/windmill-labs/windmill/commit/d45e6c94abed609357b18d4daa7de6b2ea0ba978))
|
||||
* **frontend:** Add option to return file names ([#1380](https://github.com/windmill-labs/windmill/issues/1380)) ([3dabac1](https://github.com/windmill-labs/windmill/commit/3dabac153f302f48210d15ebaec514e72717300f))
|
||||
* **python:** cache dependency resolution ([facb670](https://github.com/windmill-labs/windmill/commit/facb67093ce7d3b0874d0d559fb272ed822ce360))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** nested deno relative imports ([955a213](https://github.com/windmill-labs/windmill/commit/955a213a504c1f3b8811c930823e87fe7dba101a))
|
||||
* **cli:** overwrite archived scripts ([1f705ca](https://github.com/windmill-labs/windmill/commit/1f705cab2ce8c79829f22fc6af9e06ecba7450b1))
|
||||
* **frontend:** Add missing stopPropagation ([#1394](https://github.com/windmill-labs/windmill/issues/1394)) ([58d4b55](https://github.com/windmill-labs/windmill/commit/58d4b556ebbd76c6f07f1a16d601a9d824b99f7e))
|
||||
* **frontend:** fix app init issue ([d0e0e1f](https://github.com/windmill-labs/windmill/commit/d0e0e1fdf27d9a7fb86c66e43398786b64d8b6b7))
|
||||
* **frontend:** Fix frontend dependencies ([#1379](https://github.com/windmill-labs/windmill/issues/1379)) ([8e9c491](https://github.com/windmill-labs/windmill/commit/8e9c49165060a4a7f831b8be075593f89d867784))
|
||||
* **frontend:** Fix icon picker input ([#1389](https://github.com/windmill-labs/windmill/issues/1389)) ([8a44f8e](https://github.com/windmill-labs/windmill/commit/8a44f8e7796f13698e2a99af9f3772f5e676604b))
|
||||
* **frontend:** Fix mac shortcuts ([#1381](https://github.com/windmill-labs/windmill/issues/1381)) ([41831d5](https://github.com/windmill-labs/windmill/commit/41831d58ed593bb283600b76170f6e76783e0eae))
|
||||
* **frontend:** fix popover configuration to avoid content shift ([#1377](https://github.com/windmill-labs/windmill/issues/1377)) ([2031e1e](https://github.com/windmill-labs/windmill/commit/2031e1ebd0dc020da104ee84a0294c86babcefaf))
|
||||
* **frontend:** remove stopPropagation that was preventing components dnd ([#1378](https://github.com/windmill-labs/windmill/issues/1378)) ([de8dc1e](https://github.com/windmill-labs/windmill/commit/de8dc1e9cd7beea2ce62656e9e7676214f77a110))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* parallelize more operations for deno jobs ([e911869](https://github.com/windmill-labs/windmill/commit/e911869d990956463834ac9ff35c52ba8236e362))
|
||||
|
||||
## [1.86.0](https://github.com/windmill-labs/windmill/compare/v1.85.0...v1.86.0) (2023-04-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** add /ready endpoint for workers ([94eecea](https://github.com/windmill-labs/windmill/commit/94eecea02b6295ad5674db4b010bf6ab7984fa17))
|
||||
* **backend:** add GET endpoint to trigger scripts ([15c75d9](https://github.com/windmill-labs/windmill/commit/15c75d9d00a69ae97123ed371b9657e298345bdb))
|
||||
* **backend:** lowercase all emails in relevant endpoints ([#1361](https://github.com/windmill-labs/windmill/issues/1361)) ([7f9050b](https://github.com/windmill-labs/windmill/commit/7f9050b285cf8f7f6baf05452b673f58988c452c))
|
||||
* **cli:** add getFullResource ([3a232db](https://github.com/windmill-labs/windmill/commit/3a232dbb5792c28b26747e1ba260fffcdd4a8416))
|
||||
* do cache bucket syncing in background + check tar before pushing it ([#1360](https://github.com/windmill-labs/windmill/issues/1360)) ([3e5ff86](https://github.com/windmill-labs/windmill/commit/3e5ff8682a298ba9e59b2662c4c04c5698447204))
|
||||
* **frontend:** add flow expand button ([34a8b01](https://github.com/windmill-labs/windmill/commit/34a8b01b762c0b210d76101e7da7bd2397258e8d))
|
||||
* **frontend:** add impersonate api + local resolution of import by lsp v0 ([7675f08](https://github.com/windmill-labs/windmill/commit/7675f08b7bfe319e496a86a7ef1ab7cc8c1d12d2))
|
||||
* **frontend:** add workspace to ctx ([8f7a11b](https://github.com/windmill-labs/windmill/commit/8f7a11b8964e2c3405ce3689f9cf2298f9e71c75))
|
||||
* **frontend:** Improve login + toasts ([#1363](https://github.com/windmill-labs/windmill/issues/1363)) ([92be102](https://github.com/windmill-labs/windmill/commit/92be102a070b1f17b9d3e40524cd21b54301b5a7))
|
||||
* **frontend:** make script editor a single page ([b84be60](https://github.com/windmill-labs/windmill/commit/b84be60c53ca1ef65826123f39099d33c1f549c0))
|
||||
* **frontend:** Tone down text + display whole text ([#1366](https://github.com/windmill-labs/windmill/issues/1366)) ([f214d5f](https://github.com/windmill-labs/windmill/commit/f214d5f96b6ac26cd3ef90a6ab696a6dfe02b3f0))
|
||||
* improved cron/schedule editor ([#1362](https://github.com/windmill-labs/windmill/issues/1362)) ([17176bb](https://github.com/windmill-labs/windmill/commit/17176bb8d112b35228ce9183f4b2f81abe9e5b6e))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** allow cors ([8a594a8](https://github.com/windmill-labs/windmill/commit/8a594a89adba9915508884f900f58c4ab53cdfec))
|
||||
* **backend:** allow longer name/company ([eff61bb](https://github.com/windmill-labs/windmill/commit/eff61bb8d3496bc1c5be4b1051f99ed4470a47ff))
|
||||
* **backend:** always flush bash output ([517b2c9](https://github.com/windmill-labs/windmill/commit/517b2c9cca54628c8ee692d65c05bc2513eaaf22))
|
||||
* **backend:** always flush bash output ([7a9091f](https://github.com/windmill-labs/windmill/commit/7a9091fed6aa99201b75bab88d4faddbe041eee4))
|
||||
* **backend:** inline script app python fix ([8c72722](https://github.com/windmill-labs/windmill/commit/8c72722710db8e3720b01180b504cbc66e79f5ca))
|
||||
* **frontend:** Add FlowGraph display on Safari ([#1351](https://github.com/windmill-labs/windmill/issues/1351)) ([2819b09](https://github.com/windmill-labs/windmill/commit/2819b09ce5011a467e994ee8b1f09cf33145003d))
|
||||
* **frontend:** Fix button poppup ([#1368](https://github.com/windmill-labs/windmill/issues/1368)) ([a344928](https://github.com/windmill-labs/windmill/commit/a344928f251d697f53e40c517b0b86bd90e0ad52))
|
||||
* **frontend:** Fix connected property ([#1371](https://github.com/windmill-labs/windmill/issues/1371)) ([4af39f0](https://github.com/windmill-labs/windmill/commit/4af39f081bf3d07aaade39e5a5a221741fe8f973))
|
||||
* **frontend:** Fix flow templateEditor ([#1367](https://github.com/windmill-labs/windmill/issues/1367)) ([51fc436](https://github.com/windmill-labs/windmill/commit/51fc436456104c2d6a3cd6f6d62f08929e40d450))
|
||||
* **frontend:** make croninput a builder rather than a tab ([266b5b0](https://github.com/windmill-labs/windmill/commit/266b5b00da3bd7643eaa5dba1b8c1456f11c5e30))
|
||||
* **frontend:** Minor fixes ([#1374](https://github.com/windmill-labs/windmill/issues/1374)) ([76a2a1d](https://github.com/windmill-labs/windmill/commit/76a2a1db363facbaf9a0e9618f169d6cc66e946f))
|
||||
* no need to map internal ports to hosts ([#1365](https://github.com/windmill-labs/windmill/issues/1365)) ([4ec035b](https://github.com/windmill-labs/windmill/commit/4ec035b09a58f8859bc576b03c24cc73f335f32d))
|
||||
|
||||
## [1.85.0](https://github.com/windmill-labs/windmill/compare/v1.84.1...v1.85.0) (2023-04-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add local cache for folder path used + invalidate cache on folder creation ([018b051](https://github.com/windmill-labs/windmill/commit/018b051781e3f40b9d1da8ccdd5edb1cd49877ba))
|
||||
* **frontend:** add agGrid api hooks + ready ([de1e294](https://github.com/windmill-labs/windmill/commit/de1e29492c9aefdfc59f605ba81f7c51a96bf2f3))
|
||||
* **frontend:** Add ID renaming popup ([#1344](https://github.com/windmill-labs/windmill/issues/1344)) ([0b8a08c](https://github.com/windmill-labs/windmill/commit/0b8a08cb49644da7c354c3631751e925ac5353b9))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** improve handling subflow with many depth using tailrec ([8c53598](https://github.com/windmill-labs/windmill/commit/8c53598aba3fb89f4174d1c0ab3912096ac07c96))
|
||||
* **backend:** improve subflow processing ([390a988](https://github.com/windmill-labs/windmill/commit/390a988d4c96256a4fbd6a9302fc47a5648c2c43))
|
||||
* **frontend:** PDF reader header positioning ([#1350](https://github.com/windmill-labs/windmill/issues/1350)) ([daf8276](https://github.com/windmill-labs/windmill/commit/daf827666b13917f8c9abeab5bb2b072bd0fef0b))
|
||||
|
||||
## [1.84.1](https://github.com/windmill-labs/windmill/compare/v1.84.0...v1.84.1) (2023-03-31)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** overwrite instead of smart diff ([b6d5eef](https://github.com/windmill-labs/windmill/commit/b6d5eef5479e38cc36af2db67d4c45f78c622b9a))
|
||||
|
||||
## [1.84.0](https://github.com/windmill-labs/windmill/compare/v1.83.1...v1.84.0) (2023-03-31)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add force cancel ([fbe5c18](https://github.com/windmill-labs/windmill/commit/fbe5c18da02763371e6f32c898b31a6a29984b45))
|
||||
* add the ability to edit previous versions ([2368da2](https://github.com/windmill-labs/windmill/commit/2368da214660ff1835b49b4c2c87256c9bd565cf))
|
||||
* **backend:** reduce memory allocation for big forloops of flows ([c7506e4](https://github.com/windmill-labs/windmill/commit/c7506e4daec5b12bf908e6954bf6f3521a97b3ba))
|
||||
* **frontend:** App component style input grouping ([#1334](https://github.com/windmill-labs/windmill/issues/1334)) ([01564f0](https://github.com/windmill-labs/windmill/commit/01564f0a1c26ee9f065bb0adeb7d5e8df0b2e5b5))
|
||||
* **frontend:** Display frontend execution result in Debug Runs ([#1341](https://github.com/windmill-labs/windmill/issues/1341)) ([57f8dd9](https://github.com/windmill-labs/windmill/commit/57f8dd9570577a58fe91d93c7a9d1a9b4dc69598))
|
||||
* **frontend:** improve input connection UI ([#1333](https://github.com/windmill-labs/windmill/issues/1333)) ([5ac646e](https://github.com/windmill-labs/windmill/commit/5ac646e859a07efb65542aae9365aa7791ce1097))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** add a refresh button to workspace script/hub ([bb61cef](https://github.com/windmill-labs/windmill/commit/bb61cef0e56bf7fa7f8a5f91dabd590afd5db791))
|
||||
* **backend:** backend compatability on macos ([#1340](https://github.com/windmill-labs/windmill/issues/1340)) ([dfd2abc](https://github.com/windmill-labs/windmill/commit/dfd2abc76466cddca98f93fd82be91ba5d3076e0))
|
||||
* **frontend:** Export python code as string ([#1339](https://github.com/windmill-labs/windmill/issues/1339)) ([2779891](https://github.com/windmill-labs/windmill/commit/277989141100b033b26b496b8a55d97d48cf7e81))
|
||||
* **frontend:** improve app tables ([cd1f9b6](https://github.com/windmill-labs/windmill/commit/cd1f9b6baa0dadfb14fee3a586a4b6b164e5e402))
|
||||
* **frontend:** improve loading of big args in job details ([71619ac](https://github.com/windmill-labs/windmill/commit/71619acdfac010822c1eac496a6f3f869e6ca6fb))
|
||||
* **frontend:** improve loading of big jobs in run form ([b325493](https://github.com/windmill-labs/windmill/commit/b3254938fe58d8c00a0c4347e7ef519e3a6e4031))
|
||||
|
||||
## [1.83.1](https://github.com/windmill-labs/windmill/compare/v1.83.0...v1.83.1) (2023-03-28)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** plain secrets might be undefined ([569a55e](https://github.com/windmill-labs/windmill/commit/569a55e45b34641b0fb4569387166f3aa89ce35f))
|
||||
|
||||
## [1.83.0](https://github.com/windmill-labs/windmill/compare/v1.82.0...v1.83.0) (2023-03-28)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** allow relative imports for python ([a5500ea](https://github.com/windmill-labs/windmill/commit/a5500ea40a77b2e0408e2a644190a8f65b18cd1d))
|
||||
* **backend:** execute /bin/bash instead of /bin/sh for bash scripts ([021fa23](https://github.com/windmill-labs/windmill/commit/021fa23f9ffcd11548977a4589eb9bc2815243cf))
|
||||
* **backend:** improve relative importsfor deno ([eaac598](https://github.com/windmill-labs/windmill/commit/eaac598af308cedea8f0f8fc7c189a4640b4366b))
|
||||
* **backend:** increase timeout for premium workspace ([00b70d9](https://github.com/windmill-labs/windmill/commit/00b70d9aaac8ae979782492d7754060a3c2c9567))
|
||||
* **frontend:** add pagination ([33c07d3](https://github.com/windmill-labs/windmill/commit/33c07d3e63f96673719ecb15e45f4cd9e18be80e))
|
||||
* **frontend:** Add quick style settings to app editor ([#1308](https://github.com/windmill-labs/windmill/issues/1308)) ([ac24862](https://github.com/windmill-labs/windmill/commit/ac2486219cd91df3a7fe11d37894797a881cac6c))
|
||||
* **frontend:** add recompute as a primitive ([449d3ae](https://github.com/windmill-labs/windmill/commit/449d3ae5ddeceef3fbcb7a815a4dba16c9639fb3))
|
||||
* **frontend:** add textareacomponent + fix multiselect style + select multi components ([2b31653](https://github.com/windmill-labs/windmill/commit/2b31653a8aa06807678e8609cfa62cf0f2f55dce))
|
||||
* **frontend:** multiselect components for apps ([577dec5](https://github.com/windmill-labs/windmill/commit/577dec5c5733cdf88e8586ce6c27159920c69c8a))
|
||||
* **frontend:** use rich json editor for arrays of objects and for object in ArgInput ([b95afaa](https://github.com/windmill-labs/windmill/commit/b95afaa9bb41b102181657453a564f44f4511983))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **apps:** improve app table actionButtons behavior under many clicks ([8e3d8ac](https://github.com/windmill-labs/windmill/commit/8e3d8acc80de971ee115d6903d24864d8263f08b))
|
||||
* **cli:** add --plain-secrets ([98d51e2](https://github.com/windmill-labs/windmill/commit/98d51e219df1680507114f9b57ec0b0a4a234b5c))
|
||||
* **frontend:** add a modal that is always mounted to make sure compon… ([#1328](https://github.com/windmill-labs/windmill/issues/1328)) ([a527cb8](https://github.com/windmill-labs/windmill/commit/a527cb8222a2ff80dae38ebae7dc5ea0979d74c5))
|
||||
* **frontend:** Disable app keyboard navigation on focused inputs ([#1326](https://github.com/windmill-labs/windmill/issues/1326)) ([da24e9a](https://github.com/windmill-labs/windmill/commit/da24e9ab0625a7503c498c179022ea4011a03170))
|
||||
* **frontend:** persist description for schemas ([1a48673](https://github.com/windmill-labs/windmill/commit/1a4867302f72aaae8f422ac8f53812c116cc383d))
|
||||
* **frontend:** Revert app upload input ([#1330](https://github.com/windmill-labs/windmill/issues/1330)) ([fa457bb](https://github.com/windmill-labs/windmill/commit/fa457bb7099bd31c2315eaf7f7f2c40900b2ae39))
|
||||
* **frontend:** Small app fixes ([#1331](https://github.com/windmill-labs/windmill/issues/1331)) ([75306c8](https://github.com/windmill-labs/windmill/commit/75306c831616d9a01cc3a4681732aab93153f1a9))
|
||||
|
||||
## [1.82.0](https://github.com/windmill-labs/windmill/compare/v1.81.0...v1.82.0) (2023-03-24)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** introduce RESTART_ZOMBIE_JOBS and ZOMBIE_JOB_TIMEOUT ([47a7f71](https://github.com/windmill-labs/windmill/commit/47a7f7163aae3fe807e766c824085b4d1b75c8c8))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** do not consider FlowPreview as potential zombie job ([f7c30b5](https://github.com/windmill-labs/windmill/commit/f7c30b5d2f16e15f36208e07126557fd7ed84801))
|
||||
* **backend:** increase dynamic js timeout + improve client passing ([34e25f0](https://github.com/windmill-labs/windmill/commit/34e25f0f96fe637cc42f4017a064c40def5d67ef))
|
||||
* **cli:** improve diff speed + fix replacing cli ([b999c98](https://github.com/windmill-labs/windmill/commit/b999c9894b4011b735f37df485fe403c22c00512))
|
||||
* **frontend:** Fix AppTable error display + clear errors when removing a component + properly detect that latest component run had an error ([#1322](https://github.com/windmill-labs/windmill/issues/1322)) ([c15bc8a](https://github.com/windmill-labs/windmill/commit/c15bc8a7bfb3bef2634e6093088967137cd06239))
|
||||
* **frontend:** fix refresh with manual dependencies ([#1319](https://github.com/windmill-labs/windmill/issues/1319)) ([a47031a](https://github.com/windmill-labs/windmill/commit/a47031a41e6a3392101e280dcd1aea098f898447))
|
||||
* **frontend:** fix settings panel ([#1323](https://github.com/windmill-labs/windmill/issues/1323)) ([30b8e47](https://github.com/windmill-labs/windmill/commit/30b8e474df5b71b7e7b36d3fe5974a289cf0dfae))
|
||||
* **frontend:** Fix transformer ([#1321](https://github.com/windmill-labs/windmill/issues/1321)) ([addabcc](https://github.com/windmill-labs/windmill/commit/addabcceb0c90782ba4a934bb3822f8cc9865069))
|
||||
* **frontend:** remove unnecessary div ([#1318](https://github.com/windmill-labs/windmill/issues/1318)) ([e193a0b](https://github.com/windmill-labs/windmill/commit/e193a0bcdf6690b007594d2f1325a7ec26603129))
|
||||
|
||||
## [1.81.0](https://github.com/windmill-labs/windmill/compare/v1.80.1...v1.81.0) (2023-03-21)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **apps:** add action on form/button/formbutton ([2593218](https://github.com/windmill-labs/windmill/commit/2593218cbf07c05521a270797055ddb22dc22b8d))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** Remove action outline on preview mode ([#1313](https://github.com/windmill-labs/windmill/issues/1313)) ([a7c4f1a](https://github.com/windmill-labs/windmill/commit/a7c4f1a12e02e8627a5955b75d572e9cf11d8122))
|
||||
|
||||
## [1.80.1](https://github.com/windmill-labs/windmill/compare/v1.80.0...v1.80.1) (2023-03-21)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** add support for non metadataed scripts ([42f6d2e](https://github.com/windmill-labs/windmill/commit/42f6d2e0ee6294f8a1d97f5f62f2adb6edfd2fed))
|
||||
|
||||
## [1.80.0](https://github.com/windmill-labs/windmill/compare/v1.79.0...v1.80.0) (2023-03-20)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **apps:** add transformers for data sources ([0abacac](https://github.com/windmill-labs/windmill/commit/0abacac06c7dd586b48c66ff47b7589fe692205b))
|
||||
* **frontend:** App set tab ([#1307](https://github.com/windmill-labs/windmill/issues/1307)) ([48413a7](https://github.com/windmill-labs/windmill/commit/48413a78c5e7e0ee8208711f15135d81136b7386))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** add missing optional chaining ([#1306](https://github.com/windmill-labs/windmill/issues/1306)) ([29b1cc6](https://github.com/windmill-labs/windmill/commit/29b1cc6ff0ebc5edcad24a1780113889c507075d))
|
||||
* **frontend:** App button triggered by ([#1304](https://github.com/windmill-labs/windmill/issues/1304)) ([cf2d031](https://github.com/windmill-labs/windmill/commit/cf2d031e8e89faa2cd7fa58436cbe7cf4d9045f9))
|
||||
|
||||
## [1.79.0](https://github.com/windmill-labs/windmill/compare/v1.78.0...v1.79.0) (2023-03-17)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add listeners for frontend scripts ([597e38e](https://github.com/windmill-labs/windmill/commit/597e38ef367d38fa97fc443ccb2c721e5964fece))
|
||||
* **frontend:** add table actions navigation ([#1298](https://github.com/windmill-labs/windmill/issues/1298)) ([c3ba1a6](https://github.com/windmill-labs/windmill/commit/c3ba1a6ab97484a08a5a20187bb858a5af7025cb))
|
||||
* **frontend:** App component triggers ([#1303](https://github.com/windmill-labs/windmill/issues/1303)) ([078cb1b](https://github.com/windmill-labs/windmill/commit/078cb1bf3e4de08cb018578f04d24392a6462f69))
|
||||
* **frontend:** Component control ([#1293](https://github.com/windmill-labs/windmill/issues/1293)) ([bd927a2](https://github.com/windmill-labs/windmill/commit/bd927a27ed9581dbf67ea3694f9d989f8d71d2ed))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** App panel styling ([#1284](https://github.com/windmill-labs/windmill/issues/1284)) ([c1dd35c](https://github.com/windmill-labs/windmill/commit/c1dd35c3f0fcbc1be43273f82a873c3c07863417))
|
||||
* **frontend:** Display app context search on top ([#1300](https://github.com/windmill-labs/windmill/issues/1300)) ([bd3ee81](https://github.com/windmill-labs/windmill/commit/bd3ee81b14846f16ccd16461de99b46fe68be6ba))
|
||||
* **frontend:** fix horizontal splitpanes ([#1301](https://github.com/windmill-labs/windmill/issues/1301)) ([ea3dab4](https://github.com/windmill-labs/windmill/commit/ea3dab411b3d5dd772e04c8831e789e2470aaf28))
|
||||
* **frontend:** fix map render ([#1297](https://github.com/windmill-labs/windmill/issues/1297)) ([0092721](https://github.com/windmill-labs/windmill/commit/00927210fd68c31cb793ef4f0efea05711ebcf00))
|
||||
* **frontend:** Hide archive toggle with empty list ([#1296](https://github.com/windmill-labs/windmill/issues/1296)) ([bac831b](https://github.com/windmill-labs/windmill/commit/bac831b23ce85a683ddbd4537900670a0b7d12a8))
|
||||
|
||||
## [1.78.0](https://github.com/windmill-labs/windmill/compare/v1.77.0...v1.78.0) (2023-03-16)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** app textcomponent editable + tooltip ([11567d6](https://github.com/windmill-labs/windmill/commit/11567d6280ea60f1a8c3c6607c724179775cbbe3))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** whitelist for include_header was ignored in some cases ([183a459](https://github.com/windmill-labs/windmill/commit/183a4591df700ab4720de6e92a83631256940089))
|
||||
* **frontend:** improve rendering performance after component moving ([6f890f2](https://github.com/windmill-labs/windmill/commit/6f890f2120885f90d986fbd655096b45bf9de539))
|
||||
* **frontend:** remove staticOutputs from apps ([dbdfd62](https://github.com/windmill-labs/windmill/commit/dbdfd626386398180ecba7976714f86365eeccd8))
|
||||
|
||||
## [1.77.0](https://github.com/windmill-labs/windmill/compare/v1.76.0...v1.77.0) (2023-03-14)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **apps:** state can be used as input in apps ([2f0acb9](https://github.com/windmill-labs/windmill/commit/2f0acb9ffa8dace4a886527dcee49809d019b271))
|
||||
* **apps:** tabs can be made pages or invisible + better frontend scripts reactivity ([cd645d0](https://github.com/windmill-labs/windmill/commit/cd645d0935f2d06e0ff71f14d2cf63accd378ff3))
|
||||
* **deno:** add support for custom npm repo ([#1291](https://github.com/windmill-labs/windmill/issues/1291)) ([944795f](https://github.com/windmill-labs/windmill/commit/944795f6eeaa7d01ab1a35a80570a55c363723e6))
|
||||
* **frontend:** add setTab to frontend scripts ([c2a97c5](https://github.com/windmill-labs/windmill/commit/c2a97c53cfff0fdb35dd8bc249490566eebdc1a9))
|
||||
* **frontend:** app components output panel ([#1283](https://github.com/windmill-labs/windmill/issues/1283)) ([751edcf](https://github.com/windmill-labs/windmill/commit/751edcf9b8e0976a1d073603c9eff5dc6e714490))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** do not cache reference to workspace scripts ([eb73f2a](https://github.com/windmill-labs/windmill/commit/eb73f2a687f6faad301b9038ab8585450bec7481))
|
||||
* **frontend:** fix app tabs ([#1288](https://github.com/windmill-labs/windmill/issues/1288)) ([c71a577](https://github.com/windmill-labs/windmill/commit/c71a577fead90c9cd01a736b54d859ec4f0b7807))
|
||||
* **frontend:** fix container deletion ([#1287](https://github.com/windmill-labs/windmill/issues/1287)) ([bc870bd](https://github.com/windmill-labs/windmill/commit/bc870bd03eb76cb8bc0e0c861f6cd8a9c661186b))
|
||||
* **frontend:** Update setting accordion ([#1285](https://github.com/windmill-labs/windmill/issues/1285)) ([dea12e8](https://github.com/windmill-labs/windmill/commit/dea12e8870ece998bb6607723cbaab9b9a958f22))
|
||||
|
||||
## [1.76.0](https://github.com/windmill-labs/windmill/compare/v1.75.0...v1.76.0) (2023-03-13)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add frontend (JS) scripts to apps ([f0b1b1f](https://github.com/windmill-labs/windmill/commit/f0b1b1f752731ba434b960a75624118152f53c00))
|
||||
* **frontend:** Copy, Cut and Paste ([#1279](https://github.com/windmill-labs/windmill/issues/1279)) ([82c139e](https://github.com/windmill-labs/windmill/commit/82c139ed0992be401e250cfb7ecc0fca61b76772))
|
||||
* **frontend:** disabled for action buttons can now depend on row ([75f87e7](https://github.com/windmill-labs/windmill/commit/75f87e7e1117a9c12afcf626379e94b134a9a493))
|
||||
* **frontend:** improve drag-n-drop behavior ([cfd489a](https://github.com/windmill-labs/windmill/commit/cfd489a55059e7b6843f99bab261c70b3852e6a2))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** improve worker ping api ([c958480](https://github.com/windmill-labs/windmill/commit/c958480ce83844a989f58dd5a70eb288582e2194))
|
||||
* **frontend:** General fixes and updates ([#1281](https://github.com/windmill-labs/windmill/issues/1281)) ([3e5a179](https://github.com/windmill-labs/windmill/commit/3e5a179eb8cd8001f49c92305141dade1571e20f))
|
||||
|
||||
## [1.75.0](https://github.com/windmill-labs/windmill/compare/v1.74.2...v1.75.0) (2023-03-11)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add filter jobs by args or result ([3b44f9a](https://github.com/windmill-labs/windmill/commit/3b44f9a72ca0466a44963a4b9657a0ee59b44753))
|
||||
* **apps:** add resource picker ([8681e83](https://github.com/windmill-labs/windmill/commit/8681e83b574141acbf7e5a389a9e8a4f340336d1))
|
||||
* **bash:** add default argument handling for bash ([1d5c194](https://github.com/windmill-labs/windmill/commit/1d5c194f09ffba963d52e418c5954843d84ae337))
|
||||
* **frontend-apps:** add variable picker for static string input on apps ([bc440f8](https://github.com/windmill-labs/windmill/commit/bc440f8d4154ce464c0e027d93b7a0a3b76d782e))
|
||||
* **frontend:** make runs filters synced with query args ([61a5e1f](https://github.com/windmill-labs/windmill/commit/61a5e1f1accc988628b785b3b9be04c4ea719874))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** add killpill for lines reading ([7c825c2](https://github.com/windmill-labs/windmill/commit/7c825c212dd0f1e8be427eabd9a9756303241d1b))
|
||||
* **cli:** many small fixes ([ce32370](https://github.com/windmill-labs/windmill/commit/ce323709a94d27fb24214719180ea1aafc66d646))
|
||||
|
||||
## [1.74.2](https://github.com/windmill-labs/windmill/compare/v1.74.1...v1.74.2) (2023-03-09)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** fix splitpanes navigation ([#1276](https://github.com/windmill-labs/windmill/issues/1276)) ([8d5c5b8](https://github.com/windmill-labs/windmill/commit/8d5c5b88a35d7a3bad1d8ddf2d940026825241eb))
|
||||
|
||||
## [1.74.1](https://github.com/windmill-labs/windmill/compare/v1.74.0...v1.74.1) (2023-03-09)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **apps:** proper reactivity for non rendered static components ([ae53baf](https://github.com/windmill-labs/windmill/commit/ae53bafaf6777f928113f84b2c6ed6a2ed341844))
|
||||
* **ci:** make windmill compile again by pinning swc deps ([2ea15d5](https://github.com/windmill-labs/windmill/commit/2ea15d5035e5e15473968db3c0501a4dddff5cd0))
|
||||
|
||||
## [1.74.0](https://github.com/windmill-labs/windmill/compare/v1.73.1...v1.74.0) (2023-03-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add delete by path for scripts ([0c2cf92](https://github.com/windmill-labs/windmill/commit/0c2cf92dd3df9610e649f15e23921a4ca0d94e6a))
|
||||
* **frontend:** Add color picker input to app ([#1270](https://github.com/windmill-labs/windmill/issues/1270)) ([88e537a](https://github.com/windmill-labs/windmill/commit/88e537ad1fb4c207f38fbe951c82106bef6491a3))
|
||||
* **frontend:** add expand ([#1268](https://github.com/windmill-labs/windmill/issues/1268)) ([b854ee3](https://github.com/windmill-labs/windmill/commit/b854ee34393534bde104e2e6f606108fd66d38dc))
|
||||
* **frontend:** add hash to ctx in apps ([b1a45b1](https://github.com/windmill-labs/windmill/commit/b1a45b1e708aa6f19f8be9c949507083e044f2d8))
|
||||
* **frontend:** Add key navigation in app editor ([#1273](https://github.com/windmill-labs/windmill/issues/1273)) ([6b0fb75](https://github.com/windmill-labs/windmill/commit/6b0fb75d23e2151c88b07814139d203c1bd0578d))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** improve visibility of the active workspace ([e6344da](https://github.com/windmill-labs/windmill/commit/e6344dac6d1be04b46231fa8ef8579fd12ca8f37))
|
||||
* **frontend:** add confirmation modal to delete script/flow/app ([a4adcb5](https://github.com/windmill-labs/windmill/commit/a4adcb5192c11f7bf47a0d259825e474779378d7))
|
||||
* **frontend:** Clean up app editor ([#1267](https://github.com/windmill-labs/windmill/issues/1267)) ([0a5e181](https://github.com/windmill-labs/windmill/commit/0a5e181a3aa966fb8211bee0d9174fc16353b31f))
|
||||
* **frontend:** Minor changes ([#1272](https://github.com/windmill-labs/windmill/issues/1272)) ([3b6ae0c](https://github.com/windmill-labs/windmill/commit/3b6ae0cc49461b858d9cfff79eae9a7569465235))
|
||||
* **frontend:** simplify input bindings ([b2de531](https://github.com/windmill-labs/windmill/commit/b2de531a46e4b120d7106d361b727746bec516dd))
|
||||
|
||||
## [1.73.1](https://github.com/windmill-labs/windmill/compare/v1.73.0...v1.73.1) (2023-03-07)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** load flow is not initialized ([719d475](https://github.com/windmill-labs/windmill/commit/719d4752621d462b1cfaa0d27930fba7586be779))
|
||||
|
||||
## [1.73.0](https://github.com/windmill-labs/windmill/compare/v1.72.0...v1.73.0) (2023-03-07)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add a way to automatically resize ([#1259](https://github.com/windmill-labs/windmill/issues/1259)) ([24f58ef](https://github.com/windmill-labs/windmill/commit/24f58efd9994a2201c1b1d9bbfb11734c57068e3))
|
||||
* **frontend:** add ability to move nodes ([614fb50](https://github.com/windmill-labs/windmill/commit/614fb5022aa7d5428fb96b7ee3a20794edd1e9d3))
|
||||
* **frontend:** Add app PDF viewer ([#1254](https://github.com/windmill-labs/windmill/issues/1254)) ([3e5d09e](https://github.com/windmill-labs/windmill/commit/3e5d09ef0b5619186bee5ec6d442cbfd12a6e8d5))
|
||||
* **frontend:** add fork/save buttons + consistent styling for slider/range ([9e9f8ef](https://github.com/windmill-labs/windmill/commit/9e9f8efb8ee389ea75e99b67ef720756959ca737))
|
||||
* **frontend:** add history to flows and apps ([9e4d90a](https://github.com/windmill-labs/windmill/commit/9e4d90ad37a57ff1f515eea0c82cf603649e915d))
|
||||
* **frontend:** Fix object viewer style ([#1255](https://github.com/windmill-labs/windmill/issues/1255)) ([94f1aad](https://github.com/windmill-labs/windmill/commit/94f1aadef2b09ac1962478f11b27cc708b8328f1))
|
||||
* **frontend:** refactor entire flow builder UX ([2ac51b0](https://github.com/windmill-labs/windmill/commit/2ac51b0af08bdef7ce3c7e874e9983b9fc00478a))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** arginput + apppreview fixes ([e2c4545](https://github.com/windmill-labs/windmill/commit/e2c45452401022b00285b21551ffaf35a114be33))
|
||||
* **frontend:** fix app map reactivity ([#1260](https://github.com/windmill-labs/windmill/issues/1260)) ([2557e13](https://github.com/windmill-labs/windmill/commit/2557e136bd0df1a023819b7d9b2235e30d7140b6))
|
||||
* **frontend:** fix branch deletion ([#1261](https://github.com/windmill-labs/windmill/issues/1261)) ([a999eb2](https://github.com/windmill-labs/windmill/commit/a999eb21121a7c0010621448324e0c77caf2b3f6))
|
||||
* **frontend:** Side menu z-index issue ([#1265](https://github.com/windmill-labs/windmill/issues/1265)) ([c638897](https://github.com/windmill-labs/windmill/commit/c638897fdcd58f55b0929f91641b21a6f9d25ead))
|
||||
|
||||
## [1.72.0](https://github.com/windmill-labs/windmill/compare/v1.71.0...v1.72.0) (2023-03-02)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** get_result_by_id do a downward pass to find node at any depth ([#1249](https://github.com/windmill-labs/windmill/issues/1249)) ([4c913dc](https://github.com/windmill-labs/windmill/commit/4c913dc4b6be03571a015c97a13829adffb61479))
|
||||
* **frontend:** Add app map component ([#1251](https://github.com/windmill-labs/windmill/issues/1251)) ([ed25d9f](https://github.com/windmill-labs/windmill/commit/ed25d9f186d9925f75404cb193a025d8a41c4540))
|
||||
* **frontend:** app splitpanes ([#1248](https://github.com/windmill-labs/windmill/issues/1248)) ([f4d79ee](https://github.com/windmill-labs/windmill/commit/f4d79ee2633e6cdab0fa2410108b31cfa77e10da))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** improve result retrieval ([c4463bb](https://github.com/windmill-labs/windmill/commit/c4463bb029907f3c8d77abb194f872aae7876bf6))
|
||||
* **backend:** incorrect get_result_by_id for list_result job ([2a75cd2](https://github.com/windmill-labs/windmill/commit/2a75cd250ea5e01849fc8bbb69bf44f147d0acb8))
|
||||
* **cli:** fix workspace option + run script/flow + whoami ([35ea2b2](https://github.com/windmill-labs/windmill/commit/35ea2b27b12159c68c8507ec1f8686028c975387))
|
||||
* **frontend:** background script not showing inputs ([55eb48c](https://github.com/windmill-labs/windmill/commit/55eb48c55332431304cedbf3bcbbbcff61ec3645))
|
||||
* **frontend:** fix table bindings ([2679386](https://github.com/windmill-labs/windmill/commit/2679386bf87a56352269911bd89e52df5ee9f314))
|
||||
* **frontend:** rework app reactivity ([94b20d2](https://github.com/windmill-labs/windmill/commit/94b20d2f5e3b551974c57ea82b6e3dc16e97b9b8))
|
||||
* **frontend:** rework app reactivity ([1753cb7](https://github.com/windmill-labs/windmill/commit/1753cb7da658f47be974c15da82c71a8e19309a6))
|
||||
|
||||
## [1.71.0](https://github.com/windmill-labs/windmill/compare/v1.70.1...v1.71.0) (2023-02-28)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** use counter for sleep/execution/pull durations ([e568690](https://github.com/windmill-labs/windmill/commit/e56869092a03fec4703ddd9ef65c89edb8122962))
|
||||
* **cli:** add autocompletions ([287b2db](https://github.com/windmill-labs/windmill/commit/287b2db22f7b56e90bcd0c4727c00096695c2e0d))
|
||||
* **frontend:** App drawer ([#1246](https://github.com/windmill-labs/windmill/issues/1246)) ([8a0d115](https://github.com/windmill-labs/windmill/commit/8a0d1158c4d7e970cb91e1adf4838e5efdbb39ff))
|
||||
* **frontend:** drawer for editing workspace scripts in flows ([6adc875](https://github.com/windmill-labs/windmill/commit/6adc87561070d8aceaba1838008cd7e6be2e2660))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** Add more app custom css ([#1229](https://github.com/windmill-labs/windmill/issues/1229)) ([a4e4d18](https://github.com/windmill-labs/windmill/commit/a4e4d188ad10443dd0b7f104389594efc768dc59))
|
||||
* **frontend:** Add more app custom css ([#1247](https://github.com/windmill-labs/windmill/issues/1247)) ([1bb5ed9](https://github.com/windmill-labs/windmill/commit/1bb5ed9ae01fd7998b06833b6222e5dd5d774d35))
|
||||
* **frontend:** display currently selected filter even if not in list ([42d1cd6](https://github.com/windmill-labs/windmill/commit/42d1cd6456620ba917c560c87d736dc93634adff))
|
||||
* **frontend:** Fix deeply nested move ([#1245](https://github.com/windmill-labs/windmill/issues/1245)) ([a67f10e](https://github.com/windmill-labs/windmill/commit/a67f10eeb6fdb44bbb3a510badcc5ad0ae187a2b))
|
||||
* **frontend:** invisible subgrids have h-0 + app policies fix ([2244e83](https://github.com/windmill-labs/windmill/commit/2244e83b9da803a4cf46ab0825d7cb6cb0e24872))
|
||||
|
||||
## [1.70.1](https://github.com/windmill-labs/windmill/compare/v1.70.0...v1.70.1) (2023-02-27)
|
||||
|
||||
|
||||
|
||||
14
Caddyfile
14
Caddyfile
@@ -1,5 +1,15 @@
|
||||
{$BASE_URL} {
|
||||
{
|
||||
auto_https off
|
||||
}
|
||||
|
||||
http://{$BASE_URL} {
|
||||
bind {$ADDRESS}
|
||||
reverse_proxy /ws/* http://lsp:3001
|
||||
reverse_proxy /* http://windmill_server:8000
|
||||
}
|
||||
}
|
||||
|
||||
https://{$BASE_URL} {
|
||||
bind {$ADDRESS}
|
||||
reverse_proxy /ws/* http://localhost:3001
|
||||
}
|
||||
}
|
||||
|
||||
12
Dockerfile
12
Dockerfile
@@ -39,7 +39,7 @@ WORKDIR /windmill
|
||||
ENV SQLX_OFFLINE=true
|
||||
ENV CARGO_INCREMENTAL=1
|
||||
|
||||
FROM node:20-alpine as frontend
|
||||
FROM node:19-alpine as frontend
|
||||
|
||||
# install dependencies
|
||||
WORKDIR /frontend
|
||||
@@ -73,7 +73,7 @@ ARG features=""
|
||||
|
||||
COPY --from=planner /windmill/recipe.json recipe.json
|
||||
|
||||
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
|
||||
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
|
||||
|
||||
COPY ./openflow.openapi.yaml /openflow.openapi.yaml
|
||||
COPY ./backend ./
|
||||
@@ -85,8 +85,7 @@ COPY .git/ .git/
|
||||
RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
|
||||
|
||||
|
||||
FROM python:3.11.3-slim-buster
|
||||
ARG TARGETPLATFORM
|
||||
FROM python:3.11.2-slim-buster
|
||||
|
||||
ARG APP=/usr/src/app
|
||||
|
||||
@@ -124,17 +123,12 @@ ENV TZ=Etc/UTC
|
||||
|
||||
RUN /usr/local/bin/python3 -m pip install pip-tools
|
||||
|
||||
COPY --from=frontend /frontend/build /static_frontend
|
||||
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
|
||||
|
||||
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
|
||||
|
||||
COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
|
||||
|
||||
# docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image
|
||||
COPY --from=lukechannings/deno:latest /usr/bin/deno /usr/bin/deno-arm
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rm /usr/bin/deno-arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then mv /usr/bin/deno-arm /usr/bin/deno; fi
|
||||
|
||||
RUN mkdir -p ${APP}
|
||||
|
||||
WORKDIR ${APP}
|
||||
|
||||
8
LICENSE
8
LICENSE
@@ -8,9 +8,5 @@ or belonging to one of the below cases:
|
||||
|
||||
The files under backend/ are AGPL Licensed.
|
||||
The files under frontend/ are AGPL Licensed.
|
||||
The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
|
||||
|
||||
The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
|
||||
|
||||
All third party components incorporated into the Windmill Software are licensed under the
|
||||
original license provided by the owner of the applicable component.
|
||||
The files under python-client/ are Apache 2.0 Licensed.
|
||||
The files under community/ are Apache 2.0 Licensed.
|
||||
|
||||
178
README.md
178
README.md
@@ -5,7 +5,7 @@
|
||||
<em>.</em>
|
||||
</p>
|
||||
<p align=center>
|
||||
Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
|
||||
Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -26,8 +26,7 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
|
||||
|
||||
# Windmill - Turn scripts into workflows and UIs that you can share and run at scale
|
||||
|
||||
Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
|
||||
dedicated instance and commercial support and licenses.
|
||||
Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers dedicated instance and commercial support and licenses.
|
||||
|
||||
|
||||
|
||||
@@ -70,12 +69,12 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
|
||||
|
||||
3. Make it flow! You can chain your scripts or scripts made by the community
|
||||
shared on [WindmillHub](https://hub.windmill.dev).
|
||||
|
||||
|
||||
|
||||
4. Build complex UI on top of your scripts and flows.
|
||||
|
||||
|
||||
|
||||
Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
|
||||
Scripts and flows can also be triggered by a cron schedule '*/5 * * * *' or
|
||||
through webhooks.
|
||||
|
||||
You can build your entire infra on top of Windmill!
|
||||
@@ -83,49 +82,46 @@ You can build your entire infra on top of Windmill!
|
||||
## Show me some actual script code
|
||||
|
||||
```typescript
|
||||
import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts";
|
||||
import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts"
|
||||
//import any dependency from npm
|
||||
|
||||
import cowsay from "npm:cowsay@1.5.0";
|
||||
import cowsay from 'npm:cowsay@1.5.0'
|
||||
|
||||
export async function main(
|
||||
a: number,
|
||||
// unions generate enums
|
||||
b: "my" | "enum",
|
||||
// default parameters prefill the field
|
||||
d = "default arg",
|
||||
// nested objects work c = { nested: "object" },
|
||||
// permissioned and typed json
|
||||
db: wmill.Resource<"postgresql">
|
||||
) {
|
||||
const email = Deno.env.get("WM_EMAIL");
|
||||
// variables are permissioned and by path
|
||||
let variable = await wmill.getVariable("f/company-folder/my_secret");
|
||||
const lastTimeRun = await wmill.getState();
|
||||
// logs are printed and always inspectable
|
||||
console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }));
|
||||
await wmill.setState(Date.now());
|
||||
a: number,
|
||||
// unions generate enums
|
||||
b: "my" | "enum",
|
||||
// default parameters prefill the field
|
||||
d = "default arg",
|
||||
// nested objects work c = { nested: "object" },
|
||||
// permissioned and typed json
|
||||
db: wmill.Resource<"postgresql">) {
|
||||
|
||||
// return is serialized as JSON
|
||||
return { foo: d, variable };
|
||||
const email = Deno.env.get('WM_EMAIL')
|
||||
// variables are permissioned and by path
|
||||
let variable = await wmill.getVariable('f/company-folder/my_secret')
|
||||
const lastTimeRun = await wmill.getState()
|
||||
// logs are printed and always inspectable
|
||||
console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }))
|
||||
await wmill.setState(Date.now())
|
||||
|
||||
// return is serialized as JSON
|
||||
return { foo: d, variable };
|
||||
}
|
||||
```
|
||||
|
||||
## CLI
|
||||
|
||||
We have a powerful CLI to interact with the windmill platform and sync your
|
||||
scripts from local files, github repos and to run scripts and flows on the
|
||||
instance from local commands. See
|
||||
scripts from local files, github repos and to run scripts and flows on the instance from local commands. See
|
||||
[more details](https://github.com/windmill-labs/windmill/tree/main/cli)
|
||||
|
||||
|
||||
|
||||
|
||||
### Running scripts locally
|
||||
|
||||
You can run your script locally easily, you simply need to pass the right
|
||||
environment variables for the `wmill` client library to fetch resources and
|
||||
variables from your instance if necessary. See more:
|
||||
<https://docs.windmill.dev/docs/advanced/local_development/>
|
||||
You can run your script locally easily, you simply need to pass the right environment variables for the `wmill` client library to fetch resource and variables from your instance if necessary. See more: <https://docs.windmill.dev/docs/advanced/local_development/>
|
||||
|
||||
## Stack
|
||||
|
||||
@@ -186,19 +182,15 @@ compiling from source or using without a postgres super user, see
|
||||
|
||||
### Docker compose
|
||||
|
||||
```
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
|
||||
|
||||
docker compose up -d --pull always
|
||||
```
|
||||
`docker compose up` with the following docker-compose is sufficient:
|
||||
<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
|
||||
|
||||
Go to http://localhost et voilà :)
|
||||
|
||||
|
||||
The default super-admin user is: admin@windmill.dev / changeme
|
||||
|
||||
From there, you can follow the setup app and creat other users.
|
||||
From there, you can create other users (do not forget to change the password!)
|
||||
|
||||
### Kubernetes (k8s) and Helm charts
|
||||
|
||||
@@ -207,9 +199,9 @@ We publish helm charts at:
|
||||
|
||||
### Postgres without superuser
|
||||
|
||||
If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
|
||||
postgres superuser, you can run `./init-db-as-superuser.sql` to init the
|
||||
required users for windmill.
|
||||
If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a postgres superuser,
|
||||
you can run `./init-db-as-superuser.sql` to init the required users for windmill.
|
||||
|
||||
|
||||
### Commercial license
|
||||
|
||||
@@ -283,66 +275,52 @@ You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
|
||||
### Resource types
|
||||
|
||||
You will also want to import all the approved resource types from
|
||||
[WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
|
||||
it being synced automatically everyday.
|
||||
[WindmillHub](https://hub.windmill.dev). A setup script will prompt
|
||||
you to have it being synced automatically everyday.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Environment Variable name | Default | Description | Api Server/Worker/All |
|
||||
| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
|
||||
| DATABASE_URL | | The Postgres database url. | All |
|
||||
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
|
||||
| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server |
|
||||
| PORT | 8000 | Exposed port | Server |
|
||||
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
|
||||
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
|
||||
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
|
||||
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
|
||||
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server |
|
||||
| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker |
|
||||
| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker |
|
||||
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
|
||||
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
|
||||
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
|
||||
| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker |
|
||||
| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker |
|
||||
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
|
||||
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
|
||||
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
|
||||
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
|
||||
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
|
||||
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
|
||||
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
|
||||
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
|
||||
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
|
||||
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
|
||||
| NETRC | | The netrc content to use a private go registry | Worker |
|
||||
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
|
||||
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
|
||||
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
|
||||
| PATH | None | The path environment variable, usually inherited | Worker |
|
||||
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
|
||||
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
|
||||
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
|
||||
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
|
||||
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
|
||||
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
|
||||
| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker |
|
||||
| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker |
|
||||
| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | |
|
||||
| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker |
|
||||
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
|
||||
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
|
||||
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
|
||||
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails |
|
||||
| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker |
|
||||
| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker |
|
||||
| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server |
|
||||
| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete |
|
||||
| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
|
||||
| WAIT_RESULT_SLOW_POLL_INTERVAL_MS | 200 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
|
||||
| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2 | The duration of fast poll mode before switching to slow poll | Server |
|
||||
| EXIT_AFTER_NO_JOB_FOR_SECS | None | Exit worker if no job is received after duration in secs if defined | Worker |
|
||||
| Environment Variable name | Default | Description | Api Server/Worker/All |
|
||||
| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
|
||||
| DATABASE_URL | | The Postgres database url. | All |
|
||||
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
|
||||
| PORT | 8000 | Exposed port | Server | |
|
||||
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
|
||||
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
|
||||
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
|
||||
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
|
||||
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server |
|
||||
| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker |
|
||||
| TIMEOUT | 300 | The timeout in seconds for the execution of a script | Worker |
|
||||
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
|
||||
| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker |
|
||||
| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker |
|
||||
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
|
||||
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
|
||||
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
|
||||
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
|
||||
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
|
||||
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server | |
|
||||
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
|
||||
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
|
||||
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
|
||||
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
|
||||
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
|
||||
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
|
||||
| PATH | None | The path environment variable, usually inherited | Worker |
|
||||
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
|
||||
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
|
||||
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
|
||||
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
|
||||
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
|
||||
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
|
||||
| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker |
|
||||
| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | |
|
||||
| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker |
|
||||
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
|
||||
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
|
||||
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
|
||||
| NEW_USER_WEBHOOK | None | Webhook to notify of a new user added, signup/invite. Can hook back to windmill to send emails | Server |
|
||||
|
||||
## Run a local dev setup
|
||||
|
||||
|
||||
@@ -8,14 +8,3 @@ rustflags = [
|
||||
]
|
||||
incremental = true
|
||||
|
||||
[target.x86_64-apple-darwin]
|
||||
rustflags = [
|
||||
"-C", "link-arg=-undefined",
|
||||
"-C", "link-arg=dynamic_lookup",
|
||||
]
|
||||
|
||||
[target.aarch64-apple-darwin]
|
||||
rustflags = [
|
||||
"-C", "link-arg=-undefined",
|
||||
"-C", "link-arg=dynamic_lookup",
|
||||
]
|
||||
3
backend/.vscode/settings.json
vendored
3
backend/.vscode/settings.json
vendored
@@ -1,4 +1,3 @@
|
||||
{
|
||||
"python.analysis.typeCheckingMode": "basic",
|
||||
"rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"]
|
||||
"python.analysis.typeCheckingMode": "basic"
|
||||
}
|
||||
|
||||
1497
backend/Cargo.lock
generated
1497
backend/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.92.1"
|
||||
version = "1.70.1"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -19,7 +19,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.92.1"
|
||||
version = "1.70.1"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -28,7 +28,11 @@ name = "windmill"
|
||||
path = "./src/main.rs"
|
||||
|
||||
[features]
|
||||
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
|
||||
enterprise = [
|
||||
"windmill-worker/enterprise",
|
||||
"windmill-queue/enterprise",
|
||||
"windmill-api/enterprise",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
@@ -48,20 +52,12 @@ git-version.workspace = true
|
||||
rsa.workspace = true
|
||||
base64.workspace = true
|
||||
sha2.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
url.workspace = true
|
||||
lazy_static.workspace = true
|
||||
once_cell.workspace = true
|
||||
prometheus.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
reqwest.workspace = true
|
||||
windmill-queue.workspace = true
|
||||
axum.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
[workspace.dependencies]
|
||||
windmill-api = { path = "./windmill-api" }
|
||||
@@ -80,7 +76,7 @@ headers = "^0"
|
||||
hyper = { version = "^0", features = ["full"] }
|
||||
tokio = { version = "^1", features = ["full", "tracing"] }
|
||||
tower = "^0"
|
||||
tower-http = { version = "^0", features = ["trace", "cors"] }
|
||||
tower-http = { version = "^0", features = ["trace"] }
|
||||
tower-cookies = "^0"
|
||||
serde = "^1"
|
||||
serde_json = { version = "^1", features = ["preserve_order"] }
|
||||
@@ -88,7 +84,6 @@ uuid = { version = "^1", features = ["serde", "v4"] }
|
||||
thiserror = "^1"
|
||||
anyhow = "^1"
|
||||
chrono = { version = "^0", features = ["serde"] }
|
||||
chrono-tz = "^0"
|
||||
tracing = "^0"
|
||||
tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
|
||||
prometheus = { version = "^0", default-features = false }
|
||||
@@ -127,9 +122,9 @@ itertools = "^0"
|
||||
regex = "^1"
|
||||
deno_core = "^0"
|
||||
async-recursion = "^1"
|
||||
swc_common = "0.29.39"
|
||||
swc_ecma_parser = "0.128.2"
|
||||
swc_ecma_ast = "0.98.1"
|
||||
swc_common = "^0"
|
||||
swc_ecma_parser = "^0"
|
||||
swc_ecma_ast = "^0"
|
||||
base64 = "0.21.0"
|
||||
unicode-general-category = "^0"
|
||||
hmac = "0.12.1"
|
||||
@@ -147,7 +142,6 @@ sqlx = { version = "^0", features = [
|
||||
dotenv = "^0"
|
||||
ulid = { version = "^1", features = ["uuid"] }
|
||||
futures = "^0"
|
||||
futures-core = "^0"
|
||||
tokio-metrics = "0.1.0"
|
||||
lazy_static = "1.4.0"
|
||||
serde_derive = "1.0.147"
|
||||
@@ -159,8 +153,3 @@ async-stripe = { version = "0.14", features = [
|
||||
"checkout",
|
||||
] }
|
||||
async_zip = { version = "0.0.11", features = ["full"] }
|
||||
once_cell = "1.17.1"
|
||||
rsmq_async = { version = "5.1.5" }
|
||||
gosyn = "0.2.2"
|
||||
|
||||
[patch.crates-io]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# Windmill Backend
|
||||
|
||||
This folder holds all backend components, the [src/](./src/) folder only
|
||||
contains files used to build the "root" binary.
|
||||
This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
|
||||
|
||||
## Components
|
||||
|
||||
@@ -14,9 +13,3 @@ contains files used to build the "root" binary.
|
||||
| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
|
||||
| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. |
|
||||
| [parsers](./parsers/) | Contains code to parse signatures in different langauges. |
|
||||
|
||||
### Compile sqlx for offline ci
|
||||
|
||||
```
|
||||
cargo sqlx prepare --merged -- --bin windmill --features enterprise
|
||||
```
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE queue ADD COLUMN root_job uuid;
|
||||
ALTER TABLE queue ADD COLUMN leaf_jobs jsonb;
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,11 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE POLICY see_extra_perms_user ON app FOR ALL
|
||||
USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
|
||||
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
|
||||
|
||||
CREATE POLICY see_extra_perms_groups ON app FOR ALL
|
||||
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
|
||||
WITH CHECK (exists(
|
||||
SELECT key, value FROM jsonb_each_text(extra_perms)
|
||||
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
|
||||
AND value::boolean));
|
||||
@@ -1,2 +0,0 @@
|
||||
ALTER TABLE schedule DROP COLUMN timezone;
|
||||
ALTER TABLE schedule ADD COLUMN offset_ INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -1,26 +0,0 @@
|
||||
ALTER TABLE schedule ADD COLUMN timezone VARCHAR(255) NOT NULL DEFAULT 'UTC';
|
||||
|
||||
-- INSERT the correct IANA timezone string for each offset value
|
||||
|
||||
UPDATE schedule SET timezone = 'Pacific/Honolulu' WHERE offset_ = 600;
|
||||
UPDATE schedule SET timezone = 'America/Anchorage' WHERE offset_ = 540;
|
||||
UPDATE schedule SET timezone = 'America/Los_Angeles' WHERE offset_ = 480;
|
||||
UPDATE schedule SET timezone = 'America/Chicago' WHERE offset_ = 360;
|
||||
UPDATE schedule SET timezone = 'America/New_York' WHERE offset_ = 300;
|
||||
UPDATE schedule SET timezone = 'America/Halifax' WHERE offset_ = 240;
|
||||
UPDATE schedule SET timezone = 'America/Sao_Paulo' WHERE offset_ = 180;
|
||||
UPDATE schedule SET timezone = 'Atlantic/South_Georgia' WHERE offset_ = 120;
|
||||
UPDATE schedule SET timezone = 'Atlantic/Cape_Verde' WHERE offset_ = 60;
|
||||
UPDATE schedule SET timezone = 'Europe/London' WHERE offset_ = 0;
|
||||
UPDATE schedule SET timezone = 'Europe/Berlin' WHERE offset_ = -60;
|
||||
UPDATE schedule SET timezone = 'Europe/Athens' WHERE offset_ = -120;
|
||||
UPDATE schedule SET timezone = 'Europe/Moscow' WHERE offset_ = -180;
|
||||
UPDATE schedule SET timezone = 'Asia/Dubai' WHERE offset_ = -240;
|
||||
UPDATE schedule SET timezone = 'Asia/Aqtau' WHERE offset_ = -300;
|
||||
UPDATE schedule SET timezone = 'Asia/Almaty' WHERE offset_ = -360;
|
||||
UPDATE schedule SET timezone = 'Asia/Bangkok' WHERE offset_ = -420;
|
||||
UPDATE schedule SET timezone = 'Asia/Hong_Kong' WHERE offset_ = -480;
|
||||
UPDATE schedule SET timezone = 'Asia/Tokyo' WHERE offset_ = -540;
|
||||
UPDATE schedule SET timezone = 'Australia/Sydney' WHERE offset_ = -600;
|
||||
|
||||
ALTER TABLE schedule DROP COLUMN offset_;
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE password ALTER COLUMN company TYPE VARCHAR(255);
|
||||
ALTER TABLE password ALTER COLUMN name TYPE VARCHAR(255);
|
||||
@@ -1,2 +0,0 @@
|
||||
-- Add down migration script here
|
||||
DROP TABLE pip_resolution_cache;
|
||||
@@ -1,6 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE TABLE pip_resolution_cache(
|
||||
hash VARCHAR(255) PRIMARY KEY,
|
||||
expiration TIMESTAMP NOT NULL,
|
||||
lockfile TEXT NOT NULL
|
||||
);
|
||||
@@ -1,2 +0,0 @@
|
||||
DROP TABLE IF EXISTS input;
|
||||
DROP TYPE RUNNABLE_TYPE;
|
||||
@@ -1,13 +0,0 @@
|
||||
CREATE TYPE RUNNABLE_TYPE AS ENUM ('ScriptHash', 'ScriptPath', 'FlowPath');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS input (
|
||||
id UUID PRIMARY KEY,
|
||||
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
|
||||
runnable_id VARCHAR(255) NOT NULL,
|
||||
runnable_type RUNNABLE_TYPE NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
args JSONB NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
created_by VARCHAR(50) NOT NULL,
|
||||
is_public BOOLEAN NOT NULL DEFAULT FALSE
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
GRANT ALL ON input TO windmill_user;
|
||||
GRANT ALL ON input TO windmill_admin;
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add down migration script here
|
||||
ALTER TABLE script DROP COLUMN tag;
|
||||
ALTER TABLE completed_job DROP COLUMN tag;
|
||||
ALTER TABLE queue DROP COLUMN tag;
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE script ADD COLUMN tag VARCHAR(50);
|
||||
ALTER TABLE completed_job ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
|
||||
ALTER TABLE queue ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add down migration script here
|
||||
DROP TABLE draft;
|
||||
DROP TYPE DRAFT_TYPE;
|
||||
@@ -1,18 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE TYPE DRAFT_TYPE AS ENUM ('script', 'flow', 'app');
|
||||
|
||||
CREATE TABLE draft (
|
||||
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
|
||||
path VARCHAR(255) NOT NULL,
|
||||
typ DRAFT_TYPE NOT NULL,
|
||||
value JSONB NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (workspace_id, path, typ)
|
||||
);
|
||||
|
||||
GRANT ALL ON draft TO windmill_admin;
|
||||
GRANT ALL ON draft TO windmill_user;
|
||||
|
||||
ALTER TABLE script ADD COLUMN draft_only BOOLEAN;
|
||||
ALTER TABLE flow ADD COLUMN draft_only BOOLEAN;
|
||||
ALTER TABLE app ADD COLUMN draft_only BOOLEAN;
|
||||
@@ -16,5 +16,4 @@ unicode-general-category.workspace = true
|
||||
itertools.workspace = true
|
||||
anyhow.workspace = true
|
||||
regex.workspace = true
|
||||
lazy_static.workspace = true
|
||||
serde_json.workspace = true
|
||||
lazy_static.workspace = true
|
||||
@@ -1,8 +1,6 @@
|
||||
#![allow(non_snake_case)] // TODO: switch to parse_* function naming
|
||||
|
||||
use anyhow::anyhow;
|
||||
use regex::Regex;
|
||||
use serde_json::json;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use windmill_parser::{Arg, MainArgSignature, Typ};
|
||||
@@ -19,32 +17,19 @@ pub fn parse_bash_sig(code: &str) -> windmill_common::error::Result<MainArgSigna
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref RE: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+):-(.*)\})"$"#).unwrap();
|
||||
}
|
||||
|
||||
fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
let mut hm: HashMap<i32, (String, Option<String>)> = HashMap::new();
|
||||
for cap in RE.captures_iter(code) {
|
||||
hm.insert(
|
||||
cap.get(2)
|
||||
.or(cap.get(3))
|
||||
.and_then(|x| x.as_str().parse::<i32>().ok())
|
||||
.ok_or_else(|| anyhow!("Impossible to parse arg digit"))?,
|
||||
(
|
||||
cap[1].to_string(),
|
||||
cap.get(4).map(|x| x.as_str().to_string()),
|
||||
),
|
||||
);
|
||||
let mut hm = HashMap::new();
|
||||
let re = Regex::new(r#"(?m)^(\w+)="\$(\d+)"$"#).unwrap();
|
||||
for cap in re.captures_iter(code) {
|
||||
hm.insert(cap[2].parse::<i32>()?, cap[1].to_string());
|
||||
}
|
||||
let mut args = vec![];
|
||||
for i in 1..20 {
|
||||
if hm.contains_key(&i) {
|
||||
let (name, default) = hm.get(&i).unwrap();
|
||||
args.push(Arg {
|
||||
name: name.clone(),
|
||||
name: hm[&i].clone(),
|
||||
typ: Typ::Str(None),
|
||||
default: default.clone().map(|x| json!(x)),
|
||||
default: None,
|
||||
otyp: None,
|
||||
has_default: false,
|
||||
});
|
||||
@@ -58,8 +43,6 @@ fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
@@ -67,7 +50,8 @@ mod tests {
|
||||
let code = r#"
|
||||
token="$1"
|
||||
image="$2"
|
||||
digest="${3:-latest with spaces}"
|
||||
digest="${3:-latest}"
|
||||
foo="$4"
|
||||
|
||||
"#;
|
||||
//println!("{}", serde_json::to_string()?);
|
||||
@@ -90,13 +74,6 @@ digest="${3:-latest with spaces}"
|
||||
typ: Typ::Str(None),
|
||||
default: None,
|
||||
has_default: false
|
||||
},
|
||||
Arg {
|
||||
otyp: None,
|
||||
name: "digest".to_string(),
|
||||
typ: Typ::Str(None),
|
||||
default: Some(json!("latest with spaces")),
|
||||
has_default: false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -15,4 +15,3 @@ phf.workspace = true
|
||||
unicode-general-category.workspace = true
|
||||
itertools.workspace = true
|
||||
anyhow.workspace = true
|
||||
gosyn.workspace = true
|
||||
File diff suppressed because it is too large
Load Diff
347
backend/parsers/windmill-parser-go/src/parser_go_ast.rs
Normal file
347
backend/parsers/windmill-parser-go/src/parser_go_ast.rs
Normal file
@@ -0,0 +1,347 @@
|
||||
#![allow(clippy::large_enum_variant)] // TODO: we allow large enum variant for now, let's profile properly to see if we want to box.
|
||||
|
||||
use crate::parser_go_token::{Position, Token};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
// https://pkg.go.dev/go/ast#CommentGroup
|
||||
#[derive(Debug)]
|
||||
pub struct CommentGroup {
|
||||
// List []*Comment // len(List) > 0
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#FieldList
|
||||
#[derive(Debug)]
|
||||
pub struct FieldList<'a> {
|
||||
pub opening: Option<Position<'a>>, // position of opening parenthesis/brace, if any
|
||||
pub list: Vec<Field<'a>>, // field list; or nil
|
||||
pub closing: Option<Position<'a>>, // position of closing parenthesis/brace, if any
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Field
|
||||
#[derive(Debug)]
|
||||
pub struct Field<'a> {
|
||||
pub doc: Option<CommentGroup>, // associated documentation; or nil
|
||||
pub names: Option<Vec<Ident<'a>>>, // field/method/(type) parameter names, or type "type"; or nil
|
||||
pub type_: Option<Expr<'a>>, // field/method/parameter type, type list type; or nil
|
||||
pub tag: Option<BasicLit<'a>>, // field tag; or nil
|
||||
pub comment: Option<CommentGroup>, // line comments; or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#File
|
||||
#[derive(Debug)]
|
||||
pub struct File<'a> {
|
||||
// package name
|
||||
pub decls: Vec<Decl<'a>>, // top-level declarations; or nil // list of all comments in the source file
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#FuncDecl
|
||||
#[derive(Debug)]
|
||||
pub struct FuncDecl<'a> {
|
||||
pub doc: Option<CommentGroup>, // associated documentation; or nil
|
||||
pub recv: Option<FieldList<'a>>, // receiver (methods); or nil (functions)
|
||||
pub name: Ident<'a>, // function/method name
|
||||
pub type_: FuncType<'a>, // function signature: type and value parameters, results, and position of "func" keyword
|
||||
pub body: Option<BlockStmt<'a>>, // function body; or nil for external (non-Go) function
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#BlockStmt
|
||||
#[derive(Debug)]
|
||||
pub struct BlockStmt<'a> {
|
||||
pub lbrace: Position<'a>, // position of "{"
|
||||
pub list: Vec<Stmt>,
|
||||
pub rbrace: Position<'a>, // position of "}", if any (may be absent due to syntax error)
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#FuncType
|
||||
#[derive(Debug)]
|
||||
pub struct FuncType<'a> {
|
||||
pub func: Option<Position<'a>>, // position of "func" keyword (token.NoPos if there is no "func")
|
||||
pub params: FieldList<'a>, // (incoming) parameters; non-nil
|
||||
pub results: Option<FieldList<'a>>, // (outgoing) results; or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Ident
|
||||
#[derive(Debug)]
|
||||
pub struct Ident<'a> {
|
||||
pub name_pos: Position<'a>, // identifier position
|
||||
pub name: &'a str, // identifier name
|
||||
pub obj: Option<Box<Object<'a>>>, // denoted object; or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ValueSpec
|
||||
#[derive(Debug)]
|
||||
pub struct ValueSpec<'a> {
|
||||
pub doc: Option<CommentGroup>, // associated documentation; or nil
|
||||
pub names: Vec<Ident<'a>>, // value names (len(Names) > 0)
|
||||
pub type_: Option<Expr<'a>>, // value type; or nil
|
||||
pub values: Option<Vec<Expr<'a>>>, // initial values; or nil
|
||||
pub comment: Option<CommentGroup>, // line comments; or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#BasicLit
|
||||
#[derive(Debug)]
|
||||
pub struct BasicLit<'a> {
|
||||
pub value_pos: Position<'a>, // literal position
|
||||
pub kind: Token, // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
|
||||
pub value: &'a str, // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Object
|
||||
#[derive(Debug)]
|
||||
pub struct Object<'a> {
|
||||
pub kind: ObjKind,
|
||||
pub name: &'a str, // declared name
|
||||
pub decl: Option<ObjDecl>, // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil
|
||||
pub data: Option<usize>, // object-specific data; or nil
|
||||
pub type_: Option<()>, // placeholder for type information; may be nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Ellipsis
|
||||
#[derive(Debug)]
|
||||
pub struct Ellipsis<'a> {
|
||||
pub ellipsis: Position<'a>, // position of "..."
|
||||
pub elt: Option<Box<Expr<'a>>>, // ellipsis element type (parameter lists only); or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Ellipsis
|
||||
#[derive(Debug)]
|
||||
pub struct TypeAssertExpr<'a> {
|
||||
pub x: Box<Expr<'a>>, // expression
|
||||
pub lparen: Position<'a>, // position of "("
|
||||
pub type_: Box<Expr<'a>>, // asserted type; nil means type switch X.(type)
|
||||
pub rparen: Position<'a>, // position of ")"
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#SliceExpr
|
||||
#[derive(Debug)]
|
||||
pub struct SliceExpr<'a> {
|
||||
pub x: Box<Expr<'a>>, // expression
|
||||
pub lbrack: Position<'a>, // position of "["
|
||||
pub low: Option<Box<Expr<'a>>>, // begin of slice range; or nil
|
||||
pub high: Option<Box<Expr<'a>>>, // end of slice range; or nil
|
||||
pub max: Option<Box<Expr<'a>>>, // maximum capacity of slice; or nil
|
||||
pub slice3: bool, // true if 3-index slice (2 colons present)
|
||||
pub rbrack: Position<'a>, // position of "]"
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ObjKind
|
||||
#[derive(Debug)]
|
||||
pub enum ObjKind {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ObjDecl {}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Decl
|
||||
#[derive(Debug)]
|
||||
pub enum Decl<'a> {
|
||||
FuncDecl(FuncDecl<'a>),
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Scope
|
||||
#[derive(Debug)]
|
||||
pub struct Scope<'a> {
|
||||
pub outer: Option<Box<Scope<'a>>>,
|
||||
pub objects: BTreeMap<&'a str, Object<'a>>,
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#GenDecl
|
||||
#[derive(Debug)]
|
||||
pub struct GenDecl<'a> {
|
||||
pub doc: Option<CommentGroup>, // associated documentation; or nil
|
||||
pub tok_pos: Position<'a>, // position of Tok
|
||||
pub tok: Token, // IMPORT, CONST, TYPE, or VAR
|
||||
pub lparen: Option<Position<'a>>, // position of '(', if any
|
||||
pub specs: Vec<Spec>,
|
||||
pub rparen: Option<Position<'a>>, // position of ')', if any
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#AssignStmt
|
||||
#[derive(Debug)]
|
||||
pub struct AssignStmt<'a> {
|
||||
pub lhs: Vec<Expr<'a>>,
|
||||
pub tok_pos: Position<'a>, // position of Tok
|
||||
pub tok: Token, // assignment token, DEFINE
|
||||
pub rhs: Vec<Expr<'a>>,
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#BinaryExpr
|
||||
#[derive(Debug)]
|
||||
pub struct BinaryExpr<'a> {
|
||||
pub x: Box<Expr<'a>>, // left operand
|
||||
pub op_pos: Position<'a>, // position of Op
|
||||
pub op: Token, // operator
|
||||
pub y: Box<Expr<'a>>, // right operand
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ReturnStmt
|
||||
#[derive(Debug)]
|
||||
pub struct ReturnStmt<'a> {
|
||||
pub return_: Position<'a>, // position of "return" keyword
|
||||
pub results: Vec<Expr<'a>>, // result expressions; or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#TypeSpec
|
||||
#[derive(Debug)]
|
||||
pub struct TypeSpec<'a> {
|
||||
pub doc: Option<CommentGroup>, // associated documentation; or nil
|
||||
pub name: Option<Ident<'a>>, // type name
|
||||
pub assign: Option<Position<'a>>, // position of '=', if any
|
||||
pub type_: Expr<'a>, // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes
|
||||
pub comment: Option<CommentGroup>, // line comments; or nil
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#StructType
|
||||
#[derive(Debug)]
|
||||
pub struct StructType<'a> {
|
||||
pub struct_: Position<'a>, // position of "struct" keyword
|
||||
pub fields: Option<FieldList<'a>>, // list of field declarations
|
||||
pub incomplete: bool, // true if (source) fields are missing in the Fields list
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#StarExpr
|
||||
#[derive(Debug)]
|
||||
pub struct StarExpr<'a> {
|
||||
pub star: Position<'a>, // position of "*"
|
||||
pub x: Box<Expr<'a>>, // operand
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#InterfaceType
|
||||
#[derive(Debug)]
|
||||
pub struct InterfaceType<'a> {
|
||||
pub interface: Position<'a>, // position of "interface" keyword
|
||||
pub methods: Option<FieldList<'a>>, // list of embedded interfaces, methods, or types
|
||||
pub incomplete: bool, // true if (source) methods or types are missing in the Methods list
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#UnaryExpr
|
||||
#[derive(Debug)]
|
||||
pub struct UnaryExpr<'a> {
|
||||
pub op_pos: Position<'a>, // position of Op
|
||||
pub op: Token, // operator
|
||||
pub x: Box<Expr<'a>>, // operand
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#CallExpr
|
||||
#[derive(Debug)]
|
||||
pub struct CallExpr<'a> {
|
||||
pub fun: Box<Expr<'a>>, // function expression
|
||||
pub lparen: Position<'a>, // position of "("
|
||||
pub args: Option<Vec<Expr<'a>>>, // function arguments; or nil
|
||||
pub ellipsis: Option<Position<'a>>, // position of "..." (token.NoPos if there is no "...")
|
||||
pub rparen: Position<'a>, // position of ")"
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#SelectorExpr
|
||||
#[derive(Debug)]
|
||||
pub struct SelectorExpr<'a> {
|
||||
pub x: Box<Expr<'a>>, // expression
|
||||
pub sel: Ident<'a>, // field selector
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ParenExpr
|
||||
#[derive(Debug)]
|
||||
pub struct ParenExpr<'a> {
|
||||
pub lparen: Position<'a>, // position of "("
|
||||
pub x: Box<Expr<'a>>, // parenthesized expression
|
||||
pub rparen: Position<'a>, // position of ")"
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#FuncLit
|
||||
#[derive(Debug)]
|
||||
pub struct FuncLit<'a> {
|
||||
pub type_: FuncType<'a>, // function type
|
||||
pub body: BlockStmt<'a>, // function body
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ChanType
|
||||
#[derive(Debug)]
|
||||
pub struct ChanType<'a> {
|
||||
pub begin: Position<'a>, // position of "chan" keyword or "<-" (whichever comes first)
|
||||
pub arrow: Option<Position<'a>>, // position of "<-" (token.NoPos if there is no "<-")
|
||||
pub dir: u8, // channel direction
|
||||
pub value: Box<Expr<'a>>, // value type
|
||||
}
|
||||
|
||||
// htt/opt/visual-studio-code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.htmlps://pkg.go.dev/go/ast#IndexExpr
|
||||
#[derive(Debug)]
|
||||
pub struct IndexExpr<'a> {
|
||||
pub x: Box<Expr<'a>>, // expression
|
||||
pub lbrack: Position<'a>, // position of "["
|
||||
pub index: Box<Expr<'a>>, // index expression
|
||||
pub rbrack: Position<'a>, // position of "]"
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#MapType
|
||||
#[derive(Debug)]
|
||||
pub struct MapType<'a> {
|
||||
pub map: Position<'a>,
|
||||
pub key: Box<Expr<'a>>,
|
||||
pub value: Box<Expr<'a>>,
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#CompositeLit
|
||||
#[derive(Debug)]
|
||||
pub struct CompositeLit<'a> {
|
||||
pub type_: Box<Expr<'a>>, // literal type; or nil
|
||||
pub lbrace: Position<'a>, // position of "{"
|
||||
pub elts: Option<Vec<Expr<'a>>>, // list of composite elements; or nil
|
||||
pub rbrace: Position<'a>, // position of "}"
|
||||
pub incomplete: bool, // true if (source) expressions are missing in the Elts list
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#KeyValueExpr
|
||||
#[derive(Debug)]
|
||||
pub struct KeyValueExpr<'a> {
|
||||
pub key: Box<Expr<'a>>,
|
||||
pub colon: Position<'a>, // position of ":"
|
||||
pub value: Box<Expr<'a>>,
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ArrayType
|
||||
#[derive(Debug)]
|
||||
pub struct ArrayType<'a> {
|
||||
pub lbrack: Position<'a>, // position of "["
|
||||
pub len: Option<Box<Expr<'a>>>, // Ellipsis node for [...]T array types, nil for slice types
|
||||
pub elt: Box<Expr<'a>>, // element type
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#ChanDir
|
||||
#[derive(Debug)]
|
||||
pub enum ChanDir {
|
||||
SEND = 1 << 0,
|
||||
RECV = 1 << 1,
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Spec
|
||||
#[derive(Debug)]
|
||||
pub enum Spec {}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Expr
|
||||
#[derive(Debug)]
|
||||
pub enum Expr<'a> {
|
||||
ArrayType(ArrayType<'a>),
|
||||
BasicLit(BasicLit<'a>),
|
||||
BinaryExpr(BinaryExpr<'a>),
|
||||
CallExpr(CallExpr<'a>),
|
||||
ChanType(ChanType<'a>),
|
||||
CompositeLit(CompositeLit<'a>),
|
||||
Ellipsis(Ellipsis<'a>),
|
||||
FuncLit(FuncLit<'a>),
|
||||
FuncType(FuncType<'a>),
|
||||
Ident(Ident<'a>),
|
||||
IndexExpr(IndexExpr<'a>),
|
||||
InterfaceType(InterfaceType<'a>),
|
||||
KeyValueExpr(KeyValueExpr<'a>),
|
||||
MapType(MapType<'a>),
|
||||
ParenExpr(ParenExpr<'a>),
|
||||
SelectorExpr(SelectorExpr<'a>),
|
||||
SliceExpr(SliceExpr<'a>),
|
||||
StarExpr(StarExpr<'a>),
|
||||
StructType(StructType<'a>),
|
||||
TypeAssertExpr(TypeAssertExpr<'a>),
|
||||
UnaryExpr(UnaryExpr<'a>),
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/go/ast#Stmt
|
||||
#[derive(Debug)]
|
||||
pub enum Stmt {}
|
||||
948
backend/parsers/windmill-parser-go/src/parser_go_scanner.rs
Normal file
948
backend/parsers/windmill-parser-go/src/parser_go_scanner.rs
Normal file
@@ -0,0 +1,948 @@
|
||||
// https://golang.org/ref/spec#Lexical_elements
|
||||
|
||||
use crate::parser_go_token::{Position, Token};
|
||||
use phf::{phf_map, Map};
|
||||
use std::fmt;
|
||||
use unicode_general_category::{get_general_category, GeneralCategory};
|
||||
|
||||
pub type Step<'a> = (Position<'a>, Token, &'a str);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ScannerError {
|
||||
HexadecimalNotFound,
|
||||
OctalNotFound,
|
||||
UnterminatedComment,
|
||||
UnterminatedEscapedChar,
|
||||
UnterminatedRune,
|
||||
UnterminatedString,
|
||||
InvalidDirective,
|
||||
}
|
||||
|
||||
impl std::error::Error for ScannerError {}
|
||||
|
||||
impl fmt::Display for ScannerError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "scanner error: {:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, ScannerError>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Scanner<'a> {
|
||||
directory: &'a str,
|
||||
file: &'a str,
|
||||
buffer: &'a str,
|
||||
//
|
||||
chars: std::iter::Peekable<std::str::Chars<'a>>,
|
||||
current_char: Option<char>,
|
||||
current_char_len: usize,
|
||||
//
|
||||
offset: usize,
|
||||
line: usize,
|
||||
column: usize,
|
||||
start_offset: usize,
|
||||
start_line: usize,
|
||||
start_column: usize,
|
||||
//
|
||||
hide_column: bool,
|
||||
insert_semi: bool,
|
||||
pending_line_info: Option<LineInfo<'a>>,
|
||||
}
|
||||
|
||||
type LineInfo<'a> = (Option<&'a str>, usize, Option<usize>, bool);
|
||||
|
||||
impl<'a> Scanner<'a> {
|
||||
pub fn new(filename: &'a str, buffer: &'a str) -> Self {
|
||||
let (directory, file) = filename.rsplit_once('/').unwrap_or(("", filename));
|
||||
let mut s = Scanner {
|
||||
directory,
|
||||
file,
|
||||
buffer,
|
||||
//
|
||||
chars: buffer.chars().peekable(),
|
||||
current_char: None,
|
||||
current_char_len: 0,
|
||||
//
|
||||
offset: 0,
|
||||
line: 1,
|
||||
column: 1,
|
||||
start_offset: 0,
|
||||
start_line: 1,
|
||||
start_column: 1,
|
||||
//
|
||||
hide_column: false,
|
||||
insert_semi: false,
|
||||
pending_line_info: None,
|
||||
};
|
||||
s.next(); // read the first character
|
||||
s
|
||||
}
|
||||
|
||||
#[allow(clippy::cognitive_complexity)] // Allow complex scan function
|
||||
pub fn scan(&mut self) -> Result<Step<'a>> {
|
||||
let insert_semi = self.insert_semi;
|
||||
self.insert_semi = false;
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
self.reset_start();
|
||||
|
||||
match c {
|
||||
' ' | '\t' | '\r' => {
|
||||
self.next();
|
||||
}
|
||||
|
||||
'\n' => {
|
||||
self.next();
|
||||
if insert_semi {
|
||||
return Ok((self.position(), Token::SEMICOLON, "\n"));
|
||||
}
|
||||
}
|
||||
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(c) = self.current_char {
|
||||
match c {
|
||||
'+' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::ADD_ASSIGN, ""));
|
||||
}
|
||||
Some('+') => {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
return Ok((self.position(), Token::INC, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::ADD, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'-' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::SUB_ASSIGN, ""));
|
||||
}
|
||||
Some('-') => {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
return Ok((self.position(), Token::DEC, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::SUB, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'*' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::MUL_ASSIGN, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::MUL, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'/' => match self.peek() {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
self.next();
|
||||
return Ok((self.position(), Token::QUO_ASSIGN, ""));
|
||||
}
|
||||
Some('/') => {
|
||||
if insert_semi {
|
||||
return Ok((self.position(), Token::SEMICOLON, "\n"));
|
||||
}
|
||||
return self.scan_line_comment();
|
||||
}
|
||||
Some('*') => {
|
||||
if insert_semi && self.find_line_end() {
|
||||
return Ok((self.position(), Token::SEMICOLON, "\n"));
|
||||
}
|
||||
return self.scan_general_comment();
|
||||
}
|
||||
_ => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::QUO, ""));
|
||||
}
|
||||
},
|
||||
|
||||
'%' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::REM_ASSIGN, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::REM, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'&' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::AND_ASSIGN, ""));
|
||||
}
|
||||
Some('&') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::LAND, ""));
|
||||
}
|
||||
Some('^') => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::AND_NOT_ASSIGN, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::AND_NOT, "")),
|
||||
}
|
||||
}
|
||||
_ => return Ok((self.position(), Token::AND, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'|' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::OR_ASSIGN, ""));
|
||||
}
|
||||
Some('|') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::LOR, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::OR, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'^' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::XOR_ASSIGN, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::XOR, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'<' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('<') => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::SHL_ASSIGN, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::SHL, "")),
|
||||
}
|
||||
}
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::LEQ, ""));
|
||||
}
|
||||
Some('-') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::ARROW, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::LSS, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'>' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('>') => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::SHR_ASSIGN, ""));
|
||||
}
|
||||
_ => {
|
||||
return Ok((self.position(), Token::SHR, ""));
|
||||
}
|
||||
}
|
||||
}
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::GEQ, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::GTR, "")),
|
||||
}
|
||||
}
|
||||
|
||||
':' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::DEFINE, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::COLON, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'!' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::NEQ, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::NOT, "")),
|
||||
}
|
||||
}
|
||||
|
||||
',' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::COMMA, ""));
|
||||
}
|
||||
|
||||
'(' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::LPAREN, ""));
|
||||
}
|
||||
|
||||
')' => {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
return Ok((self.position(), Token::RPAREN, ""));
|
||||
}
|
||||
|
||||
'[' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::LBRACK, ""));
|
||||
}
|
||||
|
||||
']' => {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
return Ok((self.position(), Token::RBRACK, ""));
|
||||
}
|
||||
|
||||
'{' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::LBRACE, ""));
|
||||
}
|
||||
|
||||
'}' => {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
return Ok((self.position(), Token::RBRACE, ""));
|
||||
}
|
||||
|
||||
';' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::SEMICOLON, ";"));
|
||||
}
|
||||
|
||||
'.' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('0'..='9') => return self.scan_int_or_float_or_imag(true),
|
||||
Some('.') => match self.peek() {
|
||||
Some('.') => {
|
||||
self.next();
|
||||
self.next();
|
||||
return Ok((self.position(), Token::ELLIPSIS, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::PERIOD, "")),
|
||||
},
|
||||
_ => return Ok((self.position(), Token::PERIOD, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'=' => {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('=') => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::EQL, ""));
|
||||
}
|
||||
_ => return Ok((self.position(), Token::ASSIGN, "")),
|
||||
}
|
||||
}
|
||||
|
||||
'0'..='9' => return self.scan_int_or_float_or_imag(false),
|
||||
'\'' => return self.scan_rune(),
|
||||
'"' => return self.scan_interpreted_string(),
|
||||
'`' => return self.scan_raw_string(),
|
||||
_ => return self.scan_pkg_or_keyword_or_ident(),
|
||||
};
|
||||
}
|
||||
|
||||
self.reset_start();
|
||||
if insert_semi {
|
||||
Ok((self.position(), Token::SEMICOLON, "\n"))
|
||||
} else {
|
||||
Ok((self.position(), Token::EOF, ""))
|
||||
}
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Keywords
|
||||
// https://golang.org/ref/spec#Identifiers
|
||||
fn scan_pkg_or_keyword_or_ident(&mut self) -> Result<Step<'a>> {
|
||||
self.next();
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
if !(is_letter(c) || is_unicode_digit(c)) {
|
||||
break;
|
||||
}
|
||||
self.next()
|
||||
}
|
||||
|
||||
let pos = self.position();
|
||||
let literal = self.literal();
|
||||
|
||||
if literal.len() > 1 {
|
||||
if let Some(&token) = KEYWORDS.get(literal) {
|
||||
self.insert_semi = matches!(
|
||||
token,
|
||||
Token::BREAK | Token::CONTINUE | Token::FALLTHROUGH | Token::RETURN
|
||||
);
|
||||
return Ok((pos, token, literal));
|
||||
}
|
||||
}
|
||||
|
||||
self.insert_semi = true;
|
||||
Ok((pos, Token::IDENT, literal))
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Integer_literals
|
||||
// https://golang.org/ref/spec#Floating-point_literals
|
||||
// https://golang.org/ref/spec#Imaginary_literals
|
||||
fn scan_int_or_float_or_imag(&mut self, preceding_dot: bool) -> Result<Step<'a>> {
|
||||
self.insert_semi = true;
|
||||
|
||||
let mut token = Token::INT;
|
||||
let mut digits = "_0123456789";
|
||||
let mut exp = "eE";
|
||||
|
||||
if !preceding_dot {
|
||||
if matches!(self.current_char, Some('0')) {
|
||||
self.next();
|
||||
match self.current_char {
|
||||
Some('b' | 'B') => {
|
||||
digits = "_01";
|
||||
exp = "";
|
||||
self.next();
|
||||
}
|
||||
Some('o' | 'O') => {
|
||||
digits = "_01234567";
|
||||
exp = "";
|
||||
self.next();
|
||||
}
|
||||
Some('x' | 'X') => {
|
||||
digits = "_0123456789abcdefABCDEF";
|
||||
exp = "pP";
|
||||
self.next();
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
}
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
if !digits.contains(c) {
|
||||
break;
|
||||
}
|
||||
self.next();
|
||||
}
|
||||
}
|
||||
|
||||
if preceding_dot || matches!(self.current_char, Some('.')) {
|
||||
token = Token::FLOAT;
|
||||
self.next();
|
||||
while let Some(c) = self.current_char {
|
||||
if !digits.contains(c) {
|
||||
break;
|
||||
}
|
||||
self.next();
|
||||
}
|
||||
}
|
||||
|
||||
if !exp.is_empty() {
|
||||
if let Some(c) = self.current_char {
|
||||
if exp.contains(c) {
|
||||
token = Token::FLOAT;
|
||||
self.next();
|
||||
if matches!(self.current_char, Some('-' | '+')) {
|
||||
self.next();
|
||||
}
|
||||
while let Some(c) = self.current_char {
|
||||
if !matches!(c, '_' | '0'..='9') {
|
||||
break;
|
||||
}
|
||||
self.next();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if matches!(self.current_char, Some('i')) {
|
||||
token = Token::IMAG;
|
||||
self.next();
|
||||
}
|
||||
|
||||
Ok((self.position(), token, self.literal()))
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Rune_literals
|
||||
fn scan_rune(&mut self) -> Result<Step<'a>> {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
|
||||
match self.current_char {
|
||||
Some('\\') => self.require_escaped_char::<'\''>()?,
|
||||
Some(_) => self.next(),
|
||||
_ => return Err(ScannerError::UnterminatedRune),
|
||||
}
|
||||
|
||||
if matches!(self.current_char, Some('\'')) {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::CHAR, self.literal()));
|
||||
}
|
||||
|
||||
Err(ScannerError::UnterminatedRune)
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#String_literals
|
||||
fn scan_interpreted_string(&mut self) -> Result<Step<'a>> {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
match c {
|
||||
'"' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::STRING, self.literal()));
|
||||
}
|
||||
'\\' => self.require_escaped_char::<'"'>()?,
|
||||
_ => self.next(),
|
||||
}
|
||||
}
|
||||
|
||||
Err(ScannerError::UnterminatedString)
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#String_literals
|
||||
fn scan_raw_string(&mut self) -> Result<Step<'a>> {
|
||||
self.insert_semi = true;
|
||||
self.next();
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
match c {
|
||||
'`' => {
|
||||
self.next();
|
||||
return Ok((self.position(), Token::STRING, self.literal()));
|
||||
}
|
||||
_ => self.next(),
|
||||
}
|
||||
}
|
||||
|
||||
Err(ScannerError::UnterminatedString)
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Comments
|
||||
fn scan_general_comment(&mut self) -> Result<Step<'a>> {
|
||||
self.next();
|
||||
self.next();
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
match c {
|
||||
'*' => {
|
||||
self.next();
|
||||
if matches!(self.current_char, Some('/')) {
|
||||
self.next();
|
||||
|
||||
let pos = self.position();
|
||||
let lit = self.literal();
|
||||
|
||||
// look for compiler directives
|
||||
self.directive(&lit["/*".len()..lit.len() - "*/".len()], true)?;
|
||||
|
||||
return Ok((pos, Token::COMMENT, lit));
|
||||
}
|
||||
}
|
||||
_ => self.next(),
|
||||
}
|
||||
}
|
||||
|
||||
Err(ScannerError::UnterminatedComment)
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Comments
|
||||
fn scan_line_comment(&mut self) -> Result<Step<'a>> {
|
||||
self.next();
|
||||
self.next();
|
||||
|
||||
while let Some(c) = self.current_char {
|
||||
if is_newline(c) {
|
||||
break;
|
||||
}
|
||||
self.next();
|
||||
}
|
||||
|
||||
let pos = self.position();
|
||||
let lit = self.literal();
|
||||
|
||||
// look for compiler directives (at the beginning of line)
|
||||
if self.start_column == 1 {
|
||||
self.directive(lit["//".len()..].trim_end(), false)?;
|
||||
}
|
||||
|
||||
Ok((pos, Token::COMMENT, self.literal()))
|
||||
}
|
||||
|
||||
// https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives
|
||||
fn directive(&mut self, input: &'a str, immediate: bool) -> Result<()> {
|
||||
if let Some(line_directive) = input.strip_prefix("line ") {
|
||||
self.pending_line_info = self.parse_line_directive(line_directive)?;
|
||||
if immediate {
|
||||
self.consume_pending_line_info();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_line_directive(&mut self, line_directive: &'a str) -> Result<Option<LineInfo<'a>>> {
|
||||
if let Some((file, line)) = line_directive.rsplit_once(':') {
|
||||
let line = line.parse().map_err(|_| ScannerError::InvalidDirective)?;
|
||||
|
||||
if let Some((file, l)) = file.rsplit_once(':') {
|
||||
if let Ok(l) = l.parse() {
|
||||
//line :line:col
|
||||
//line filename:line:col
|
||||
/*line :line:col*/
|
||||
/*line filename:line:col*/
|
||||
let file = if !file.is_empty() { Some(file) } else { None };
|
||||
let col = Some(line);
|
||||
let line = l;
|
||||
let hide_column = false;
|
||||
return Ok(Some((file, line, col, hide_column)));
|
||||
}
|
||||
}
|
||||
|
||||
//line :line
|
||||
//line filename:line
|
||||
/*line :line*/
|
||||
/*line filename:line*/
|
||||
Ok(Some((Some(file), line, None, true)))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
const fn find_line_end(&self) -> bool {
|
||||
let buffer = self.buffer.as_bytes();
|
||||
let mut in_comment = true;
|
||||
|
||||
let mut i = self.offset;
|
||||
let max = self.buffer.len();
|
||||
while i < max {
|
||||
let c = buffer[i] as char;
|
||||
|
||||
if i < max - 1 {
|
||||
let n = buffer[i + 1] as char;
|
||||
|
||||
if !in_comment && c == '/' && n == '/' {
|
||||
return true;
|
||||
}
|
||||
|
||||
if c == '/' && n == '*' {
|
||||
i += 2;
|
||||
in_comment = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if c == '*' && n == '/' {
|
||||
i += 2;
|
||||
in_comment = false;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if is_newline(c) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if !in_comment && !matches!(c, ' ' | '\t' | '\r') {
|
||||
return false;
|
||||
}
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
!in_comment
|
||||
}
|
||||
|
||||
fn consume_pending_line_info(&mut self) {
|
||||
if let Some(line_info) = self.pending_line_info.take() {
|
||||
if let Some(file) = line_info.0 {
|
||||
self.file = file;
|
||||
}
|
||||
|
||||
self.line = line_info.1;
|
||||
|
||||
if let Some(column) = line_info.2 {
|
||||
self.column = column;
|
||||
}
|
||||
|
||||
self.hide_column = line_info.3;
|
||||
}
|
||||
}
|
||||
|
||||
fn peek(&mut self) -> Option<char> {
|
||||
self.chars.peek().copied()
|
||||
}
|
||||
|
||||
fn next(&mut self) {
|
||||
self.offset += self.current_char_len;
|
||||
self.column += self.current_char_len;
|
||||
let last_char = self.current_char;
|
||||
|
||||
self.current_char = self.chars.next();
|
||||
if let Some(c) = self.current_char {
|
||||
self.current_char_len = c.len_utf8();
|
||||
if matches!(last_char, Some('\n')) {
|
||||
self.line += 1;
|
||||
self.column = 1;
|
||||
self.consume_pending_line_info();
|
||||
}
|
||||
} else {
|
||||
self.current_char_len = 0
|
||||
}
|
||||
}
|
||||
|
||||
const fn position(&self) -> Position<'a> {
|
||||
Position {
|
||||
directory: self.directory,
|
||||
file: self.file,
|
||||
offset: self.start_offset,
|
||||
line: self.start_line,
|
||||
column: if self.hide_column {
|
||||
0
|
||||
} else {
|
||||
self.start_column
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn reset_start(&mut self) {
|
||||
self.start_offset = self.offset;
|
||||
self.start_line = self.line;
|
||||
self.start_column = self.column;
|
||||
}
|
||||
|
||||
fn literal(&self) -> &'a str {
|
||||
&self.buffer[self.start_offset..self.offset]
|
||||
}
|
||||
|
||||
fn require_escaped_char<const DELIM: char>(&mut self) -> Result<()> {
|
||||
self.next();
|
||||
|
||||
let c = self
|
||||
.current_char
|
||||
.ok_or(ScannerError::UnterminatedEscapedChar)?;
|
||||
|
||||
// TODO: move this to the match when const generics can be referenced in patterns
|
||||
if c == DELIM {
|
||||
self.next();
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
match c {
|
||||
'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | '\\' => self.next(),
|
||||
'x' => {
|
||||
self.next();
|
||||
self.require_hex_digits::<2>()?
|
||||
}
|
||||
'u' => {
|
||||
self.next();
|
||||
self.require_hex_digits::<4>()?;
|
||||
}
|
||||
'U' => {
|
||||
self.next();
|
||||
self.require_hex_digits::<8>()?;
|
||||
}
|
||||
'0'..='7' => self.require_octal_digits::<3>()?,
|
||||
_ => return Err(ScannerError::UnterminatedEscapedChar),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_octal_digits<const COUNT: usize>(&mut self) -> Result<()> {
|
||||
for _ in 0..COUNT {
|
||||
let c = self.current_char.ok_or(ScannerError::OctalNotFound)?;
|
||||
|
||||
if !is_octal_digit(c) {
|
||||
return Err(ScannerError::OctalNotFound);
|
||||
}
|
||||
|
||||
self.next();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_hex_digits<const COUNT: usize>(&mut self) -> Result<()> {
|
||||
for _ in 0..COUNT {
|
||||
let c = self.current_char.ok_or(ScannerError::HexadecimalNotFound)?;
|
||||
|
||||
if !is_hex_digit(c) {
|
||||
return Err(ScannerError::HexadecimalNotFound);
|
||||
}
|
||||
|
||||
self.next();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> IntoIterator for Scanner<'a> {
|
||||
type Item = Result<Step<'a>>;
|
||||
type IntoIter = IntoIter<'a>;
|
||||
|
||||
fn into_iter(self) -> Self::IntoIter {
|
||||
Self::IntoIter::new(self)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct IntoIter<'a> {
|
||||
scanner: Scanner<'a>,
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl<'a> IntoIter<'a> {
|
||||
const fn new(scanner: Scanner<'a>) -> Self {
|
||||
Self { scanner, done: false }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for IntoIter<'a> {
|
||||
type Item = Result<Step<'a>>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.done {
|
||||
return None;
|
||||
}
|
||||
|
||||
match self.scanner.scan() {
|
||||
Ok((pos, tok, lit)) => {
|
||||
if tok == Token::EOF {
|
||||
self.done = true;
|
||||
}
|
||||
Some(Ok((pos, tok, lit)))
|
||||
}
|
||||
Err(err) => {
|
||||
self.done = true;
|
||||
Some(Err(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Letters_and_digits
|
||||
|
||||
fn is_letter(c: char) -> bool {
|
||||
c == '_' || is_unicode_letter(c)
|
||||
}
|
||||
|
||||
//const fn is_decimal_digit(c: char) -> bool {
|
||||
//matches!(c, '0'..='9')
|
||||
//}
|
||||
|
||||
//const fn is_binary_digit(c: char) -> bool {
|
||||
//matches!(c, '0'..='1')
|
||||
//}
|
||||
|
||||
const fn is_octal_digit(c: char) -> bool {
|
||||
matches!(c, '0'..='7')
|
||||
}
|
||||
|
||||
const fn is_hex_digit(c: char) -> bool {
|
||||
matches!(c, '0'..='9' | 'A'..='F' | 'a'..='f')
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Characters
|
||||
|
||||
const fn is_newline(c: char) -> bool {
|
||||
c == '\n'
|
||||
}
|
||||
|
||||
//const fn is_unicode_char(c: char) -> bool {
|
||||
//c != '\n'
|
||||
//}
|
||||
|
||||
fn is_unicode_letter(c: char) -> bool {
|
||||
matches!(
|
||||
get_general_category(c),
|
||||
GeneralCategory::UppercaseLetter
|
||||
| GeneralCategory::LowercaseLetter
|
||||
| GeneralCategory::TitlecaseLetter
|
||||
| GeneralCategory::ModifierLetter
|
||||
| GeneralCategory::OtherLetter
|
||||
)
|
||||
}
|
||||
|
||||
fn is_unicode_digit(c: char) -> bool {
|
||||
get_general_category(c) == GeneralCategory::DecimalNumber
|
||||
}
|
||||
|
||||
// https://golang.org/ref/spec#Keywords
|
||||
|
||||
static KEYWORDS: Map<&'static str, Token> = phf_map! {
|
||||
"break" => Token::BREAK,
|
||||
"case" => Token::CASE,
|
||||
"chan" => Token::CHAN,
|
||||
"const" => Token::CONST,
|
||||
"continue" => Token::CONTINUE,
|
||||
|
||||
"default" => Token::DEFAULT,
|
||||
"defer" => Token::DEFER,
|
||||
"else" => Token::ELSE,
|
||||
"fallthrough" => Token::FALLTHROUGH,
|
||||
"for" => Token::FOR,
|
||||
|
||||
"func" => Token::FUNC,
|
||||
"go" => Token::GO,
|
||||
"goto" => Token::GOTO,
|
||||
"if" => Token::IF,
|
||||
"import" => Token::IMPORT,
|
||||
|
||||
"interface" => Token::INTERFACE,
|
||||
"map" => Token::MAP,
|
||||
"package" => Token::PACKAGE,
|
||||
"range" => Token::RANGE,
|
||||
"return" => Token::RETURN,
|
||||
|
||||
"select" => Token::SELECT,
|
||||
"struct" => Token::STRUCT,
|
||||
"switch" => Token::SWITCH,
|
||||
"type" => Token::TYPE,
|
||||
"var" => Token::VAR,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::Scanner;
|
||||
|
||||
#[test] // fuzz
|
||||
fn it_should_return_an_error_on_missing_line_number() {
|
||||
let input = "/*line :*/";
|
||||
let mut out: Vec<_> = Scanner::new(file!(), input).into_iter().collect();
|
||||
assert!(out.pop().unwrap().is_err());
|
||||
}
|
||||
}
|
||||
273
backend/parsers/windmill-parser-go/src/parser_go_token.rs
Normal file
273
backend/parsers/windmill-parser-go/src/parser_go_token.rs
Normal file
@@ -0,0 +1,273 @@
|
||||
// https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/go/token/token.go
|
||||
|
||||
#![allow(non_camel_case_types)] // For consistency with the Go tokens
|
||||
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct Position<'a> {
|
||||
pub directory: &'a str,
|
||||
pub file: &'a str,
|
||||
pub offset: usize,
|
||||
pub line: usize,
|
||||
pub column: usize,
|
||||
}
|
||||
|
||||
impl<'a> fmt::Display for Position<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
if self.file.is_empty() {
|
||||
write!(f, ":{}:{}", self.line, self.column)
|
||||
} else if self.file.starts_with('/') {
|
||||
write!(f, "{}:{}:{}", self.file, self.line, self.column)
|
||||
} else {
|
||||
write!(
|
||||
f,
|
||||
"{}/{}:{}:{}",
|
||||
self.directory, self.file, self.line, self.column
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
pub enum Token {
|
||||
EOF,
|
||||
COMMENT,
|
||||
|
||||
IDENT, // main
|
||||
INT, // 12345
|
||||
FLOAT, // 123.45
|
||||
IMAG, // 123.45i
|
||||
CHAR, // 'a'
|
||||
STRING, // "abc"
|
||||
|
||||
ADD, // +
|
||||
SUB, // -
|
||||
MUL, // *
|
||||
QUO, // /
|
||||
REM, // %
|
||||
|
||||
AND, // &
|
||||
OR, // |
|
||||
XOR, // ^
|
||||
SHL, // <<
|
||||
SHR, // >>
|
||||
AND_NOT, // &^
|
||||
|
||||
ADD_ASSIGN, // +=
|
||||
SUB_ASSIGN, // -=
|
||||
MUL_ASSIGN, // *=
|
||||
QUO_ASSIGN, // /=
|
||||
REM_ASSIGN, // %=
|
||||
|
||||
AND_ASSIGN, // &=
|
||||
OR_ASSIGN, // |=
|
||||
XOR_ASSIGN, // ^=
|
||||
SHL_ASSIGN, // <<=
|
||||
SHR_ASSIGN, // >>=
|
||||
AND_NOT_ASSIGN, // &^=
|
||||
|
||||
LAND, // &&
|
||||
LOR, // ||
|
||||
ARROW, // <-
|
||||
INC, // ++
|
||||
DEC, // --
|
||||
|
||||
EQL, // ==
|
||||
LSS, // <
|
||||
GTR, // >
|
||||
ASSIGN, // =
|
||||
NOT, // !
|
||||
|
||||
NEQ, // !=
|
||||
LEQ, // <=
|
||||
GEQ, // >=
|
||||
DEFINE, // :=
|
||||
ELLIPSIS, // ...
|
||||
|
||||
LPAREN, // (
|
||||
LBRACK, // [
|
||||
LBRACE, // {
|
||||
COMMA, // ,
|
||||
PERIOD, // .
|
||||
|
||||
RPAREN, // )
|
||||
RBRACK, // ]
|
||||
RBRACE, // }
|
||||
SEMICOLON, // ;
|
||||
COLON, // :
|
||||
|
||||
BREAK,
|
||||
CASE,
|
||||
CHAN,
|
||||
CONST,
|
||||
CONTINUE,
|
||||
|
||||
DEFAULT,
|
||||
DEFER,
|
||||
ELSE,
|
||||
FALLTHROUGH,
|
||||
FOR,
|
||||
|
||||
FUNC,
|
||||
GO,
|
||||
GOTO,
|
||||
IF,
|
||||
IMPORT,
|
||||
|
||||
INTERFACE,
|
||||
MAP,
|
||||
PACKAGE,
|
||||
RANGE,
|
||||
RETURN,
|
||||
|
||||
SELECT,
|
||||
STRUCT,
|
||||
SWITCH,
|
||||
TYPE,
|
||||
VAR,
|
||||
}
|
||||
|
||||
impl Token {
|
||||
pub const fn is_assign_op(&self) -> bool {
|
||||
use Token::*;
|
||||
matches!(
|
||||
self,
|
||||
ADD_ASSIGN
|
||||
| SUB_ASSIGN
|
||||
| MUL_ASSIGN
|
||||
| QUO_ASSIGN
|
||||
| REM_ASSIGN
|
||||
| AND_ASSIGN
|
||||
| OR_ASSIGN
|
||||
| XOR_ASSIGN
|
||||
| SHL_ASSIGN
|
||||
| SHR_ASSIGN
|
||||
| AND_NOT_ASSIGN
|
||||
)
|
||||
}
|
||||
|
||||
// https://go.dev/ref/spec#Operator_precedence
|
||||
pub fn precedence(&self) -> u8 {
|
||||
use Token::*;
|
||||
match self {
|
||||
MUL | QUO | REM | SHL | SHR | AND | AND_NOT => 5,
|
||||
ADD | SUB | OR | XOR => 4,
|
||||
EQL | NEQ | LSS | LEQ | GTR | GEQ => 3,
|
||||
LAND => 2,
|
||||
LOR => 1,
|
||||
_ => unreachable!(
|
||||
"precedence() is only supported for binary operators, called with: {:?}",
|
||||
self
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn lowest_precedence() -> u8 {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&Token> for &'static str {
|
||||
fn from(token: &Token) -> Self {
|
||||
use Token::*;
|
||||
|
||||
match token {
|
||||
EOF => "EOF",
|
||||
COMMENT => "COMMENT",
|
||||
|
||||
IDENT => "IDENT",
|
||||
INT => "INT",
|
||||
FLOAT => "FLOAT",
|
||||
IMAG => "IMAG",
|
||||
CHAR => "CHAR",
|
||||
STRING => "STRING",
|
||||
|
||||
ADD => "+",
|
||||
SUB => "-",
|
||||
MUL => "*",
|
||||
QUO => "/",
|
||||
REM => "%",
|
||||
|
||||
AND => "&",
|
||||
OR => "|",
|
||||
XOR => "^",
|
||||
SHL => "<<",
|
||||
SHR => ">>",
|
||||
AND_NOT => "&^",
|
||||
|
||||
ADD_ASSIGN => "+=",
|
||||
SUB_ASSIGN => "-=",
|
||||
MUL_ASSIGN => "*=",
|
||||
QUO_ASSIGN => "/=",
|
||||
REM_ASSIGN => "%=",
|
||||
|
||||
AND_ASSIGN => "&=",
|
||||
OR_ASSIGN => "|=",
|
||||
XOR_ASSIGN => "^=",
|
||||
SHL_ASSIGN => "<<=",
|
||||
SHR_ASSIGN => ">>=",
|
||||
AND_NOT_ASSIGN => "&^=",
|
||||
|
||||
LAND => "&&",
|
||||
LOR => "||",
|
||||
ARROW => "<-",
|
||||
INC => "++",
|
||||
DEC => "--",
|
||||
|
||||
EQL => "==",
|
||||
LSS => "<",
|
||||
GTR => ">",
|
||||
ASSIGN => "=",
|
||||
NOT => "!",
|
||||
|
||||
NEQ => "!=",
|
||||
LEQ => "<=",
|
||||
GEQ => ">=",
|
||||
DEFINE => ":=",
|
||||
ELLIPSIS => "...",
|
||||
|
||||
LPAREN => "(",
|
||||
LBRACK => "[",
|
||||
LBRACE => "{",
|
||||
COMMA => ",",
|
||||
PERIOD => ".",
|
||||
|
||||
RPAREN => ")",
|
||||
RBRACK => "]",
|
||||
RBRACE => "}",
|
||||
SEMICOLON => ";",
|
||||
COLON => ":",
|
||||
|
||||
BREAK => "break",
|
||||
CASE => "case",
|
||||
CHAN => "chan",
|
||||
CONST => "const",
|
||||
CONTINUE => "continue",
|
||||
|
||||
DEFAULT => "default",
|
||||
DEFER => "defer",
|
||||
ELSE => "else",
|
||||
FALLTHROUGH => "fallthrough",
|
||||
FOR => "for",
|
||||
|
||||
FUNC => "func",
|
||||
GO => "go",
|
||||
GOTO => "goto",
|
||||
IF => "if",
|
||||
IMPORT => "import",
|
||||
|
||||
INTERFACE => "interface",
|
||||
MAP => "map",
|
||||
PACKAGE => "package",
|
||||
RANGE => "range",
|
||||
RETURN => "return",
|
||||
|
||||
SELECT => "select",
|
||||
STRUCT => "struct",
|
||||
SWITCH => "switch",
|
||||
TYPE => "type",
|
||||
VAR => "var",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -181,7 +181,6 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
|
||||
"git" => "GitPython",
|
||||
"u" => "requests",
|
||||
"f" => "requests",
|
||||
"." => "requests",
|
||||
"shopify" => "ShopifyAPI",
|
||||
"seleniumwire" => "selenium-wire",
|
||||
"openbb-terminal" => "openbb[all]",
|
||||
@@ -218,29 +217,20 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
|
||||
let ast = parser::parse_program(code, "main.py").map_err(|e| {
|
||||
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
|
||||
})?;
|
||||
let mut imports: Vec<String> = ast
|
||||
let imports = ast
|
||||
.into_iter()
|
||||
.filter_map(|x| match x {
|
||||
Located { node, .. } => match node {
|
||||
StmtKind::Import { names } => Some(
|
||||
names
|
||||
.into_iter()
|
||||
.map(|x| {
|
||||
let name = x.node.name;
|
||||
if name.starts_with('.') {
|
||||
".".to_string()
|
||||
} else {
|
||||
name.split('.').next().unwrap_or("").to_string()
|
||||
}
|
||||
})
|
||||
.map(|x| x.node.name.split('.').next().unwrap_or("").to_string())
|
||||
.map(replace_import)
|
||||
.collect::<Vec<String>>(),
|
||||
),
|
||||
StmtKind::ImportFrom { level: Some(i), .. } if i > 0 => {
|
||||
Some(vec!["requests".to_string()])
|
||||
}
|
||||
StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
|
||||
let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
|
||||
|
||||
Some(vec![replace_import(imprt)])
|
||||
}
|
||||
_ => None,
|
||||
@@ -250,7 +240,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
|
||||
.filter(|x| !STDIMPORTS.contains(&x.as_str()))
|
||||
.unique()
|
||||
.collect();
|
||||
imports.sort();
|
||||
|
||||
Ok(imports)
|
||||
}
|
||||
}
|
||||
@@ -452,7 +442,6 @@ import os
|
||||
import wmill
|
||||
from zanzibar.estonie import talin
|
||||
import matplotlib.pyplot as plt
|
||||
from . import tests
|
||||
|
||||
def main():
|
||||
pass
|
||||
@@ -460,7 +449,7 @@ def main():
|
||||
";
|
||||
let r = parse_python_imports(code)?;
|
||||
// println!("{}", serde_json::to_string(&r)?);
|
||||
assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
|
||||
assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -6,22 +6,21 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
|
||||
use serde_json::Value;
|
||||
use windmill_common::error;
|
||||
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};
|
||||
|
||||
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
|
||||
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
|
||||
use swc_ecma_ast::{
|
||||
ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
|
||||
ModuleDecl, ModuleItem, Number, ObjectLit, Param, Pat, Str, TsArrayType, TsEntityName,
|
||||
TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsPropertySignature,
|
||||
TsType, TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
|
||||
ModuleDecl, ModuleItem, Number, ObjectLit, Pat, Str, TsArrayType, TsEntityName, TsKeywordType,
|
||||
TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsPropertySignature, TsType,
|
||||
TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
|
||||
};
|
||||
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
|
||||
|
||||
pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainArgSignature> {
|
||||
pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
|
||||
let cm: Lrc<SourceMap> = Default::default();
|
||||
let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
|
||||
let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
|
||||
let lexer = Lexer::new(
|
||||
// We want to parse ecmascript
|
||||
Syntax::Typescript(TsConfig::default()),
|
||||
@@ -55,17 +54,65 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainAr
|
||||
})) if &sym.to_string() == "main" => Some(function.params),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
if let Some(params) = params {
|
||||
let r = MainArgSignature {
|
||||
Ok(MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
args: params
|
||||
.into_iter()
|
||||
.map(|x| parse_param(x, &cm, skip_dflt))
|
||||
.map(|x| match x.pat {
|
||||
Pat::Ident(ident) => {
|
||||
let (name, typ, nullable) = binding_ident_to_arg(&ident);
|
||||
Ok(Arg {
|
||||
otyp: None,
|
||||
name,
|
||||
typ,
|
||||
default: None,
|
||||
has_default: ident.id.optional || nullable,
|
||||
})
|
||||
}
|
||||
Pat::Assign(AssignPat { left, right, .. }) => {
|
||||
let (name, mut typ, _nullable) =
|
||||
left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
|
||||
error::Error::ExecutionErr(format!(
|
||||
"parameter syntax unsupported: `{}`",
|
||||
cm.span_to_snippet(left.span())
|
||||
.unwrap_or_else(|_| cm.span_to_string(left.span()))
|
||||
))
|
||||
})?;
|
||||
|
||||
let span = match *right {
|
||||
Expr::Lit(Lit::Str(Str { span, .. })) => Some(span),
|
||||
Expr::Lit(Lit::Num(Number { span, .. })) => Some(span),
|
||||
Expr::Lit(Lit::BigInt(BigInt { span, .. })) => Some(span),
|
||||
Expr::Lit(Lit::Bool(Bool { span, .. })) => Some(span),
|
||||
Expr::Object(ObjectLit { span, .. }) => Some(span),
|
||||
Expr::Array(ArrayLit { span, .. }) => Some(span),
|
||||
_ => None,
|
||||
};
|
||||
let expr = span
|
||||
.and_then(|x| cm.span_to_snippet(x).ok())
|
||||
.map(|x| serde_json::from_str(&x).map_err(|_| x));
|
||||
|
||||
let default = match expr.clone() {
|
||||
Some(Ok(x)) => Some(x),
|
||||
Some(Err(x)) => eval_sync(&x).ok(),
|
||||
None => None,
|
||||
};
|
||||
|
||||
if typ == Typ::Unknown && default.is_some() {
|
||||
typ = json_to_typ(default.as_ref().unwrap());
|
||||
}
|
||||
Ok(Arg { otyp: None, name, typ, default, has_default: true })
|
||||
}
|
||||
_ => Err(error::Error::ExecutionErr(format!(
|
||||
"parameter syntax unsupported: `{}`",
|
||||
cm.span_to_snippet(x.span())
|
||||
.unwrap_or_else(|_| cm.span_to_string(x.span()))
|
||||
))),
|
||||
})
|
||||
.collect::<Result<Vec<Arg>, error::Error>>()?,
|
||||
};
|
||||
Ok(r)
|
||||
})
|
||||
} else {
|
||||
Err(error::Error::ExecutionErr(
|
||||
"main function was not findable (expected to find 'export function main(...)'"
|
||||
@@ -74,75 +121,6 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainAr
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<Arg> {
|
||||
let r = match x.pat {
|
||||
Pat::Ident(ident) => {
|
||||
let (name, typ, nullable) = binding_ident_to_arg(&ident);
|
||||
Ok(Arg {
|
||||
otyp: None,
|
||||
name,
|
||||
typ,
|
||||
default: None,
|
||||
has_default: ident.id.optional || nullable,
|
||||
})
|
||||
}
|
||||
Pat::Assign(AssignPat { left, right, .. }) => {
|
||||
let (name, mut typ, _nullable) =
|
||||
left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
|
||||
error::Error::ExecutionErr(format!(
|
||||
"parameter syntax unsupported: `{}`",
|
||||
cm.span_to_snippet(left.span())
|
||||
.unwrap_or_else(|_| cm.span_to_string(left.span()))
|
||||
))
|
||||
})?;
|
||||
|
||||
let dflt = if skip_dflt {
|
||||
None
|
||||
} else {
|
||||
match *right {
|
||||
Expr::Lit(Lit::Str(Str { value, .. })) => {
|
||||
Some(Value::String(value.to_string()))
|
||||
}
|
||||
Expr::Lit(Lit::Num(Number { value, .. }))
|
||||
if (value == (value as u64) as f64) =>
|
||||
{
|
||||
Some(serde_json::json!(value as u64))
|
||||
}
|
||||
Expr::Lit(Lit::Num(Number { value, .. })) => Some(serde_json::json!(value)),
|
||||
Expr::Lit(Lit::BigInt(BigInt { value, .. })) => Some(serde_json::json!(value)),
|
||||
Expr::Lit(Lit::Bool(Bool { value, .. })) => Some(Value::Bool(value)),
|
||||
Expr::Object(ObjectLit { span, .. }) => eval_span(span, cm),
|
||||
Expr::Array(ArrayLit { span, .. }) => eval_span(span, cm),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
|
||||
if typ == Typ::Unknown && dflt.is_some() {
|
||||
typ = json_to_typ(dflt.as_ref().unwrap());
|
||||
}
|
||||
Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true })
|
||||
}
|
||||
_ => Err(error::Error::ExecutionErr(format!(
|
||||
"parameter syntax unsupported: `{}`",
|
||||
cm.span_to_snippet(x.span())
|
||||
.unwrap_or_else(|_| cm.span_to_string(x.span()))
|
||||
))),
|
||||
};
|
||||
r
|
||||
}
|
||||
|
||||
fn eval_span(span: Span, cm: &Lrc<SourceMap>) -> Option<Value> {
|
||||
let expr = cm
|
||||
.span_to_snippet(span)
|
||||
.ok()
|
||||
.map(|x| serde_json::from_str(&x).map_err(|_| x));
|
||||
|
||||
match expr {
|
||||
Some(Ok(x)) => Some(x),
|
||||
Some(Err(x)) => eval_sync(&x).ok(),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
|
||||
let (typ, nullable) = type_ann
|
||||
.as_ref()
|
||||
@@ -268,7 +246,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
|
||||
pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
|
||||
let mut context = JsRuntime::new(RuntimeOptions::default());
|
||||
let code = format!("let x = {}; x", code);
|
||||
let res = context.execute_script("<anon>", code.into());
|
||||
let res = context.execute_script("<anon>", &code);
|
||||
match res {
|
||||
Ok(global) => {
|
||||
let scope = &mut context.handle_scope();
|
||||
@@ -304,7 +282,7 @@ export function main(test1?: string, test2: string = \"burkina\",
|
||||
}
|
||||
";
|
||||
assert_eq!(
|
||||
parse_deno_signature(code, false)?,
|
||||
parse_deno_signature(code)?,
|
||||
MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
@@ -416,7 +394,7 @@ export function main(test2 = \"burkina\",
|
||||
}
|
||||
";
|
||||
assert_eq!(
|
||||
parse_deno_signature(code, false)?,
|
||||
parse_deno_signature(code)?,
|
||||
MainArgSignature {
|
||||
star_args: false,
|
||||
star_kwargs: false,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,4 @@
|
||||
#[cfg(feature = "enterprise")]
|
||||
use base64::Engine;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use rsa::{pkcs8::DecodePublicKey, signature::Verifier};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use sha2::Sha256;
|
||||
@@ -13,9 +11,9 @@ pub fn verify_license_key(license_key: Option<String>) -> anyhow::Result<()> {
|
||||
.expect("license_key can be splitted with a .");
|
||||
|
||||
let pub_key = rsa::RsaPublicKey::from_public_key_der(
|
||||
&base64::engine::general_purpose::STANDARD.decode("MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB")?)?;
|
||||
let msg = base64::engine::general_purpose::STANDARD.decode(splitted_lk.0)?;
|
||||
let signature = base64::engine::general_purpose::STANDARD.decode(splitted_lk.1)?;
|
||||
&base64::decode("MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB")?)?;
|
||||
let msg = base64::decode(splitted_lk.0)?;
|
||||
let signature = base64::decode(splitted_lk.1)?;
|
||||
rsa::pss::VerifyingKey::<Sha256>::new(pub_key)
|
||||
.verify(&msg, &rsa::pss::Signature::from(signature))
|
||||
.map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?;
|
||||
|
||||
@@ -6,32 +6,17 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
sync::Arc,
|
||||
};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use git_version::git_version;
|
||||
use monitor::handle_zombie_jobs_periodically;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{
|
||||
fs::{metadata, DirBuilder},
|
||||
join,
|
||||
sync::RwLock,
|
||||
};
|
||||
use windmill_common::{utils::rd_string, METRICS_ADDR};
|
||||
use windmill_worker::{
|
||||
DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, PIP_CACHE_DIR,
|
||||
ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
|
||||
};
|
||||
use windmill_common::utils::rd_string;
|
||||
|
||||
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
|
||||
const DEFAULT_NUM_WORKERS: usize = 3;
|
||||
const DEFAULT_PORT: u16 = 8000;
|
||||
const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
|
||||
|
||||
mod ee;
|
||||
mod monitor;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
@@ -44,12 +29,15 @@ async fn main() -> anyhow::Result<()> {
|
||||
.and_then(|x| x.parse::<i32>().ok())
|
||||
.unwrap_or(DEFAULT_NUM_WORKERS as i32);
|
||||
|
||||
let metrics_addr: Option<SocketAddr> = *METRICS_ADDR;
|
||||
|
||||
let server_bind_address: IpAddr = std::env::var("SERVER_BIND_ADDR")
|
||||
let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(IpAddr::from(DEFAULT_SERVER_BIND_ADDR));
|
||||
.map(|s| {
|
||||
s.parse::<bool>()
|
||||
.map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
|
||||
.or_else(|_| s.parse::<SocketAddr>().map(Some))
|
||||
})
|
||||
.transpose()?
|
||||
.flatten();
|
||||
|
||||
let port: u16 = std::env::var("PORT")
|
||||
.ok()
|
||||
@@ -63,164 +51,109 @@ async fn main() -> anyhow::Result<()> {
|
||||
.and_then(|x| x.parse::<bool>().ok())
|
||||
.unwrap_or(false);
|
||||
|
||||
let rsmq_config = std::env::var("REDIS_URL").ok().map(|x| {
|
||||
let url = x.parse::<url::Url>().unwrap();
|
||||
let mut config = rsmq_async::RsmqOptions { ..Default::default() };
|
||||
|
||||
config.host = url.host_str().expect("redis host required").to_owned();
|
||||
config.password = url.password().map(|s| s.to_owned());
|
||||
config.db = url
|
||||
.path_segments()
|
||||
.and_then(|mut segments| segments.next())
|
||||
.and_then(|segment| segment.parse().ok())
|
||||
.unwrap_or(0);
|
||||
config.ns = url
|
||||
.query_pairs()
|
||||
.find(|s| s.0 == "rsmq_namespace")
|
||||
.map(|s| s.1)
|
||||
.unwrap_or(std::borrow::Cow::Borrowed("rsmq"))
|
||||
.into_owned();
|
||||
config.port = url.port().unwrap_or(6379).to_string();
|
||||
|
||||
config
|
||||
});
|
||||
|
||||
let db = windmill_common::connect_db(server_mode).await?;
|
||||
|
||||
let rsmq = if let Some(config) = rsmq_config {
|
||||
let mut rsmq = rsmq_async::MultiplexedRsmq::new(config).await.unwrap();
|
||||
|
||||
let _ = rsmq_async::RsmqConnection::create_queue(&mut rsmq, "main_queue", None, None, None)
|
||||
.await;
|
||||
Some(rsmq)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if server_mode {
|
||||
windmill_api::migrate_db(&db).await?;
|
||||
}
|
||||
|
||||
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
|
||||
let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
tracing::info!(
|
||||
"
|
||||
##############################
|
||||
Windmill Enterprise Edition {GIT_VERSION}
|
||||
##############################"
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
tracing::info!(
|
||||
"
|
||||
##############################
|
||||
Windmill Community Edition {GIT_VERSION}
|
||||
##############################"
|
||||
);
|
||||
|
||||
display_config(vec![
|
||||
"DISABLE_NSJAIL",
|
||||
"DISABLE_SERVER",
|
||||
"NUM_WORKERS",
|
||||
"METRICS_ADDR",
|
||||
"JSON_FMT",
|
||||
"BASE_URL",
|
||||
"BASE_INTERNAL_URL",
|
||||
"TIMEOUT",
|
||||
"ZOMBIE_JOB_TIMEOUT",
|
||||
"RESTART_ZOMBIE_JOBS",
|
||||
"SLEEP_QUEUE",
|
||||
"MAX_LOG_SIZE",
|
||||
"SERVER_BIND_ADDR",
|
||||
"PORT",
|
||||
"KEEP_JOB_DIR",
|
||||
"S3_CACHE_BUCKET",
|
||||
"TAR_CACHE_RATE",
|
||||
"COOKIE_DOMAIN",
|
||||
"PYTHON_PATH",
|
||||
"DENO_PATH",
|
||||
"GO_PATH",
|
||||
"GOPRIVATE",
|
||||
"NETRC",
|
||||
"PIP_INDEX_URL",
|
||||
"PIP_EXTRA_INDEX_URL",
|
||||
"PIP_TRUSTED_HOST",
|
||||
"PATH",
|
||||
"HOME",
|
||||
"DATABASE_CONNECTIONS",
|
||||
"TIMEOUT_WAIT_RESULT",
|
||||
"QUEUE_LIMIT_WAIT_RESULT",
|
||||
"DENO_AUTH_TOKENS",
|
||||
"DENO_FLAGS",
|
||||
"NPM_CONFIG_REGISTRY",
|
||||
"PIP_LOCAL_DEPENDENCIES",
|
||||
"ADDITIONAL_PYTHON_PATHS",
|
||||
"INCLUDE_HEADERS",
|
||||
"WHITELIST_WORKSPACES",
|
||||
"BLACKLIST_WORKSPACES",
|
||||
"INSTANCE_EVENTS_WEBHOOK",
|
||||
"CLOUD_HOSTED",
|
||||
"GLOBAL_CACHE_INTERVAL",
|
||||
"WORKER_TAGS",
|
||||
"CUSTOM_TAGS",
|
||||
"JOB_RETENTION_SECS",
|
||||
"WAIT_RESULT_FAST_POLL_DURATION_SECS",
|
||||
"WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
|
||||
"WAIT_RESULT_FAST_POLL_INTERVAL_MS",
|
||||
"EXIT_AFTER_NO_JOB_FOR_SECS",
|
||||
]);
|
||||
let shutdown_signal = windmill_common::shutdown_signal(tx);
|
||||
|
||||
if server_mode || num_workers > 0 {
|
||||
let addr = SocketAddr::from((server_bind_address, port));
|
||||
let addr = SocketAddr::from(([0, 0, 0, 0], port));
|
||||
|
||||
let rsmq2 = rsmq.clone();
|
||||
let server_f = async {
|
||||
if server_mode {
|
||||
windmill_api::run_server(db.clone(), rsmq2, addr, rx.resubscribe()).await?;
|
||||
windmill_api::run_server(db.clone(), addr, rx.resubscribe()).await?;
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let workers_f = async {
|
||||
if num_workers > 0 {
|
||||
#[cfg(feature = "enterprise")]
|
||||
tracing::info!(
|
||||
"
|
||||
##############################
|
||||
Windmill Enterprise Edition {GIT_VERSION}
|
||||
##############################"
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
tracing::info!(
|
||||
"
|
||||
##############################
|
||||
Windmill Community Edition {GIT_VERSION}
|
||||
##############################"
|
||||
);
|
||||
|
||||
display_config(vec![
|
||||
"DISABLE_NSJAIL",
|
||||
"DISABLE_SERVER",
|
||||
"NUM_WORKERS",
|
||||
"METRICS_ADDR",
|
||||
"JSON_FMT",
|
||||
"BASE_URL",
|
||||
"BASE_INTERNAL_URL",
|
||||
"TIMEOUT",
|
||||
"SLEEP_QUEUE",
|
||||
"MAX_LOG_SIZE",
|
||||
"PORT",
|
||||
"KEEP_JOB_DIR",
|
||||
"S3_CACHE_BUCKET",
|
||||
"TAR_CACHE_RATE",
|
||||
"COOKIE_DOMAIN",
|
||||
"PYTHON_PATH",
|
||||
"DENO_PATH",
|
||||
"GO_PATH",
|
||||
"PIP_INDEX_URL",
|
||||
"PIP_EXTRA_INDEX_URL",
|
||||
"PIP_TRUSTED_HOST",
|
||||
"PATH",
|
||||
"HOME",
|
||||
"DATABASE_CONNECTIONS",
|
||||
"TIMEOUT_WAIT_RESULT",
|
||||
"QUEUE_LIMIT_WAIT_RESULT",
|
||||
"DENO_AUTH_TOKENS",
|
||||
"DENO_FLAGS",
|
||||
"PIP_LOCAL_DEPENDENCIES",
|
||||
"ADDITIONAL_PYTHON_PATHS",
|
||||
"INCLUDE_HEADERS",
|
||||
"WHITELIST_WORKSPACES",
|
||||
"BLACKLIST_WORKSPACES",
|
||||
"NEW_USER_WEBHOOK",
|
||||
"CLOUD_HOSTED",
|
||||
]);
|
||||
|
||||
run_workers(
|
||||
db.clone(),
|
||||
rx.resubscribe(),
|
||||
num_workers,
|
||||
base_internal_url.clone(),
|
||||
rsmq.clone(),
|
||||
)
|
||||
.await?;
|
||||
tracing::info!("All workers exited.");
|
||||
tx.send(())?; // signal server to shutdown
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let rsmq2 = rsmq.clone();
|
||||
let monitor_f = async {
|
||||
if server_mode {
|
||||
monitor_db(&db, rx.resubscribe(), &base_internal_url, rsmq2);
|
||||
monitor_db(&db, rx.resubscribe(), &base_internal_url);
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
|
||||
let metrics_f = async {
|
||||
match metrics_addr {
|
||||
Some(addr) => {
|
||||
windmill_common::serve_metrics(addr, rx.resubscribe(), num_workers > 0)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)
|
||||
}
|
||||
Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
|
||||
.await
|
||||
.map_err(anyhow::Error::from),
|
||||
None => Ok(()),
|
||||
}
|
||||
};
|
||||
|
||||
futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
|
||||
} else {
|
||||
tracing::info!("Nothing to do, exiting.");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -242,11 +175,10 @@ fn display_config(envs: Vec<&str>) {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
pub fn monitor_db(
|
||||
db: &Pool<Postgres>,
|
||||
rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) {
|
||||
let db1 = db.clone();
|
||||
let db2 = db.clone();
|
||||
@@ -254,19 +186,16 @@ pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>
|
||||
let rx2 = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
tokio::spawn(async move {
|
||||
join!(
|
||||
handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
|
||||
windmill_api::delete_expired_items_perdiodically(&db2, rx2)
|
||||
);
|
||||
windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url).await
|
||||
});
|
||||
tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
|
||||
}
|
||||
|
||||
pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
pub async fn run_workers(
|
||||
db: Pool<Postgres>,
|
||||
rx: tokio::sync::broadcast::Receiver<()>,
|
||||
num_workers: i32,
|
||||
base_internal_url: String,
|
||||
rsmq: Option<R>,
|
||||
) -> anyhow::Result<()> {
|
||||
let license_key = std::env::var("LICENSE_KEY").ok();
|
||||
#[cfg(feature = "enterprise")]
|
||||
@@ -289,28 +218,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
|
||||
let mut handles = Vec::with_capacity(num_workers as usize);
|
||||
|
||||
if metadata(&ROOT_TMP_CACHE_DIR).await.is_ok() {
|
||||
if let Err(e) = tokio::fs::remove_dir_all(&ROOT_TMP_CACHE_DIR).await {
|
||||
tracing::info!(error = %e, "Could not remove root tmp cache dir");
|
||||
}
|
||||
}
|
||||
|
||||
for x in [
|
||||
PIP_CACHE_DIR,
|
||||
DENO_CACHE_DIR,
|
||||
GO_CACHE_DIR,
|
||||
TAR_PIP_TMP_CACHE_DIR,
|
||||
DENO_TMP_CACHE_DIR,
|
||||
GO_TMP_CACHE_DIR,
|
||||
] {
|
||||
DirBuilder::new()
|
||||
.recursive(true)
|
||||
.create(x)
|
||||
.await
|
||||
.expect("could not create initial worker dir");
|
||||
}
|
||||
|
||||
let sync_barrier = Arc::new(RwLock::new(None));
|
||||
for i in 1..(num_workers + 1) {
|
||||
let db1 = db.clone();
|
||||
let instance_name = instance_name.clone();
|
||||
@@ -318,8 +225,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
let ip = ip.clone();
|
||||
let rx = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.clone();
|
||||
let rsmq2 = rsmq.clone();
|
||||
let sync_barrier = sync_barrier.clone();
|
||||
handles.push(tokio::spawn(monitor.instrument(async move {
|
||||
tracing::info!(worker = %worker_name, "starting worker");
|
||||
windmill_worker::run_worker(
|
||||
@@ -327,12 +232,9 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
&instance_name,
|
||||
worker_name,
|
||||
i as u64,
|
||||
num_workers as u32,
|
||||
&ip,
|
||||
rx,
|
||||
&base_internal_url,
|
||||
rsmq2,
|
||||
sync_barrier,
|
||||
)
|
||||
.await
|
||||
})));
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{error, jobs::{JobKind, QueuedJob}, METRICS_ENABLED};
|
||||
use windmill_worker::{
|
||||
create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
|
||||
};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<String>().ok())
|
||||
.unwrap_or_else(|| "30".to_string());
|
||||
|
||||
|
||||
pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<bool>().ok())
|
||||
.unwrap_or(true);
|
||||
|
||||
static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
|
||||
"queue_zombie_restart_count",
|
||||
"Total number of jobs restarted due to ping timeout."
|
||||
)
|
||||
.unwrap();
|
||||
static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
|
||||
"queue_zombie_delete_count",
|
||||
"Total number of jobs deleted due to their ping timing out in an unrecoverable state."
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub async fn handle_zombie_jobs_periodically<
|
||||
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) {
|
||||
loop {
|
||||
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(30)) => (),
|
||||
_ = rx.recv() => {
|
||||
println!("received killpill for monitor job");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
db: &Pool<Postgres>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) {
|
||||
if *RESTART_ZOMBIE_JOBS {
|
||||
let restarted = sqlx::query!(
|
||||
"UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
|
||||
*ZOMBIE_JOB_TIMEOUT,
|
||||
JobKind::Flow: JobKind,
|
||||
JobKind::FlowPreview: JobKind,
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_else(|| vec![]);
|
||||
|
||||
if *METRICS_ENABLED {
|
||||
QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
|
||||
}
|
||||
for r in restarted {
|
||||
tracing::info!(
|
||||
"restarted zombie job {} {} {}",
|
||||
r.id,
|
||||
r.workspace_id,
|
||||
r.last_ping
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut timeout_query = "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2".to_string();
|
||||
if *RESTART_ZOMBIE_JOBS {
|
||||
timeout_query.push_str(" AND same_worker = true");
|
||||
};
|
||||
let timeouts = sqlx::query_as::<_, QueuedJob>(&timeout_query)
|
||||
.bind(ZOMBIE_JOB_TIMEOUT.as_str())
|
||||
.bind(JobKind::Flow)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_else(|| vec![]);
|
||||
|
||||
if *METRICS_ENABLED {
|
||||
QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
|
||||
}
|
||||
|
||||
for job in timeouts {
|
||||
tracing::info!("timedout zombie job {} {}", job.id, job.workspace_id,);
|
||||
|
||||
// since the job is unrecoverable, the same worker queue should never be sent anything
|
||||
let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
|
||||
|
||||
let token = create_token_for_owner(
|
||||
&db,
|
||||
&job.workspace_id,
|
||||
&job.permissioned_as,
|
||||
"ephemeral-zombie-jobs",
|
||||
*SESSION_TOKEN_EXPIRY,
|
||||
&job.email,
|
||||
)
|
||||
.await
|
||||
.expect("could not create job token");
|
||||
|
||||
let client = AuthedClient {
|
||||
base_internal_url: base_internal_url.to_string(),
|
||||
token,
|
||||
workspace: job.workspace_id.to_string(),
|
||||
client: OnceCell::new(),
|
||||
};
|
||||
|
||||
let last_ping = job.last_ping.clone();
|
||||
let _ = handle_job_error(
|
||||
db,
|
||||
&client,
|
||||
job,
|
||||
error::Error::ExecutionErr(format!(
|
||||
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
|
||||
last_ping
|
||||
.map(|x| x.to_string())
|
||||
.unwrap_or_else(|| "no ping".to_string()),
|
||||
*ZOMBIE_JOB_TIMEOUT
|
||||
)),
|
||||
None,
|
||||
true,
|
||||
same_worker_tx_never_used,
|
||||
"",
|
||||
base_internal_url,
|
||||
rsmq.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,13 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{stream, Stream};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
|
||||
use tokio::sync::RwLock;
|
||||
use windmill_api::jobs::{CompletedJob, Job};
|
||||
use windmill_api_client::types::CreateFlowBody;
|
||||
use windmill_common::{
|
||||
flow_status::{FlowStatus, FlowStatusModule},
|
||||
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
|
||||
jobs::{JobPayload, RawCode},
|
||||
scripts::ScriptLang,
|
||||
};
|
||||
use windmill_queue::get_queued_job;
|
||||
use windmill_queue::{get_queued_job, JobPayload, RawCode};
|
||||
|
||||
async fn initialize_tracing() {
|
||||
use std::sync::Once;
|
||||
@@ -93,7 +87,7 @@ impl ApiServer {
|
||||
let addr = sock.local_addr().unwrap();
|
||||
drop(sock);
|
||||
|
||||
let task = tokio::task::spawn(windmill_api::run_server(db.clone(), None, addr, rx));
|
||||
let task = tokio::task::spawn(windmill_api::run_server(db.clone(), addr, rx));
|
||||
|
||||
return Self { addr, tx, task };
|
||||
}
|
||||
@@ -133,7 +127,8 @@ mod suspend_resume {
|
||||
use futures::{Stream, StreamExt};
|
||||
use serde_json::json;
|
||||
use sqlx::{query_scalar, types::Uuid};
|
||||
use windmill_common::{flows::FlowValue, jobs::JobPayload};
|
||||
use windmill_common::flows::FlowValue;
|
||||
use windmill_queue::JobPayload;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -159,12 +154,12 @@ mod suspend_resume {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"modules": [{
|
||||
"id": "a",
|
||||
"input_transform": {
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port", },
|
||||
"op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", },
|
||||
},
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port", },
|
||||
"op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": "\
|
||||
@@ -198,12 +193,12 @@ mod suspend_resume {
|
||||
},
|
||||
}, {
|
||||
"id": "b",
|
||||
"input_transform": {
|
||||
"n": { "type": "javascript", "expr": "results.a", },
|
||||
"resume": { "type": "javascript", "expr": "resume", },
|
||||
"resumes": { "type": "javascript", "expr": "resumes", },
|
||||
},
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": { "type": "javascript", "expr": "results.a", },
|
||||
"resume": { "type": "javascript", "expr": "resume", },
|
||||
"resumes": { "type": "javascript", "expr": "resumes", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }"
|
||||
@@ -212,12 +207,12 @@ mod suspend_resume {
|
||||
"required_events": 1
|
||||
},
|
||||
}, {
|
||||
"input_transform": {
|
||||
"last": { "type": "javascript", "expr": "results.b", },
|
||||
"resume": { "type": "javascript", "expr": "resume", },
|
||||
"resumes": { "type": "javascript", "expr": "resumes", },
|
||||
},
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"last": { "type": "javascript", "expr": "results.b", },
|
||||
"resume": { "type": "javascript", "expr": "resume", },
|
||||
"resumes": { "type": "javascript", "expr": "resumes", },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": "export function main(last, resume, resumes) { return { last, resume, resumes } }"
|
||||
@@ -257,7 +252,9 @@ mod suspend_resume {
|
||||
let second = completed.next().await.unwrap();
|
||||
// print_job(second, &db).await;
|
||||
|
||||
let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
|
||||
let tx = db.begin().await.unwrap();
|
||||
let (tx, token) = windmill_worker::create_token_for_owner(tx, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
|
||||
tx.commit().await.unwrap();
|
||||
let secret = reqwest::get(format!(
|
||||
"http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben"
|
||||
))
|
||||
@@ -358,7 +355,9 @@ mod suspend_resume {
|
||||
/* ... and send a request resume it. */
|
||||
let second = completed.next().await.unwrap();
|
||||
|
||||
let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
|
||||
let tx = db.begin().await.unwrap();
|
||||
let (tx, token) = windmill_worker::create_token_for_owner(tx, "test-workspace", "u/test-user", "", 100, "").await.unwrap();
|
||||
tx.commit().await.unwrap();
|
||||
let secret = reqwest::get(format!(
|
||||
"http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}"
|
||||
))
|
||||
@@ -397,6 +396,7 @@ mod retry {
|
||||
use serde_json::json;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use windmill_common::flows::FlowValue;
|
||||
use windmill_queue::JobPayload;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -477,11 +477,11 @@ def main(last, port):
|
||||
"iterator": { "type": "javascript", "expr": "flow_input.items" },
|
||||
"skip_failures": false,
|
||||
"modules": [{
|
||||
"input_transform": {
|
||||
"index": { "type": "javascript", "expr": "flow_input.iter.index" },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"index": { "type": "javascript", "expr": "flow_input.iter.index" },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": inner_step(),
|
||||
@@ -490,11 +490,11 @@ def main(last, port):
|
||||
},
|
||||
"retry": { "constant": { "attempts": 2, "seconds": 0 } },
|
||||
}, {
|
||||
"input_transform": {
|
||||
"last": { "type": "javascript", "expr": "results.a" },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"last": { "type": "javascript", "expr": "results.a" },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" },
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": last_step(),
|
||||
@@ -632,7 +632,7 @@ def main(last, port):
|
||||
"modules": [{
|
||||
"id": "a",
|
||||
"value": {
|
||||
"input_transforms": { "port": { "type": "javascript", "expr": "flow_input.port" } },
|
||||
"input_transform": { "port": { "type": "javascript", "expr": "flow_input.port" } },
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": r#"
|
||||
@@ -645,7 +645,7 @@ def main(port):
|
||||
}],
|
||||
"failure_module": {
|
||||
"value": {
|
||||
"input_transforms": { "error": { "type": "javascript", "expr": "previous_result", },
|
||||
"input_transform": { "error": { "type": "javascript", "expr": "previous_result", },
|
||||
"port": { "type": "javascript", "expr": "flow_input.port" } },
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
@@ -659,7 +659,7 @@ def main(error, port):
|
||||
},
|
||||
}))
|
||||
.unwrap();
|
||||
let (_attempts, responses) = [
|
||||
let (attempts, responses) = [
|
||||
/* fail the first step twice */
|
||||
(0x00, None),
|
||||
(0x00, None),
|
||||
@@ -681,20 +681,14 @@ def main(error, port):
|
||||
_ => panic!("expected failure module"),
|
||||
}
|
||||
|
||||
println!("result: {:#?}", result);
|
||||
assert_eq!(server.close().await, attempts);
|
||||
assert_eq!(
|
||||
result
|
||||
.get("from failure module")
|
||||
.unwrap()
|
||||
.get("error")
|
||||
.unwrap()
|
||||
.get("name")
|
||||
.unwrap()
|
||||
.clone(),
|
||||
json!("IndexError")
|
||||
result,
|
||||
json!({
|
||||
"recv": 42,
|
||||
"from failure module": {"error": {"name": "IndexError", "stack": " File \"/tmp/inner.py\", line 5, in main\n return sock.recv(1)[0]\n", "message": "index out of range"}},
|
||||
})
|
||||
);
|
||||
|
||||
assert_eq!(result.get("recv").unwrap().clone(), json!(42));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -711,13 +705,13 @@ async fn test_iteration(db: Pool<Postgres>) {
|
||||
"iterator": { "type": "javascript", "expr": "result.items" },
|
||||
"skip_failures": false,
|
||||
"modules": [{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
},
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(n):\n if 1 < n:\n raise StopIteration(n)",
|
||||
@@ -768,13 +762,13 @@ async fn test_iteration_parallel(db: Pool<Postgres>) {
|
||||
"skip_failures": false,
|
||||
"parallel": true,
|
||||
"modules": [{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
},
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(n):\n if 1 < n:\n raise StopIteration(n)",
|
||||
@@ -830,8 +824,9 @@ impl RunJob {
|
||||
|
||||
async fn push(self, db: &Pool<Postgres>) -> Uuid {
|
||||
let RunJob { payload, args } = self;
|
||||
let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
|
||||
(None, db.begin().await.unwrap()).into(),
|
||||
let tx = db.begin().await.unwrap();
|
||||
let (uuid, tx) = windmill_queue::push(
|
||||
tx,
|
||||
"test-workspace",
|
||||
payload,
|
||||
args,
|
||||
@@ -841,16 +836,15 @@ impl RunJob {
|
||||
/* scheduled_for_o */ None,
|
||||
/* schedule_path */ None,
|
||||
/* parent_job */ None,
|
||||
/* root job */ None,
|
||||
/* is_flow_step */ false,
|
||||
/* running */ false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("push has to succeed");
|
||||
tx.commit().await.unwrap();
|
||||
|
||||
tx.commit().await.expect("push has to commit");
|
||||
|
||||
uuid
|
||||
}
|
||||
@@ -881,7 +875,7 @@ async fn in_test_worker<Fut: std::future::Future>(
|
||||
port: u16,
|
||||
) -> <Fut as std::future::Future>::Output {
|
||||
let (quit, worker) = spawn_test_worker(db, port);
|
||||
let worker = tokio::time::timeout(std::time::Duration::from_secs(45), worker);
|
||||
let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker);
|
||||
tokio::pin!(worker);
|
||||
|
||||
let res = tokio::select! {
|
||||
@@ -916,20 +910,18 @@ fn spawn_test_worker(
|
||||
let db = db.to_owned();
|
||||
let worker_instance: &str = "test worker instance";
|
||||
let worker_name: String = next_worker_name();
|
||||
let i_worker: u64 = Default::default();
|
||||
let ip: &str = Default::default();
|
||||
let future = async move {
|
||||
let base_internal_url = format!("http://localhost:{}", port);
|
||||
windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
|
||||
windmill_worker::run_worker(
|
||||
&db,
|
||||
worker_instance,
|
||||
worker_name,
|
||||
1,
|
||||
1,
|
||||
i_worker,
|
||||
ip,
|
||||
rx,
|
||||
&base_internal_url,
|
||||
None,
|
||||
Arc::new(RwLock::new(None)),
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -1007,8 +999,8 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
content: numbers.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
input_transforms: Default::default(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1035,8 +1027,8 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
content: doubles.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
input_transforms: Default::default(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1044,6 +1036,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
sleep: None,
|
||||
}],
|
||||
},
|
||||
input_transforms: Default::default(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1136,8 +1129,8 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
content: write_file.clone(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
input_transforms: Default::default(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1153,29 +1146,29 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
modules: vec![
|
||||
FlowModule {
|
||||
id: "d".to_string(),
|
||||
input_transforms: [
|
||||
(
|
||||
"i".to_string(),
|
||||
InputTransform::Javascript {
|
||||
expr: "flow_input.iter.value".to_string(),
|
||||
},
|
||||
),
|
||||
(
|
||||
"loop".to_string(),
|
||||
InputTransform::Static { value: json!(true) },
|
||||
),
|
||||
(
|
||||
"path".to_string(),
|
||||
InputTransform::Static { value: json!("inner.txt") },
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
value: FlowModuleValue::RawScript {
|
||||
input_transforms: [
|
||||
(
|
||||
"i".to_string(),
|
||||
InputTransform::Javascript {
|
||||
expr: "flow_input.iter.value".to_string(),
|
||||
},
|
||||
),
|
||||
(
|
||||
"loop".to_string(),
|
||||
InputTransform::Static { value: json!(true) },
|
||||
),
|
||||
(
|
||||
"path".to_string(),
|
||||
InputTransform::Static { value: json!("inner.txt") },
|
||||
),
|
||||
]
|
||||
.into(),
|
||||
input_transforms: [].into(),
|
||||
language: ScriptLang::Deno,
|
||||
content: write_file,
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -1201,8 +1194,8 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
input_transforms: [].into(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1211,6 +1204,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
},
|
||||
],
|
||||
},
|
||||
input_transforms: Default::default(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1244,8 +1238,8 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
input_transforms: [].into(),
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
suspend: Default::default(),
|
||||
@@ -1442,6 +1436,7 @@ async fn test_python_flow(db: Pool<Postgres>) {
|
||||
let doubles = "def main(n): return n * 2";
|
||||
|
||||
let flow: FlowValue = serde_json::from_value(serde_json::json!( {
|
||||
"input_transform": {},
|
||||
"modules": [
|
||||
{
|
||||
"value": {
|
||||
@@ -1457,16 +1452,16 @@ async fn test_python_flow(db: Pool<Postgres>) {
|
||||
"skip_failures": false,
|
||||
"modules": [{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
},
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": doubles,
|
||||
},
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
@@ -1499,11 +1494,11 @@ async fn test_python_flow_2(db: Pool<Postgres>) {
|
||||
"modules": [
|
||||
{
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"content": "import wmill\ndef main(): return \"Hello\"",
|
||||
"language": "python3"
|
||||
},
|
||||
"input_transform": {}
|
||||
}
|
||||
]
|
||||
}))
|
||||
@@ -1531,8 +1526,6 @@ async fn test_go_job(db: Pool<Postgres>) {
|
||||
let port = server.addr.port();
|
||||
|
||||
let content = r#"
|
||||
package inner
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main(derp string) (string, error) {
|
||||
@@ -1684,7 +1677,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
|
||||
"modules": [
|
||||
{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
@@ -1700,7 +1693,7 @@ async fn test_empty_loop(db: Pool<Postgres>) {
|
||||
},
|
||||
{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"input_transform": {
|
||||
"items": {
|
||||
"type": "javascript",
|
||||
"expr": "results.a",
|
||||
@@ -1777,13 +1770,13 @@ async fn test_empty_loop_2(db: Pool<Postgres>) {
|
||||
"iterator": { "type": "static", "value": [] },
|
||||
"modules": [
|
||||
{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
},
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(n): return n",
|
||||
@@ -1819,13 +1812,13 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
|
||||
"iterator": { "type": "static", "value": [2,3,4] },
|
||||
"modules": [
|
||||
{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"type": "javascript",
|
||||
"expr": "flow_input.iter.value",
|
||||
},
|
||||
},
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(n): return n",
|
||||
@@ -1835,13 +1828,13 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
|
||||
},
|
||||
},
|
||||
{
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"items": {
|
||||
"type": "javascript",
|
||||
"expr": "results.a",
|
||||
},
|
||||
"input_transform": {
|
||||
"items": {
|
||||
"type": "javascript",
|
||||
"expr": "results.a",
|
||||
},
|
||||
},
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "python3",
|
||||
"content": "def main(items): return sum(items)",
|
||||
@@ -1863,17 +1856,17 @@ async fn test_step_after_loop(db: Pool<Postgres>) {
|
||||
fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value {
|
||||
json!({
|
||||
"id": format!("id_{}", i.to_string().replace("-", "_")),
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"array": {
|
||||
"type": "javascript",
|
||||
"expr": format!("results.{id}"),
|
||||
},
|
||||
"i": {
|
||||
"type": "static",
|
||||
"value": json!(i),
|
||||
}
|
||||
"input_transform": {
|
||||
"array": {
|
||||
"type": "javascript",
|
||||
"expr": format!("results.{id}"),
|
||||
},
|
||||
"i": {
|
||||
"type": "static",
|
||||
"value": json!(i),
|
||||
}
|
||||
},
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": "export function main(array, i){ array.push(i); return array }",
|
||||
@@ -1883,8 +1876,8 @@ fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value {
|
||||
|
||||
fn module_failure() -> serde_json::Value {
|
||||
json!({
|
||||
"input_transform": {},
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": "export function main(){ throw Error('failure') }",
|
||||
@@ -2039,16 +2032,6 @@ async fn test_branchall_simple(db: Pool<Postgres>) {
|
||||
assert_eq!(result, serde_json::json!([[1, 2], [1, 3]]));
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ErrorResult {
|
||||
error: NamedError,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct NamedError {
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
initialize_tracing().await;
|
||||
@@ -2084,11 +2067,8 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
|
||||
.unwrap()
|
||||
.error
|
||||
.name,
|
||||
"Error"
|
||||
result,
|
||||
serde_json::json!([{"error": {"name": "Error", "stack": "Error: failure\n at main (file:///tmp/inner.ts:1:31)\n at run (file:///tmp/main.ts:9:26)\n at file:///tmp/main.ts:14:1", "message": "failure"}}, [1,3]])
|
||||
);
|
||||
|
||||
let flow: FlowValue = serde_json::from_value(json!({
|
||||
@@ -2121,11 +2101,8 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) {
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ErrorResult>(result.get(0).unwrap().clone())
|
||||
.unwrap()
|
||||
.error
|
||||
.name,
|
||||
"Error"
|
||||
result,
|
||||
serde_json::json!([ {"error": {"name": "Error", "stack": "Error: failure\n at main (file:///tmp/inner.ts:1:31)\n at run (file:///tmp/main.ts:9:26)\n at file:///tmp/main.ts:14:1", "message": "failure"}}, [1, 2]])
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2258,7 +2235,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
"modules": [{
|
||||
"id": "a",
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"input_transform": {
|
||||
"l": { "type": "javascript", "expr": "[]", },
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
},
|
||||
@@ -2269,7 +2246,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
}, {
|
||||
"id": "b",
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"input_transform": {
|
||||
"l": { "type": "javascript", "expr": "results.a.l", },
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
},
|
||||
@@ -2279,7 +2256,7 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
},
|
||||
}, {
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"input_transform": {
|
||||
"l": { "type": "javascript", "expr": "results.b.l", },
|
||||
"n": { "type": "javascript", "expr": "flow_input.n", },
|
||||
},
|
||||
@@ -2289,8 +2266,8 @@ async fn test_failure_module(db: Pool<Postgres>) {
|
||||
},
|
||||
}],
|
||||
"failure_module": {
|
||||
"input_transform": { "error": { "type": "javascript", "expr": "previous_result", } },
|
||||
"value": {
|
||||
"input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } },
|
||||
"type": "rawscript",
|
||||
"language": "deno",
|
||||
"content": "export function main(error) { return { 'from failure module': error } }",
|
||||
@@ -2443,12 +2420,9 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
|
||||
client
|
||||
.create_flow(
|
||||
"test-workspace",
|
||||
&CreateFlowBody {
|
||||
open_flow_w_path: windmill_api_client::types::OpenFlowWPath {
|
||||
open_flow: flow,
|
||||
path: "g/all/flow_lock_all".to_owned(),
|
||||
},
|
||||
draft_only: None,
|
||||
&windmill_api_client::types::OpenFlowWPath {
|
||||
open_flow: flow,
|
||||
path: "g/all/flow_lock_all".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2467,17 +2441,16 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
|
||||
.modules
|
||||
.into_iter()
|
||||
.for_each(|m| {
|
||||
tracing::error!("Module: {:?}", m.value);
|
||||
assert!(matches!(
|
||||
m.value,
|
||||
windmill_api_client::types::FlowModuleValue::RawScript {
|
||||
windmill_api_client::types::FlowModuleValue::Rawscript {
|
||||
language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
|
||||
lock: Some(ref lock),
|
||||
..
|
||||
} if lock == "")
|
||||
|| matches!(
|
||||
m.value,
|
||||
windmill_api_client::types::FlowModuleValue::RawScript {
|
||||
windmill_api_client::types::FlowModuleValue::Rawscript {
|
||||
language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
|
||||
lock: Some(ref lock),
|
||||
..
|
||||
|
||||
@@ -21,7 +21,5 @@ rand.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
prettyplease = "0.1.25"
|
||||
progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
|
||||
serde_json = "1.0"
|
||||
syn = "1.0"
|
||||
@@ -13,9 +13,7 @@ fn main() {
|
||||
let spec = serde_json::from_reader(file).unwrap();
|
||||
let mut generator = progenitor::Generator::default();
|
||||
|
||||
let tokens = generator.generate_tokens(&spec).unwrap();
|
||||
let ast = syn::parse2(tokens).unwrap();
|
||||
let content = prettyplease::unparse(&ast);
|
||||
let content = generator.generate_text(&spec).unwrap();
|
||||
|
||||
let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf();
|
||||
out_file.push("codegen.rs");
|
||||
|
||||
@@ -47,7 +47,6 @@ tracing.workspace = true
|
||||
sql-builder.workspace = true
|
||||
serde_json.workspace = true
|
||||
chrono.workspace = true
|
||||
chrono-tz.workspace = true
|
||||
hex.workspace = true
|
||||
base64.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
@@ -70,5 +69,3 @@ async-stripe = { workspace = true, optional = true }
|
||||
lazy_static.workspace = true
|
||||
prometheus.workspace = true
|
||||
async_zip.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
regex.workspace = true
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ use std::collections::HashMap;
|
||||
*/
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
jobs::script_path_to_payload,
|
||||
users::{require_owner_of_path, Authed, OptAuthed},
|
||||
variables::build_crypt,
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
@@ -31,19 +32,17 @@ use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
apps::ListAppQuery,
|
||||
error::{to_anyhow, Error, JsonResult, Result},
|
||||
jobs::{script_path_to_payload, JobPayload, RawCode},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{
|
||||
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{push, QueueTransaction};
|
||||
use windmill_queue::{push, JobPayload, RawCode};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_apps))
|
||||
.route("/get/p/*path", get(get_app))
|
||||
.route("/get/draft/*path", get(get_app_w_draft))
|
||||
.route("/secret_of/*path", get(get_secret_id))
|
||||
.route("/get/v/*id", get(get_app_by_id))
|
||||
.route("/exists/*path", get(exists_app))
|
||||
@@ -75,9 +74,6 @@ pub struct ListableApp {
|
||||
pub execution_mode: String,
|
||||
pub starred: bool,
|
||||
pub edited_at: chrono::DateTime<chrono::Utc>,
|
||||
pub has_draft: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize)]
|
||||
@@ -102,23 +98,6 @@ pub struct AppWithLastVersion {
|
||||
pub extra_perms: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct AppWithLastVersionAndDraft {
|
||||
pub id: i64,
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub policy: serde_json::Value,
|
||||
pub versions: Vec<i64>,
|
||||
pub value: serde_json::Value,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub extra_perms: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
pub type StaticFields = Map<String, Value>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
@@ -147,7 +126,6 @@ pub struct CreateApp {
|
||||
pub summary: String,
|
||||
pub value: serde_json::Value,
|
||||
pub policy: Policy,
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -178,8 +156,6 @@ async fn list_apps(
|
||||
"app_version.created_at as edited_at",
|
||||
"app.extra_perms",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
"draft.path IS NOT NULL as has_draft",
|
||||
"draft_only"
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
@@ -192,11 +168,6 @@ async fn list_apps(
|
||||
.on(
|
||||
"app_version.id = versions[array_upper(versions, 1)]"
|
||||
)
|
||||
.left()
|
||||
.join("draft")
|
||||
.on(
|
||||
"draft.path = app.path AND draft.workspace_id = app.workspace_id AND draft.typ = 'app'"
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("app_version.created_at", true)
|
||||
.and_where("app.workspace_id = ?".bind(&w_id))
|
||||
@@ -244,37 +215,6 @@ async fn get_app(
|
||||
Ok(Json(app))
|
||||
}
|
||||
|
||||
async fn get_app_w_draft(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<AppWithLastVersionAndDraft> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let app_o = sqlx::query_as!(
|
||||
AppWithLastVersionAndDraft,
|
||||
r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
|
||||
app.extra_perms, app_version.value,
|
||||
app_version.created_at, app_version.created_by,
|
||||
app.draft_only, draft.value as "draft?"
|
||||
from app
|
||||
INNER JOIN app_version ON
|
||||
app_version.id = app.versions[array_upper(app.versions, 1)]
|
||||
LEFT JOIN draft ON
|
||||
app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app'
|
||||
WHERE app.path = $1 AND app.workspace_id = $2"#,
|
||||
path.to_owned(),
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let app = not_found_if_none(app_o, "App", path)?;
|
||||
Ok(Json(app))
|
||||
}
|
||||
|
||||
async fn get_app_by_id(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -373,34 +313,18 @@ async fn create_app(
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(mut app): Json<CreateApp>,
|
||||
Json(app): Json<CreateApp>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
app.policy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
|
||||
app.policy.on_behalf_of_email = Some(authed.email);
|
||||
|
||||
if &app.path == "" {
|
||||
return Err(Error::BadRequest("App path cannot be empty".to_string()));
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
|
||||
&app.path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
let id = sqlx::query_scalar!(
|
||||
"INSERT INTO app
|
||||
(workspace_id, path, summary, policy, versions, draft_only)
|
||||
VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
|
||||
(workspace_id, path, summary, policy, versions)
|
||||
VALUES ($1, $2, $3, $4, '{}') RETURNING id",
|
||||
w_id,
|
||||
app.path,
|
||||
app.summary,
|
||||
json!(app.policy),
|
||||
app.draft_only,
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await?;
|
||||
@@ -480,14 +404,6 @@ async fn delete_app(
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM app WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
@@ -518,6 +434,7 @@ async fn update_app(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(ns): Json<EditApp>,
|
||||
) -> Result<String> {
|
||||
@@ -532,10 +449,12 @@ async fn update_app(
|
||||
sqlb.and_where_eq("path", "?".bind(&path));
|
||||
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
|
||||
|
||||
sqlb.set("draft_only", "NULL");
|
||||
if let Some(npath) = &ns.path {
|
||||
if npath != path {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
sqlb.set_str("path", npath);
|
||||
}
|
||||
@@ -544,9 +463,7 @@ async fn update_app(
|
||||
sqlb.set_str("summary", nsummary);
|
||||
}
|
||||
|
||||
if let Some(mut npolicy) = ns.policy {
|
||||
npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
|
||||
npolicy.on_behalf_of_email = Some(authed.email);
|
||||
if let Some(npolicy) = ns.policy {
|
||||
sqlb.set(
|
||||
"policy",
|
||||
&format!(
|
||||
@@ -595,15 +512,6 @@ async fn update_app(
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
@@ -648,7 +556,6 @@ fn digest(code: &str) -> String {
|
||||
async fn execute_component(
|
||||
OptAuthed(opt_authed): OptAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(payload): Json<ExecuteApp>,
|
||||
) -> Result<String> {
|
||||
@@ -667,7 +574,7 @@ async fn execute_component(
|
||||
};
|
||||
|
||||
let path = path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
|
||||
let mut hm = HashMap::new();
|
||||
@@ -726,27 +633,20 @@ async fn execute_component(
|
||||
}
|
||||
};
|
||||
|
||||
let (job_payload, args, tag) = match &payload {
|
||||
let (job_payload, args) = match &payload {
|
||||
ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
|
||||
let content = &raw_code.content;
|
||||
let payload = JobPayload::Code(raw_code.clone());
|
||||
let path = digest(content);
|
||||
let args = build_args(policy, path, args)?;
|
||||
(payload, args, None)
|
||||
(payload, args)
|
||||
}
|
||||
ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
|
||||
let (payload, tag) = if path.starts_with("script/") {
|
||||
script_path_to_payload(
|
||||
path.strip_prefix("script/").unwrap(),
|
||||
tx.transaction_mut(),
|
||||
&w_id,
|
||||
)
|
||||
.await?
|
||||
let payload = if path.starts_with("script/") {
|
||||
script_path_to_payload(path.strip_prefix("script/").unwrap(), &mut tx, &w_id)
|
||||
.await?
|
||||
} else if path.starts_with("flow/") {
|
||||
(
|
||||
JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
|
||||
None,
|
||||
)
|
||||
JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
|
||||
} else {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"path must start with script/ or flow/ (got {})",
|
||||
@@ -754,7 +654,7 @@ async fn execute_component(
|
||||
)));
|
||||
};
|
||||
let args = build_args(policy, path.to_string(), args)?;
|
||||
(payload, args, tag)
|
||||
(payload, args)
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
@@ -770,16 +670,14 @@ async fn execute_component(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(uuid.to_string())
|
||||
}
|
||||
|
||||
@@ -829,9 +727,6 @@ fn build_args(
|
||||
path: String,
|
||||
args: &Map<String, Value>,
|
||||
) -> Result<Map<String, Value>> {
|
||||
// disallow var and res access in args coming from the user for security reasons
|
||||
args.into_iter()
|
||||
.try_for_each(|x| disallow_var_res_access(x.1))?;
|
||||
let static_args = policy
|
||||
.triggerables
|
||||
.get(&path)
|
||||
@@ -852,20 +747,3 @@ fn build_args(
|
||||
}
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
fn disallow_var_res_access(args: &serde_json::Value) -> Result<()> {
|
||||
match args {
|
||||
Value::Object(v) => v.into_iter().try_for_each(|x| disallow_var_res_access(x.1)),
|
||||
Value::Array(arr) => arr.into_iter().try_for_each(|v| disallow_var_res_access(v)),
|
||||
Value::String(s) => {
|
||||
if s.starts_with("$var:") || s.starts_with("$res:") {
|
||||
Err(Error::BadRequest(format!(
|
||||
"For security reasons, variable or resource access is not allowed as dynamic argument"
|
||||
)))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,12 +7,11 @@
|
||||
*/
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
extract::{Extension, Path},
|
||||
routing::{get, post, put},
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::{HeaderMap, StatusCode};
|
||||
use serde::Deserialize;
|
||||
use hyper::StatusCode;
|
||||
use windmill_common::{
|
||||
error::{JsonResult, Result},
|
||||
utils::{not_found_if_none, StripPath},
|
||||
@@ -20,7 +19,6 @@ use windmill_common::{
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
jobs::add_include_headers,
|
||||
users::Authed,
|
||||
};
|
||||
|
||||
@@ -85,21 +83,13 @@ pub async fn new_payload(
|
||||
Ok(StatusCode::CREATED)
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
pub struct IncludeHeaderQuery {
|
||||
include_header: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn update_payload(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<IncludeHeaderQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(payload): Json<serde_json::Value>,
|
||||
) -> Result<StatusCode> {
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let args = add_include_headers(&run_query.include_header, headers, args.unwrap_or_default());
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE capture
|
||||
@@ -109,7 +99,7 @@ pub async fn update_payload(
|
||||
",
|
||||
&w_id,
|
||||
&path.to_path(),
|
||||
serde_json::json!(args),
|
||||
&payload,
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2042
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{maybe_refresh_folders, require_owner_of_path, Authed},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path},
|
||||
routing::post,
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use windmill_common::error::Result;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new().route("/create", post(create_draft))
|
||||
}
|
||||
|
||||
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[sqlx(type_name = "DRAFT_TYPE", rename_all = "lowercase")]
|
||||
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
|
||||
pub enum DraftType {
|
||||
Script,
|
||||
Flow,
|
||||
App,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug)]
|
||||
pub struct Draft {
|
||||
pub path: String,
|
||||
pub value: serde_json::Value,
|
||||
pub typ: DraftType,
|
||||
}
|
||||
|
||||
async fn create_draft(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(draft): Json<Draft>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let authed = maybe_refresh_folders(&draft.path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_owner_of_path(&authed, &draft.path)?;
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO draft
|
||||
(workspace_id, path, value, typ)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3",
|
||||
&w_id,
|
||||
draft.path,
|
||||
draft.value,
|
||||
draft.typ: DraftType,
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, format!("draft {} created", draft.path)))
|
||||
}
|
||||
|
||||
// async fn get_draft(
|
||||
// authed: Authed,
|
||||
// Extension(user_db): Extension<UserDB>,
|
||||
// Path((w_id, path)): Path<(String, StripPath)>,
|
||||
// ) -> JsonResult<Draft> {
|
||||
// let path = path.to_path();
|
||||
// let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
// let script_o = sqlx::query_as!(
|
||||
// Draft,
|
||||
// r#"SELECT path, value, typ as "typ: DraftType" FROM draft WHERE path = $1 AND workspace_id = $2"#,
|
||||
// path,
|
||||
// w_id
|
||||
// )
|
||||
// .fetch_optional(&mut tx)
|
||||
// .await?;
|
||||
// tx.commit().await?;
|
||||
|
||||
// let draft = not_found_if_none(script_o, "draft", path)?;
|
||||
// Ok(Json(draft))
|
||||
// }
|
||||
@@ -6,35 +6,34 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
schedule::clear_schedule,
|
||||
users::{maybe_refresh_folders, require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
HTTP_CLIENT,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use sql_builder::prelude::*;
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
routing::{delete, get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sql_builder::prelude::*;
|
||||
use sql_builder::SqlBuilder;
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{self, to_anyhow, Error, JsonResult, Result},
|
||||
flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
|
||||
jobs::JobPayload,
|
||||
schedule::Schedule,
|
||||
scripts::Schema,
|
||||
utils::{
|
||||
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};
|
||||
use windmill_queue::{push, schedule::push_scheduled_job, JobPayload};
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
schedule::clear_schedule,
|
||||
users::{require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
HTTP_CLIENT,
|
||||
};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -44,7 +43,6 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/archive/*path", post(archive_flow_by_path))
|
||||
.route("/delete/*path", delete(delete_flow_by_path))
|
||||
.route("/get/*path", get(get_flow_by_path))
|
||||
.route("/get/draft/*path", get(get_flow_by_path_w_draft))
|
||||
.route("/exists/*path", get(exists_flow_by_path))
|
||||
.route("/list_paths", get(list_paths))
|
||||
}
|
||||
@@ -75,8 +73,6 @@ async fn list_flows(
|
||||
"archived",
|
||||
"extra_perms",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
"draft.path IS NOT NULL as has_draft",
|
||||
"draft_only"
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
@@ -84,11 +80,6 @@ async fn list_flows(
|
||||
"favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
|
||||
.bind(&authed.username),
|
||||
)
|
||||
.left()
|
||||
.join("draft")
|
||||
.on(
|
||||
"draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'"
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("edited_at", lq.order_desc.unwrap_or(true))
|
||||
.and_where("o.workspace_id = ?".bind(&w_id))
|
||||
@@ -96,8 +87,9 @@ async fn list_flows(
|
||||
.limit(per_page)
|
||||
.clone();
|
||||
|
||||
sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false));
|
||||
|
||||
if !lq.show_archived.unwrap_or(false) {
|
||||
sqlb.and_where_eq("archived", false);
|
||||
}
|
||||
if let Some(ps) = &lq.path_start {
|
||||
sqlb.and_where_like_left("path", "?".bind(ps));
|
||||
}
|
||||
@@ -186,24 +178,20 @@ async fn check_path_conflict<'c>(
|
||||
|
||||
async fn create_flow(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(nf): Json<NewFlow>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
// cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
|
||||
let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await;
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
|
||||
check_path_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
|
||||
check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
|
||||
check_path_conflict(&mut tx, &w_id, &nf.path).await?;
|
||||
check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
|
||||
schema, dependency_job, draft_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8)",
|
||||
schema, dependency_job) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL)",
|
||||
w_id,
|
||||
nf.path,
|
||||
nf.summary,
|
||||
@@ -211,15 +199,6 @@ async fn create_flow(
|
||||
nf.value,
|
||||
&authed.username,
|
||||
nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
|
||||
nf.draft_only
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
|
||||
nf.path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
@@ -240,11 +219,13 @@ async fn create_flow(
|
||||
)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
|
||||
);
|
||||
|
||||
let tx = user_db.begin(&authed).await?;
|
||||
let (dependency_job_uuid, mut tx) = push(
|
||||
tx,
|
||||
&w_id,
|
||||
@@ -256,15 +237,12 @@ async fn create_flow(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3",
|
||||
dependency_job_uuid,
|
||||
@@ -304,18 +282,15 @@ async fn check_schedule_conflict<'c>(
|
||||
async fn update_flow(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Json(nf): Json<NewFlow>,
|
||||
) -> Result<String> {
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
|
||||
let flow_path = flow_path.to_path();
|
||||
let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
|
||||
check_schedule_conflict(tx.transaction_mut(), &w_id, flow_path).await?;
|
||||
check_schedule_conflict(&mut tx, &w_id, flow_path).await?;
|
||||
|
||||
let schema = nf.schema.map(|x| x.0);
|
||||
let old_dep_job = sqlx::query_scalar!(
|
||||
@@ -328,7 +303,7 @@ async fn update_flow(
|
||||
let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
|
||||
sqlx::query!(
|
||||
"UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
|
||||
edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL WHERE path = $7 AND workspace_id = $8",
|
||||
edited_at = now(), schema = $6::text::json, dependency_job = NULL WHERE path = $7 AND workspace_id = $8",
|
||||
nf.path,
|
||||
nf.summary,
|
||||
nf.description,
|
||||
@@ -342,52 +317,44 @@ async fn update_flow(
|
||||
.await?;
|
||||
|
||||
if nf.path != flow_path {
|
||||
check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
|
||||
check_schedule_conflict(&mut tx, &w_id, &nf.path).await?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&authed, flow_path)?;
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut schedulables: Vec<Schedule> = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
let mut schedulables = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
nf.path,
|
||||
flow_path,
|
||||
w_id,
|
||||
)
|
||||
.fetch_all(&mut tx)
|
||||
.await?;
|
||||
|
||||
let schedule = sqlx::query_as!(Schedule,
|
||||
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
nf.path,
|
||||
flow_path,
|
||||
w_id,
|
||||
)
|
||||
.fetch_all(&mut tx)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
|
||||
let schedule = sqlx::query_as!(Schedule,
|
||||
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
nf.path,
|
||||
flow_path,
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
if let Some(schedule) = schedule {
|
||||
schedulables.push(schedule);
|
||||
}
|
||||
|
||||
if let Some(schedule) = schedule {
|
||||
schedulables.push(schedule);
|
||||
}
|
||||
for schedule in schedulables {
|
||||
clear_schedule(&mut tx, flow_path, true).await?;
|
||||
|
||||
for schedule in schedulables.into_iter() {
|
||||
clear_schedule(tx.transaction_mut(), &schedule.path, true).await?;
|
||||
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
|
||||
flow_path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
@@ -404,6 +371,7 @@ async fn update_flow(
|
||||
)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::UpdateFlow {
|
||||
@@ -413,6 +381,7 @@ async fn update_flow(
|
||||
},
|
||||
);
|
||||
|
||||
let tx = user_db.begin(&authed).await?;
|
||||
let (dependency_job_uuid, mut tx) = push(
|
||||
tx,
|
||||
&w_id,
|
||||
@@ -424,12 +393,10 @@ async fn update_flow(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
@@ -472,44 +439,6 @@ async fn get_flow_by_path(
|
||||
Ok(Json(flow))
|
||||
}
|
||||
|
||||
#[derive(Serialize, sqlx::FromRow)]
|
||||
pub struct FlowWDraft {
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub description: String,
|
||||
pub schema: Option<Schema>,
|
||||
pub value: serde_json::Value,
|
||||
pub extra_perms: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
async fn get_flow_by_path_w_draft(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<FlowWDraft> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let flow_o = sqlx::query_as::<_, FlowWDraft>(
|
||||
"SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft FROM flow
|
||||
LEFT JOIN draft ON
|
||||
flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow'
|
||||
WHERE flow.path = $1 AND flow.workspace_id = $2",
|
||||
)
|
||||
.bind(path)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let flow = not_found_if_none(flow_o, "Flow", path)?;
|
||||
Ok(Json(flow))
|
||||
}
|
||||
|
||||
async fn exists_flow_by_path(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
@@ -528,24 +457,17 @@ async fn exists_flow_by_path(
|
||||
Ok(Json(exists))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Archived {
|
||||
archived: Option<bool>,
|
||||
}
|
||||
|
||||
async fn archive_flow_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(archived): Json<Archived>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3",
|
||||
archived.archived.unwrap_or(true),
|
||||
"UPDATE flow SET archived = true WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
@@ -580,14 +502,6 @@ async fn delete_flow_by_path(
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM flow WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
@@ -636,6 +550,7 @@ mod tests {
|
||||
modules: vec![
|
||||
FlowModule {
|
||||
id: "a".to_string(),
|
||||
input_transforms: [].into(),
|
||||
value: FlowModuleValue::Script {
|
||||
path: "test".to_string(),
|
||||
input_transforms: [(
|
||||
@@ -653,13 +568,13 @@ mod tests {
|
||||
},
|
||||
FlowModule {
|
||||
id: "b".to_string(),
|
||||
input_transforms: HashMap::new(),
|
||||
value: FlowModuleValue::RawScript {
|
||||
input_transforms: HashMap::new(),
|
||||
content: "test".to_string(),
|
||||
language: scripts::ScriptLang::Deno,
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Some(StopAfterIf {
|
||||
expr: "foo = 'bar'".to_string(),
|
||||
@@ -672,6 +587,7 @@ mod tests {
|
||||
},
|
||||
FlowModule {
|
||||
id: "c".to_string(),
|
||||
input_transforms: HashMap::new(),
|
||||
value: FlowModuleValue::ForloopFlow {
|
||||
iterator: InputTransform::Static { value: serde_json::json!([1, 2, 3]) },
|
||||
modules: vec![],
|
||||
@@ -690,6 +606,7 @@ mod tests {
|
||||
],
|
||||
failure_module: Some(FlowModule {
|
||||
id: "d".to_string(),
|
||||
input_transforms: HashMap::new(),
|
||||
value: FlowModuleValue::Script {
|
||||
path: "test".to_string(),
|
||||
input_transforms: HashMap::new(),
|
||||
@@ -710,6 +627,7 @@ mod tests {
|
||||
"modules": [
|
||||
{
|
||||
"id": "a",
|
||||
"input_transforms": {},
|
||||
"value": {
|
||||
"input_transforms": {
|
||||
"test": {
|
||||
@@ -720,22 +638,29 @@ mod tests {
|
||||
"type": "script",
|
||||
"path": "test"
|
||||
},
|
||||
"stop_after_if": null,
|
||||
"summary": null
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"input_transforms": {},
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "rawscript",
|
||||
"content": "test",
|
||||
"lock": null,
|
||||
"path": null,
|
||||
"language": "deno"
|
||||
},
|
||||
"stop_after_if": {
|
||||
"expr": "foo = 'bar'",
|
||||
"skip_if_stopped": false
|
||||
}
|
||||
},
|
||||
"summary": null
|
||||
},
|
||||
{
|
||||
"id": "c",
|
||||
"input_transforms": {},
|
||||
"value": {
|
||||
"type": "forloopflow",
|
||||
"iterator": {
|
||||
@@ -753,11 +678,13 @@ mod tests {
|
||||
"stop_after_if": {
|
||||
"expr": "previous.isEmpty()",
|
||||
"skip_if_stopped": false,
|
||||
}
|
||||
},
|
||||
"summary": null
|
||||
}
|
||||
],
|
||||
"failure_module": {
|
||||
"id": "d",
|
||||
"input_transforms": {},
|
||||
"value": {
|
||||
"input_transforms": {},
|
||||
"type": "script",
|
||||
@@ -766,12 +693,38 @@ mod tests {
|
||||
"stop_after_if": {
|
||||
"expr": "previous.isEmpty()",
|
||||
"skip_if_stopped": false
|
||||
}
|
||||
},
|
||||
"summary": null
|
||||
}
|
||||
});
|
||||
assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_back_compat() {
|
||||
/* renamed input_transform -> input_transforms but should deserialize old name */
|
||||
let s = r#"
|
||||
{
|
||||
"value": {
|
||||
"type": "rawscript",
|
||||
"content": "def main(n): return",
|
||||
"language": "python3"
|
||||
},
|
||||
"input_transform": {
|
||||
"n": {
|
||||
"expr": "flow_input.iter.value",
|
||||
"type": "javascript"
|
||||
}
|
||||
}
|
||||
}
|
||||
"#;
|
||||
let module: FlowModule = serde_json::from_str(s).unwrap();
|
||||
assert_eq!(
|
||||
module.input_transforms["n"],
|
||||
InputTransform::Javascript { expr: "flow_input.iter.value".to_string() }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_serde() {
|
||||
assert_eq!(Retry::default(), serde_json::from_str(r#"{}"#).unwrap());
|
||||
|
||||
@@ -6,11 +6,9 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{AuthCache, Authed, Tokened},
|
||||
users::Authed,
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
};
|
||||
use axum::{
|
||||
@@ -18,17 +16,16 @@ use axum::{
|
||||
routing::{delete, get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use itertools::Itertools;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{self, to_anyhow, JsonResult, Result},
|
||||
error::{self, Error, JsonResult, Result},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, paginate, Pagination},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{FromRow, Postgres, Transaction};
|
||||
use sqlx::{query_scalar, FromRow, Postgres, Transaction};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -41,7 +38,7 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/delete/:name", delete(delete_folder))
|
||||
.route("/addowner/:name", post(add_owner))
|
||||
.route("/removeowner/:name", post(remove_owner))
|
||||
.route("/is_owner/*path", get(is_owner_api))
|
||||
.route("/is_owner/*path", get(is_owner))
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize, Clone)]
|
||||
@@ -140,28 +137,16 @@ async fn check_name_conflict<'c>(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref VALID_FOLDER_NAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
|
||||
}
|
||||
|
||||
async fn create_folder(
|
||||
authed: Authed,
|
||||
Tokened { token }: Tokened,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(cache): Extension<Arc<AuthCache>>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(ng): Json<NewFolder>,
|
||||
) -> Result<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
if !VALID_FOLDER_NAME.is_match(&ng.name) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Folder name can only contain alphanumeric characters, underscores"
|
||||
)));
|
||||
}
|
||||
check_name_conflict(&mut tx, &w_id, &ng.name).await?;
|
||||
cache.invalidate(&w_id, token).await;
|
||||
let owner = username_to_permissioned_as(&authed.username);
|
||||
let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
|
||||
|
||||
@@ -219,29 +204,47 @@ async fn create_folder(
|
||||
Ok(format!("Created folder {}", ng.name))
|
||||
}
|
||||
|
||||
pub async fn is_owner_api(
|
||||
authed: Authed,
|
||||
Path((_w_id, name)): Path<(String, String)>,
|
||||
pub async fn is_owner(
|
||||
Authed { username, is_admin, groups, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, name)): Path<(String, String)>,
|
||||
) -> JsonResult<bool> {
|
||||
Ok(Json(is_owner(&authed, &name)))
|
||||
}
|
||||
|
||||
pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
|
||||
if *is_admin {
|
||||
true
|
||||
if is_admin {
|
||||
Ok(Json(true))
|
||||
} else {
|
||||
folders.into_iter().any(|x| x.0 == name && x.2)
|
||||
Ok(Json(
|
||||
require_is_owner(&name, &username, &groups, &w_id, &db)
|
||||
.await
|
||||
.is_ok(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
|
||||
if is_owner(authed, name) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(windmill_common::error::Error::NotAuthorized(format!(
|
||||
"You are not owner of the folder {}",
|
||||
name
|
||||
pub async fn require_is_owner(
|
||||
folder_name: &str,
|
||||
username: &str,
|
||||
groups: &Vec<String>,
|
||||
w_id: &str,
|
||||
db: &DB,
|
||||
) -> Result<()> {
|
||||
let is_owner = query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
|
||||
SELECT 1 FROM folder, unnest(folder.owners) as o
|
||||
WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
|
||||
username,
|
||||
folder_name,
|
||||
groups,
|
||||
w_id,
|
||||
).fetch_one(db)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
if !is_owner {
|
||||
Err(Error::BadRequest(format!(
|
||||
"{} is not an owner of {} and hence is not authorized to perform this operation",
|
||||
username, folder_name
|
||||
)))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,26 +262,13 @@ async fn update_folder(
|
||||
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
|
||||
|
||||
if let Some(display_name) = ng.display_name {
|
||||
sqlb.set("display_name", "?".bind(&display_name));
|
||||
sqlb.set("display_name", display_name);
|
||||
}
|
||||
if let Some(owners) = ng.owners {
|
||||
sqlb.set(
|
||||
"owners",
|
||||
"?".bind(&format!(
|
||||
"{{{}}}",
|
||||
owners
|
||||
.iter()
|
||||
.map(|x| format!("\"{x}\""))
|
||||
.collect::<Vec<_>>()
|
||||
.join(","),
|
||||
)),
|
||||
);
|
||||
sqlb.set_str("owners", format!("{{{}}}", owners.into_iter().join(",")));
|
||||
}
|
||||
if let Some(extra_perms) = ng.extra_perms {
|
||||
sqlb.set(
|
||||
"extra_perms",
|
||||
"?".bind(&serde_json::to_string(&extra_perms).map_err(to_anyhow)?),
|
||||
);
|
||||
sqlb.set_str("extra_perms", extra_perms.to_string());
|
||||
}
|
||||
|
||||
sqlb.returning("*");
|
||||
@@ -473,6 +463,7 @@ async fn delete_folder(
|
||||
|
||||
async fn add_owner(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, name)): Path<(String, String)>,
|
||||
@@ -481,7 +472,9 @@ async fn add_owner(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
|
||||
require_is_owner(&authed, &name)?;
|
||||
if !authed.is_admin {
|
||||
require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
|
||||
@@ -517,14 +510,14 @@ pub async fn get_folders_for_user(
|
||||
username: &str,
|
||||
groups: &[String],
|
||||
db: &DB,
|
||||
) -> Result<Vec<(String, bool, bool)>> {
|
||||
) -> Result<Vec<(String, bool)>> {
|
||||
let mut perms = groups
|
||||
.into_iter()
|
||||
.map(|x| format!("g/{}", x))
|
||||
.collect::<Vec<_>>();
|
||||
perms.insert(0, format!("u/{}", username));
|
||||
let folders = sqlx::query!(
|
||||
"SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t WHERE value::boolean IS true)) as write, $1 && owners::text[] as owner FROM folder
|
||||
"SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t WHERE value::boolean IS true)) as write FROM folder
|
||||
WHERE extra_perms ?| $1 AND workspace_id = $2",
|
||||
&perms[..],
|
||||
w_id,
|
||||
@@ -532,7 +525,7 @@ pub async fn get_folders_for_user(
|
||||
.fetch_all(db)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|x| (x.name, x.write.unwrap_or(false), x.owner.unwrap_or(false)))
|
||||
.map(|x| (x.name, x.write.unwrap_or(false)))
|
||||
.collect();
|
||||
|
||||
Ok(folders)
|
||||
@@ -540,6 +533,7 @@ pub async fn get_folders_for_user(
|
||||
|
||||
async fn remove_owner(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, name)): Path<(String, String)>,
|
||||
@@ -548,7 +542,9 @@ async fn remove_owner(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
|
||||
require_is_owner(&authed, &name)?;
|
||||
if !authed.is_admin {
|
||||
require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",
|
||||
|
||||
@@ -43,29 +43,25 @@ async fn add_granular_acl(
|
||||
Json(GranularAcl { owner, write }): Json<GranularAcl>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
|
||||
let (kind, path) = path
|
||||
.split_once('/')
|
||||
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
if !authed.is_admin {
|
||||
if kind == "folder" {
|
||||
crate::folders::require_is_owner(&path, &authed.username, &authed.groups, &w_id, &db)
|
||||
.await?;
|
||||
} else if kind == "group_" {
|
||||
} else {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
}
|
||||
let identifier = if kind == "group_" || kind == "folder" {
|
||||
"name"
|
||||
} else {
|
||||
"path"
|
||||
};
|
||||
|
||||
if !authed.is_admin {
|
||||
if kind == "folder" {
|
||||
crate::folders::require_is_owner(&authed, path)?;
|
||||
} else if kind == "group_" {
|
||||
crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
|
||||
.await?;
|
||||
} else {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
}
|
||||
|
||||
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
|
||||
"UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
|
||||
true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
|
||||
@@ -90,22 +86,12 @@ async fn remove_granular_acl(
|
||||
Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
let (kind, path) = path
|
||||
.split_once('/')
|
||||
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
|
||||
|
||||
if !authed.is_admin {
|
||||
if kind == "folder" {
|
||||
crate::folders::require_is_owner(&authed, path)?;
|
||||
} else if kind == "group_" {
|
||||
crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
|
||||
.await?;
|
||||
} else {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let identifier = if kind == "group_" || kind == "folder" {
|
||||
@@ -113,11 +99,6 @@ async fn remove_granular_acl(
|
||||
} else {
|
||||
"path"
|
||||
};
|
||||
|
||||
if identifier == "path" {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
|
||||
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
|
||||
"UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
|
||||
workspace_id = $3 RETURNING extra_perms"
|
||||
|
||||
@@ -1,281 +0,0 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2022
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::{db::UserDB, jobs::CompletedJob, users::Authed};
|
||||
use axum::{
|
||||
extract::{Path, Query},
|
||||
routing::{get, post},
|
||||
Extension, Json, Router,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use sqlx::types::Uuid;
|
||||
use std::{
|
||||
fmt::{Display, Formatter},
|
||||
vec,
|
||||
};
|
||||
use windmill_common::{
|
||||
error::JsonResult,
|
||||
jobs::JobKind,
|
||||
scripts::to_i64,
|
||||
utils::{paginate, Pagination},
|
||||
};
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route("/history", get(get_input_history))
|
||||
.route("/list", get(list_saved_inputs))
|
||||
.route("/create", post(create_input))
|
||||
.route("/update", post(update_input))
|
||||
.route("/delete/:id", post(delete_input))
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize, Deserialize)]
|
||||
pub struct InputRow {
|
||||
pub id: Uuid,
|
||||
pub workspace_id: String,
|
||||
pub runnable_id: String,
|
||||
pub runnable_type: RunnableType,
|
||||
pub name: String,
|
||||
pub args: Value,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub created_by: String,
|
||||
pub is_public: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, sqlx::Type)]
|
||||
#[sqlx(type_name = "runnable_type")]
|
||||
pub enum RunnableType {
|
||||
ScriptHash,
|
||||
ScriptPath,
|
||||
FlowPath,
|
||||
}
|
||||
|
||||
impl Display for RunnableType {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
RunnableType::ScriptHash => write!(f, "ScriptHash"),
|
||||
RunnableType::ScriptPath => write!(f, "ScriptPath"),
|
||||
RunnableType::FlowPath => write!(f, "FlowPath"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RunnableType {
|
||||
fn job_kind(&self) -> JobKind {
|
||||
match self {
|
||||
RunnableType::ScriptHash => JobKind::Script,
|
||||
RunnableType::ScriptPath => JobKind::Script,
|
||||
RunnableType::FlowPath => JobKind::Flow,
|
||||
}
|
||||
}
|
||||
|
||||
fn column_name(&self) -> &'static str {
|
||||
match self {
|
||||
RunnableType::ScriptHash => "script_hash",
|
||||
RunnableType::ScriptPath => "script_path",
|
||||
RunnableType::FlowPath => "script_path",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct RunnableParams {
|
||||
pub runnable_id: String,
|
||||
pub runnable_type: RunnableType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct Input {
|
||||
id: Uuid,
|
||||
name: String,
|
||||
created_at: chrono::DateTime<chrono::Utc>,
|
||||
args: serde_json::Value,
|
||||
created_by: String,
|
||||
is_public: bool,
|
||||
}
|
||||
|
||||
async fn get_input_history(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(pagination): Query<Pagination>,
|
||||
Query(r): Query<RunnableParams>,
|
||||
) -> JsonResult<Vec<Input>> {
|
||||
let (per_page, offset) = paginate(pagination);
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let sql = &format!(
|
||||
"select * from (select distinct on (args) * from completed_job \
|
||||
where {} = $1 and job_kind = $2 and workspace_id = $3 \
|
||||
order by args, started_at desc) t ORDER BY started_at desc limit $4 offset $5",
|
||||
r.runnable_type.column_name()
|
||||
);
|
||||
|
||||
let query = sqlx::query_as::<_, CompletedJob>(sql);
|
||||
|
||||
let query = match r.runnable_type {
|
||||
RunnableType::ScriptHash => query.bind(to_i64(&r.runnable_id)?),
|
||||
_ => query.bind(&r.runnable_id),
|
||||
};
|
||||
|
||||
let rows = query
|
||||
.bind(r.runnable_type.job_kind())
|
||||
.bind(&w_id)
|
||||
.bind(per_page as i32)
|
||||
.bind(offset as i32)
|
||||
.fetch_all(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
let mut inputs = vec![];
|
||||
|
||||
for row in rows {
|
||||
inputs.push(Input {
|
||||
id: row.id,
|
||||
name: format!(
|
||||
"{} {}",
|
||||
row.created_at.format("%H:%M %-d/%-m"),
|
||||
row.created_by
|
||||
),
|
||||
created_at: row.created_at,
|
||||
args: row.args.unwrap_or(serde_json::json!({})),
|
||||
created_by: row.created_by,
|
||||
is_public: true,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Json(inputs))
|
||||
}
|
||||
|
||||
async fn list_saved_inputs(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(pagination): Query<Pagination>,
|
||||
Query(r): Query<RunnableParams>,
|
||||
) -> JsonResult<Vec<Input>> {
|
||||
let (per_page, offset) = paginate(pagination);
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let rows = sqlx::query_as::<_, InputRow>(
|
||||
"select * from input \
|
||||
where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
|
||||
and is_public IS true OR created_by = $4 \
|
||||
order by created_at desc limit $5 offset $6",
|
||||
)
|
||||
.bind(&r.runnable_id)
|
||||
.bind(&r.runnable_type)
|
||||
.bind(&w_id)
|
||||
.bind(&authed.username)
|
||||
.bind(per_page as i32)
|
||||
.bind(offset as i32)
|
||||
.fetch_all(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
let mut inputs: Vec<Input> = Vec::new();
|
||||
|
||||
for row in rows {
|
||||
inputs.push(Input {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
args: row.args,
|
||||
created_by: row.created_by,
|
||||
created_at: row.created_at,
|
||||
is_public: row.is_public,
|
||||
})
|
||||
}
|
||||
|
||||
Ok(Json(inputs))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct CreateInput {
|
||||
name: String,
|
||||
args: serde_json::Value,
|
||||
}
|
||||
|
||||
async fn create_input(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(r): Query<RunnableParams>,
|
||||
Json(input): Json<CreateInput>,
|
||||
) -> JsonResult<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let id = Uuid::new_v4();
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO input (id, workspace_id, runnable_id, runnable_type, name, args, created_by) VALUES ($1, $2, $3, $4, $5, $6, $7)",
|
||||
)
|
||||
.bind(&id)
|
||||
.bind(&w_id)
|
||||
.bind(&r.runnable_id)
|
||||
.bind(&r.runnable_type)
|
||||
.bind(&input.name)
|
||||
.bind(&input.args)
|
||||
.bind(&authed.username)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(Json(id.to_string()))
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct UpdateInput {
|
||||
id: Uuid,
|
||||
name: String,
|
||||
is_public: bool,
|
||||
}
|
||||
|
||||
async fn update_input(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(input): Json<UpdateInput>,
|
||||
) -> JsonResult<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query("UPDATE input SET name = $1, is_public = $2 WHERE id = $3 and workspace_id = $4")
|
||||
.bind(&input.name)
|
||||
.bind(&input.is_public)
|
||||
.bind(&input.id)
|
||||
.bind(&w_id)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(Json(input.id.to_string()))
|
||||
}
|
||||
|
||||
async fn delete_input(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, i_id)): Path<(String, Uuid)>,
|
||||
) -> JsonResult<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query("DELETE FROM input WHERE id = $1 and workspace_id = $2")
|
||||
.bind(&i_id)
|
||||
.bind(&w_id)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(Json(i_id.to_string()))
|
||||
}
|
||||
@@ -6,12 +6,6 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{require_owner_of_path, Authed, OptAuthed},
|
||||
variables::get_workspace_key,
|
||||
BASE_URL,
|
||||
};
|
||||
use anyhow::Context;
|
||||
use axum::{
|
||||
extract::{FromRequest, Json, Path, Query},
|
||||
@@ -31,13 +25,19 @@ use windmill_common::{
|
||||
error::{self, to_anyhow, Error},
|
||||
flow_status::{Approval, FlowStatus, FlowStatusModule},
|
||||
flows::FlowValue,
|
||||
jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
|
||||
oauth2::HmacSha256,
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::{get_queued_job, push, QueueTransaction};
|
||||
use windmill_queue::{get_queued_job, push, JobKind, JobPayload, QueuedJob, RawCode};
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{require_owner_of_path, Authed},
|
||||
variables::get_workspace_key,
|
||||
BASE_URL,
|
||||
};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -47,10 +47,6 @@ pub fn workspaced_service() -> Router {
|
||||
"/run_wait_result/p/*script_path",
|
||||
post(run_wait_result_job_by_path),
|
||||
)
|
||||
.route(
|
||||
"/run_wait_result/p/*script_path",
|
||||
get(run_wait_result_job_by_path_get),
|
||||
)
|
||||
.route(
|
||||
"/run_wait_result/h/:hash",
|
||||
post(run_wait_result_job_by_hash),
|
||||
@@ -64,7 +60,7 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/run/preview_flow", post(run_preview_flow_job))
|
||||
.route("/list", get(list_jobs))
|
||||
.route("/queue/list", get(list_queue_jobs))
|
||||
.route("/queue/count", get(count_queue_jobs))
|
||||
.route("/queue/cancel/:id", post(cancel_job_api))
|
||||
.route("/completed/list", get(list_completed_jobs))
|
||||
.route("/completed/get/:id", get(get_completed_job))
|
||||
.route("/completed/get_result/:id", get(get_completed_job_result))
|
||||
@@ -102,39 +98,32 @@ pub fn global_service() -> Router {
|
||||
)
|
||||
.route("/get/:id", get(get_job))
|
||||
.route("/getupdate/:id", get(get_job_update))
|
||||
.route("/queue/cancel/:id", post(cancel_job_api))
|
||||
.route("/queue/force_cancel/:id", post(force_cancel))
|
||||
}
|
||||
|
||||
async fn get_result_by_id(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>,
|
||||
Query(ResultByIdQuery { skip_direct }): Query<ResultByIdQuery>,
|
||||
Path((w_id, flow_id, node_id)): Path<(String, String, String)>,
|
||||
) -> windmill_common::error::JsonResult<serde_json::Value> {
|
||||
let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id).await?;
|
||||
let res = windmill_queue::get_result_by_id(db, skip_direct, w_id, flow_id, node_id).await?;
|
||||
Ok(Json(res))
|
||||
}
|
||||
|
||||
async fn cancel_job_api(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
OptAuthed(opt_authed): OptAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
Json(CancelJob { reason }): Json<CancelJob>,
|
||||
) -> error::Result<String> {
|
||||
let tx = db.begin().await?;
|
||||
|
||||
let username = match opt_authed {
|
||||
Some(authed) => authed.username,
|
||||
None => "anonymous".to_string(),
|
||||
};
|
||||
let tx = user_db.begin(&authed).await?;
|
||||
|
||||
let (mut tx, job_option) =
|
||||
windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, false).await?;
|
||||
windmill_queue::cancel_job(&authed.username, reason, id, &w_id, tx).await?;
|
||||
|
||||
if let Some(id) = job_option {
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&username,
|
||||
&authed.username,
|
||||
"jobs.cancel",
|
||||
ActionKind::Delete,
|
||||
&w_id,
|
||||
@@ -157,49 +146,6 @@ async fn cancel_job_api(
|
||||
}
|
||||
}
|
||||
|
||||
async fn force_cancel(
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
OptAuthed(opt_authed): OptAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
Json(CancelJob { reason }): Json<CancelJob>,
|
||||
) -> error::Result<String> {
|
||||
let tx = db.begin().await?;
|
||||
|
||||
let username = match opt_authed {
|
||||
Some(authed) => authed.username,
|
||||
None => "anonymous".to_string(),
|
||||
};
|
||||
|
||||
let (mut tx, job_option) =
|
||||
windmill_queue::cancel_job(&username, reason, id, &w_id, tx, rsmq, true).await?;
|
||||
|
||||
if let Some(id) = job_option {
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&username,
|
||||
"jobs.force_cancel",
|
||||
ActionKind::Delete,
|
||||
&w_id,
|
||||
Some(&id.to_string()),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok(id.to_string())
|
||||
} else {
|
||||
let (job_o, tx) = get_job_by_id(tx, &w_id, id).await?;
|
||||
tx.commit().await?;
|
||||
let err = match job_o {
|
||||
Some(Job::CompletedJob(_)) => {
|
||||
return Ok(format!("queued job id {} is already completed", id))
|
||||
}
|
||||
_ => error::Error::NotFound(format!("queued job id {} does not exist", id)),
|
||||
};
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_path_for_hash<'c>(
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
@@ -220,26 +166,6 @@ pub async fn get_path_for_hash<'c>(
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub async fn get_path_and_tag_for_hash<'c>(
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
hash: i64,
|
||||
) -> error::Result<(String, Option<String>)> {
|
||||
let script = sqlx::query!(
|
||||
"select path, tag from script where hash = $1 AND workspace_id = $2",
|
||||
hash,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
Error::InternalErr(format!(
|
||||
"querying getting path for hash {hash} in {w_id}: {e}"
|
||||
))
|
||||
})?;
|
||||
Ok((script.path, script.tag))
|
||||
}
|
||||
|
||||
async fn get_job(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
@@ -251,6 +177,11 @@ async fn get_job(
|
||||
Ok(Json(job))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ResultByIdQuery {
|
||||
pub skip_direct: bool,
|
||||
}
|
||||
|
||||
pub async fn get_job_by_id<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
@@ -326,7 +257,6 @@ pub struct CompletedJob {
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
@@ -337,7 +267,6 @@ pub struct RunJobQuery {
|
||||
include_header: Option<String>,
|
||||
invisible_to_owner: Option<bool>,
|
||||
queue_limit: Option<i64>,
|
||||
payload: Option<String>,
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
@@ -366,52 +295,41 @@ impl RunJobQuery {
|
||||
fn add_include_headers(
|
||||
&self,
|
||||
headers: HeaderMap,
|
||||
args: serde_json::Map<String, serde_json::Value>,
|
||||
mut args: serde_json::Map<String, serde_json::Value>,
|
||||
) -> serde_json::Map<String, serde_json::Value> {
|
||||
return add_include_headers(&self.include_header, headers, args);
|
||||
let whitelist = self
|
||||
.include_header
|
||||
.as_ref()
|
||||
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
whitelist
|
||||
.iter()
|
||||
.chain(INCLUDE_HEADERS.iter())
|
||||
.for_each(|h| {
|
||||
if let Some(v) = headers.get(h) {
|
||||
args.insert(
|
||||
h.to_string().to_lowercase().replace('-', "_"),
|
||||
serde_json::Value::String(v.to_str().unwrap().to_string()),
|
||||
);
|
||||
}
|
||||
});
|
||||
args
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_include_headers(
|
||||
include_header: &Option<String>,
|
||||
headers: HeaderMap,
|
||||
mut args: serde_json::Map<String, serde_json::Value>,
|
||||
) -> serde_json::Map<String, serde_json::Value> {
|
||||
let whitelist = include_header
|
||||
.as_ref()
|
||||
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
|
||||
.unwrap_or_default();
|
||||
|
||||
whitelist
|
||||
.iter()
|
||||
.chain(INCLUDE_HEADERS.iter())
|
||||
.for_each(|h| {
|
||||
if let Some(v) = headers.get(h) {
|
||||
args.insert(
|
||||
h.to_string().to_lowercase().replace('-', "_"),
|
||||
serde_json::Value::String(v.to_str().unwrap().to_string()),
|
||||
);
|
||||
}
|
||||
});
|
||||
args
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ListQueueQuery {
|
||||
pub script_path_start: Option<String>,
|
||||
pub script_path_exact: Option<String>,
|
||||
pub script_hash: Option<String>,
|
||||
pub created_by: Option<String>,
|
||||
pub started_before: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub started_after: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub created_before: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub created_after: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub running: Option<bool>,
|
||||
pub parent_job: Option<String>,
|
||||
pub order_desc: Option<bool>,
|
||||
pub job_kinds: Option<String>,
|
||||
pub suspended: Option<bool>,
|
||||
// filter by matching a subset of the args using base64 encoded json subset
|
||||
pub args: Option<String>,
|
||||
pub tag: Option<String>,
|
||||
}
|
||||
|
||||
fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
|
||||
@@ -434,20 +352,17 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
|
||||
if let Some(cb) = &lq.created_by {
|
||||
sqlb.and_where_eq("created_by", "?".bind(cb));
|
||||
}
|
||||
if let Some(t) = &lq.tag {
|
||||
sqlb.and_where_eq("tag", "?".bind(t));
|
||||
}
|
||||
if let Some(r) = &lq.running {
|
||||
sqlb.and_where_eq("running", &r);
|
||||
}
|
||||
if let Some(pj) = &lq.parent_job {
|
||||
sqlb.and_where_eq("parent_job", "?".bind(pj));
|
||||
}
|
||||
if let Some(dt) = &lq.started_before {
|
||||
sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
if let Some(dt) = &lq.created_before {
|
||||
sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
}
|
||||
if let Some(dt) = &lq.started_after {
|
||||
sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
if let Some(dt) = &lq.created_after {
|
||||
sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
}
|
||||
|
||||
if let Some(s) = &lq.suspended {
|
||||
@@ -457,7 +372,6 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
|
||||
sqlb.and_where_eq("suspend", 0);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(jk) = &lq.job_kinds {
|
||||
sqlb.and_where_in(
|
||||
"job_kind",
|
||||
@@ -465,10 +379,6 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(args) = &lq.args {
|
||||
sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
|
||||
}
|
||||
|
||||
sqlb
|
||||
}
|
||||
|
||||
@@ -524,26 +434,6 @@ async fn list_queue_jobs(
|
||||
Ok(Json(jobs))
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug, FromRow)]
|
||||
struct QueueStats {
|
||||
database_length: i64,
|
||||
}
|
||||
|
||||
async fn count_queue_jobs(
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
) -> error::JsonResult<QueueStats> {
|
||||
Ok(Json(
|
||||
sqlx::query_as!(
|
||||
QueueStats,
|
||||
"SELECT coalesce(COUNT(*), 0) as \"database_length!\" FROM queue WHERE workspace_id = $1",
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?,
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_jobs(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -561,15 +451,13 @@ async fn list_jobs(
|
||||
script_path_exact: lq.script_path_exact,
|
||||
script_hash: lq.script_hash,
|
||||
created_by: lq.created_by,
|
||||
started_before: lq.started_before,
|
||||
started_after: lq.started_after,
|
||||
created_before: lq.created_before,
|
||||
created_after: lq.created_after,
|
||||
running: None,
|
||||
parent_job: lq.parent_job,
|
||||
order_desc: Some(true),
|
||||
job_kinds: lq.job_kinds,
|
||||
suspended: lq.suspended,
|
||||
args: lq.args,
|
||||
tag: lq.tag,
|
||||
},
|
||||
&[
|
||||
"'QueuedJob' as typ",
|
||||
@@ -599,7 +487,6 @@ async fn list_jobs(
|
||||
"visible_to_owner",
|
||||
"suspend",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
],
|
||||
);
|
||||
let sqlc = list_completed_jobs_query(
|
||||
@@ -635,7 +522,6 @@ async fn list_jobs(
|
||||
"visible_to_owner",
|
||||
"null as suspend",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
],
|
||||
);
|
||||
let sql = format!(
|
||||
@@ -654,7 +540,7 @@ async fn list_jobs(
|
||||
pub async fn resume_suspended_flow_as_owner(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((_w_id, flow_id)): Path<(String, Uuid)>,
|
||||
Path((w_id, flow_id)): Path<(String, Uuid)>,
|
||||
QueryOrBody(value): QueryOrBody<serde_json::Value>,
|
||||
) -> error::Result<StatusCode> {
|
||||
let value = value.unwrap_or(serde_json::Value::Null);
|
||||
@@ -662,11 +548,16 @@ pub async fn resume_suspended_flow_as_owner(
|
||||
|
||||
let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?;
|
||||
|
||||
require_owner_of_path(
|
||||
&authed,
|
||||
&flow.script_path.clone().unwrap_or_else(|| String::new()),
|
||||
)?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(
|
||||
&w_id,
|
||||
&authed.username,
|
||||
&authed.groups,
|
||||
&flow.script_path.clone().unwrap_or_else(|| String::new()),
|
||||
&db,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
insert_resume_job(0, job_id, &flow, value, Some(authed.username), &mut tx).await?;
|
||||
|
||||
resume_immediately_if_relevant(flow, job_id, &mut tx).await?;
|
||||
@@ -839,7 +730,6 @@ async fn get_suspended_flow_info<'c>(
|
||||
pub async fn cancel_suspended_job(
|
||||
/* unauthed */
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, job, resume_id, secret)): Path<(String, Uuid, u32, String)>,
|
||||
Query(approver): Query<QueryApprover>,
|
||||
) -> error::Result<String> {
|
||||
@@ -862,8 +752,6 @@ pub async fn cancel_suspended_job(
|
||||
parent_flow,
|
||||
&w_id,
|
||||
tx,
|
||||
rsmq,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
if job.is_some() {
|
||||
@@ -1112,7 +1000,6 @@ struct UnifiedJob {
|
||||
visible_to_owner: bool,
|
||||
suspend: Option<i32>,
|
||||
mem_peak: Option<i32>,
|
||||
tag: String,
|
||||
}
|
||||
|
||||
impl From<UnifiedJob> for Job {
|
||||
@@ -1148,7 +1035,6 @@ impl From<UnifiedJob> for Job {
|
||||
email: uj.email,
|
||||
visible_to_owner: uj.visible_to_owner,
|
||||
mem_peak: uj.mem_peak,
|
||||
tag: uj.tag,
|
||||
}),
|
||||
"QueuedJob" => Job::QueuedJob(QueuedJob {
|
||||
workspace_id: uj.workspace_id,
|
||||
@@ -1182,9 +1068,6 @@ impl From<UnifiedJob> for Job {
|
||||
visible_to_owner: uj.visible_to_owner,
|
||||
suspend: uj.suspend,
|
||||
mem_peak: uj.mem_peak,
|
||||
root_job: None,
|
||||
leaf_jobs: None,
|
||||
tag: uj.tag,
|
||||
}),
|
||||
t => panic!("job type {} not valid", t),
|
||||
}
|
||||
@@ -1201,7 +1084,6 @@ struct Preview {
|
||||
path: Option<String>,
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
language: ScriptLang,
|
||||
tag: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -1248,27 +1130,26 @@ where
|
||||
struct InPayload {
|
||||
payload: Option<String>,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
|
||||
let vec = base64::engine::general_purpose::URL_SAFE
|
||||
.decode(t)
|
||||
.context("invalid base64")?;
|
||||
serde_json::from_slice(vec.as_slice()).context("invalid json")
|
||||
fn decode_payload<D: DeserializeOwned, T: AsRef<[u8]>>(t: T) -> anyhow::Result<D> {
|
||||
let vec = base64::engine::general_purpose::URL_SAFE
|
||||
.decode(t)
|
||||
.context("invalid base64")?;
|
||||
serde_json::from_slice(vec.as_slice()).context("invalid json")
|
||||
}
|
||||
}
|
||||
}
|
||||
pub async fn run_flow_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let flow_path = flow_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1282,12 +1163,10 @@ pub async fn run_flow_by_path(
|
||||
scheduled_for,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1297,17 +1176,15 @@ pub async fn run_flow_by_path(
|
||||
pub async fn run_job_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let script_path = script_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let (job_payload, tag) =
|
||||
script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1321,12 +1198,10 @@ pub async fn run_job_by_path(
|
||||
scheduled_for,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1376,13 +1251,14 @@ async fn run_wait_result<T>(
|
||||
uuid: Uuid,
|
||||
Path((w_id, _)): Path<(String, T)>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
let mut result;
|
||||
let timeout_ms = if timeout <= 0 {
|
||||
2000
|
||||
let mut result = None;
|
||||
let iters = if timeout <= 0 {
|
||||
20
|
||||
} else if timeout <= 1 {
|
||||
timeout * 10
|
||||
} else {
|
||||
(timeout * 1000) as u64
|
||||
10 + ((timeout - 1) * 2)
|
||||
};
|
||||
|
||||
let mut g = Guard {
|
||||
done: false,
|
||||
id: uuid,
|
||||
@@ -1390,10 +1266,7 @@ async fn run_wait_result<T>(
|
||||
db: user_db.clone(),
|
||||
authed: authed.clone(),
|
||||
};
|
||||
|
||||
let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
|
||||
let mut accumulated_delay = 0 as u64;
|
||||
loop {
|
||||
for i in 0..iters {
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
result = sqlx::query_scalar!(
|
||||
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
@@ -1408,16 +1281,7 @@ async fn run_wait_result<T>(
|
||||
if result.is_some() {
|
||||
break;
|
||||
}
|
||||
|
||||
let delay = if accumulated_delay <= fast_poll_duration {
|
||||
*WAIT_RESULT_FAST_POLL_INTERVAL_MS
|
||||
} else {
|
||||
*WAIT_RESULT_SLOW_POLL_INTERVAL_MS
|
||||
};
|
||||
accumulated_delay += delay;
|
||||
if accumulated_delay > timeout_ms {
|
||||
break;
|
||||
};
|
||||
let delay = if i < 10 { 100 } else { 500 };
|
||||
tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
|
||||
}
|
||||
if let Some(result) = result {
|
||||
@@ -1457,80 +1321,10 @@ lazy_static::lazy_static! {
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(20);
|
||||
pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(50);
|
||||
pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(2);
|
||||
pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(200);
|
||||
}
|
||||
|
||||
pub async fn run_wait_result_job_by_path_get(
|
||||
authed: Authed,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
let payload_r = run_query
|
||||
.payload
|
||||
.map(decode_payload)
|
||||
.map(|x| x.map_err(|e| Error::InternalErr(e.to_string())));
|
||||
|
||||
let args = if let Some(payload) = payload_r {
|
||||
payload?
|
||||
} else {
|
||||
serde_json::Map::new()
|
||||
};
|
||||
|
||||
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
|
||||
let script_path = script_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let (job_payload, tag) =
|
||||
script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
|
||||
let (uuid, tx) = push(
|
||||
tx,
|
||||
&w_id,
|
||||
job_payload,
|
||||
args,
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
None,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
run_wait_result(
|
||||
authed,
|
||||
Extension(user_db),
|
||||
*TIMEOUT_WAIT_RESULT,
|
||||
uuid,
|
||||
Path((w_id, script_path)),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn run_wait_result_job_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
@@ -1539,9 +1333,9 @@ pub async fn run_wait_result_job_by_path(
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
|
||||
let script_path = script_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let (job_payload, tag) =
|
||||
script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
let job_payload = script_path_to_payload(script_path, &mut tx, &w_id).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
@@ -1553,15 +1347,13 @@ pub async fn run_wait_result_job_by_path(
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
scheduled_for,
|
||||
None,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1579,7 +1371,6 @@ pub async fn run_wait_result_job_by_path(
|
||||
pub async fn run_wait_result_job_by_hash(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
@@ -1589,9 +1380,9 @@ pub async fn run_wait_result_job_by_hash(
|
||||
check_queue_too_long(db, run_query.queue_limit).await?;
|
||||
|
||||
let hash = script_hash.0;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
|
||||
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1602,15 +1393,13 @@ pub async fn run_wait_result_job_by_hash(
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
scheduled_for,
|
||||
None,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1628,7 +1417,6 @@ pub async fn run_wait_result_job_by_hash(
|
||||
pub async fn run_wait_result_flow_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
@@ -1638,8 +1426,8 @@ pub async fn run_wait_result_flow_by_path(
|
||||
check_queue_too_long(db, run_query.queue_limit).await?;
|
||||
|
||||
let flow_path = flow_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1653,14 +1441,13 @@ pub async fn run_wait_result_flow_by_path(
|
||||
scheduled_for,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
run_wait_result(
|
||||
@@ -1673,17 +1460,31 @@ pub async fn run_wait_result_flow_by_path(
|
||||
.await
|
||||
}
|
||||
|
||||
// a similar function exists on the worker
|
||||
pub async fn script_path_to_payload<'c>(
|
||||
script_path: &str,
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &String,
|
||||
) -> std::result::Result<JobPayload, Error> {
|
||||
let job_payload = if script_path.starts_with("hub/") {
|
||||
JobPayload::ScriptHub { path: script_path.to_owned() }
|
||||
} else {
|
||||
let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
|
||||
JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
|
||||
};
|
||||
Ok(job_payload)
|
||||
}
|
||||
|
||||
async fn run_preview_job(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(preview): Json<Preview>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1702,30 +1503,26 @@ async fn run_preview_job(
|
||||
scheduled_for,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
preview.tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, uuid.to_string()))
|
||||
}
|
||||
|
||||
async fn run_preview_flow_job(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(raw_flow): Json<PreviewFlow>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1739,32 +1536,28 @@ async fn run_preview_flow_job(
|
||||
scheduled_for,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, uuid.to_string()))
|
||||
}
|
||||
|
||||
pub async fn run_job_by_hash(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let hash = script_hash.0;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let path = get_path_for_hash(&mut tx, &w_id, hash).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(&mut tx).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
@@ -1778,16 +1571,13 @@ pub async fn run_job_by_hash(
|
||||
scheduled_for,
|
||||
None,
|
||||
run_query.parent_job,
|
||||
run_query.parent_job,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, uuid.to_string()))
|
||||
}
|
||||
|
||||
@@ -1874,9 +1664,6 @@ fn list_completed_jobs_query(
|
||||
if let Some(h) = &lq.script_hash {
|
||||
sqlb.and_where_eq("script_hash", "?".bind(h));
|
||||
}
|
||||
if let Some(t) = &lq.tag {
|
||||
sqlb.and_where_eq("tag", "?".bind(t));
|
||||
}
|
||||
if let Some(cb) = &lq.created_by {
|
||||
sqlb.and_where_eq("created_by", "?".bind(cb));
|
||||
}
|
||||
@@ -1886,11 +1673,11 @@ fn list_completed_jobs_query(
|
||||
if let Some(pj) = &lq.parent_job {
|
||||
sqlb.and_where_eq("parent_job", "?".bind(pj));
|
||||
}
|
||||
if let Some(dt) = &lq.started_before {
|
||||
sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
if let Some(dt) = &lq.created_before {
|
||||
sqlb.and_where_lt("created_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
}
|
||||
if let Some(dt) = &lq.started_after {
|
||||
sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
if let Some(dt) = &lq.created_after {
|
||||
sqlb.and_where_gt("created_at", format!("to_timestamp({})", dt.timestamp()));
|
||||
}
|
||||
if let Some(sk) = &lq.is_skipped {
|
||||
sqlb.and_where_eq("is_skipped", sk);
|
||||
@@ -1905,14 +1692,6 @@ fn list_completed_jobs_query(
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(args) = &lq.args {
|
||||
sqlb.and_where("args @> ?".bind(&args.replace("'", "''")));
|
||||
}
|
||||
|
||||
if let Some(result) = &lq.result {
|
||||
sqlb.and_where("result @> ?".bind(&result.replace("'", "''")));
|
||||
}
|
||||
|
||||
sqlb
|
||||
}
|
||||
#[derive(Deserialize, Clone)]
|
||||
@@ -1921,8 +1700,8 @@ pub struct ListCompletedQuery {
|
||||
pub script_path_exact: Option<String>,
|
||||
pub script_hash: Option<String>,
|
||||
pub created_by: Option<String>,
|
||||
pub started_before: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub started_after: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub created_before: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub created_after: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub success: Option<bool>,
|
||||
pub parent_job: Option<String>,
|
||||
pub order_desc: Option<bool>,
|
||||
@@ -1930,11 +1709,6 @@ pub struct ListCompletedQuery {
|
||||
pub is_skipped: Option<bool>,
|
||||
pub is_flow_step: Option<bool>,
|
||||
pub suspended: Option<bool>,
|
||||
// filter by matching a subset of the args using base64 encoded json subset
|
||||
pub args: Option<String>,
|
||||
// filter by matching a subset of the result using base64 encoded json subset
|
||||
pub result: Option<String>,
|
||||
pub tag: Option<String>,
|
||||
}
|
||||
|
||||
async fn list_completed_jobs(
|
||||
@@ -1980,7 +1754,6 @@ async fn list_completed_jobs(
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
],
|
||||
)
|
||||
.sql()?;
|
||||
@@ -2002,7 +1775,6 @@ async fn get_completed_job(
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
tracing::info!("job_o: {:?}", job_o);
|
||||
let job = not_found_if_none(job_o, "Completed Job", id.to_string())?;
|
||||
Ok(Json(job))
|
||||
}
|
||||
|
||||
@@ -7,6 +7,17 @@
|
||||
*/
|
||||
|
||||
use crate::oauth2::AllClients;
|
||||
use argon2::Argon2;
|
||||
use axum::{middleware::from_extractor, routing::get, Extension, Router};
|
||||
use db::DB;
|
||||
use git_version::git_version;
|
||||
use reqwest::Client;
|
||||
use std::{net::SocketAddr, sync::Arc};
|
||||
use tower::ServiceBuilder;
|
||||
use tower_cookies::CookieManagerLayer;
|
||||
use tower_http::trace::TraceLayer;
|
||||
use windmill_common::utils::rd_string;
|
||||
|
||||
use crate::{
|
||||
db::UserDB,
|
||||
oauth2::{build_oauth_clients, SlackVerifier},
|
||||
@@ -14,32 +25,16 @@ use crate::{
|
||||
users::{Authed, OptAuthed},
|
||||
webhook_util::WebhookShared,
|
||||
};
|
||||
use argon2::Argon2;
|
||||
use axum::{middleware::from_extractor, routing::get, Extension, Router};
|
||||
use db::DB;
|
||||
use git_version::git_version;
|
||||
use hyper::Method;
|
||||
use reqwest::Client;
|
||||
use std::{net::SocketAddr, sync::Arc};
|
||||
use tower::ServiceBuilder;
|
||||
use tower_cookies::CookieManagerLayer;
|
||||
use tower_http::{
|
||||
cors::{Any, CorsLayer},
|
||||
trace::TraceLayer,
|
||||
};
|
||||
use windmill_common::utils::rd_string;
|
||||
|
||||
mod apps;
|
||||
mod audit;
|
||||
mod capture;
|
||||
mod db;
|
||||
mod drafts;
|
||||
mod favorite;
|
||||
mod flows;
|
||||
mod folders;
|
||||
mod granular_acls;
|
||||
mod groups;
|
||||
mod inputs;
|
||||
pub mod jobs;
|
||||
mod oauth2;
|
||||
mod resources;
|
||||
@@ -51,7 +46,7 @@ mod users;
|
||||
mod utils;
|
||||
mod variables;
|
||||
mod webhook_util;
|
||||
mod workers;
|
||||
mod worker_ping;
|
||||
mod workspaces;
|
||||
|
||||
pub const GIT_VERSION: &str =
|
||||
@@ -82,7 +77,6 @@ lazy_static::lazy_static! {
|
||||
|
||||
pub async fn run_server(
|
||||
db: DB,
|
||||
rsmq: Option<rsmq_async::MultiplexedRsmq>,
|
||||
addr: SocketAddr,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> anyhow::Result<()> {
|
||||
@@ -102,16 +96,10 @@ pub async fn run_server(
|
||||
.on_request(()),
|
||||
)
|
||||
.layer(Extension(db.clone()))
|
||||
.layer(Extension(rsmq))
|
||||
.layer(Extension(user_db))
|
||||
.layer(Extension(auth_cache.clone()))
|
||||
.layer(CookieManagerLayer::new())
|
||||
.layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
|
||||
|
||||
let cors = CorsLayer::new()
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
.allow_origin(Any);
|
||||
|
||||
// build our application with a route
|
||||
let app = Router::new()
|
||||
.nest(
|
||||
@@ -120,56 +108,44 @@ pub async fn run_server(
|
||||
.nest(
|
||||
"/w/:workspace_id",
|
||||
Router::new()
|
||||
// Reordered alphabetically
|
||||
.nest("/acls", granular_acls::workspaced_service())
|
||||
.nest("/apps", apps::workspaced_service())
|
||||
.nest("/audit", audit::workspaced_service())
|
||||
.nest("/capture", capture::workspaced_service())
|
||||
.nest("/favorites", favorite::workspaced_service())
|
||||
.nest("/flows", flows::workspaced_service())
|
||||
.nest("/folders", folders::workspaced_service())
|
||||
.nest("/groups", groups::workspaced_service())
|
||||
.nest("/inputs", inputs::workspaced_service())
|
||||
.nest("/jobs", jobs::workspaced_service().layer(cors.clone()))
|
||||
.nest("/oauth", oauth2::workspaced_service())
|
||||
.nest("/resources", resources::workspaced_service())
|
||||
.nest("/schedules", schedule::workspaced_service())
|
||||
.nest("/scripts", scripts::workspaced_service())
|
||||
.nest("/drafts", drafts::workspaced_service())
|
||||
.nest("/jobs", jobs::workspaced_service())
|
||||
.nest(
|
||||
"/users",
|
||||
users::workspaced_service().layer(Extension(argon2.clone())),
|
||||
)
|
||||
.nest("/variables", variables::workspaced_service())
|
||||
.nest("/workspaces", workspaces::workspaced_service()),
|
||||
.nest("/oauth", oauth2::workspaced_service())
|
||||
.nest("/resources", resources::workspaced_service())
|
||||
.nest("/schedules", schedule::workspaced_service())
|
||||
.nest("/groups", groups::workspaced_service())
|
||||
.nest("/audit", audit::workspaced_service())
|
||||
.nest("/acls", granular_acls::workspaced_service())
|
||||
.nest("/workspaces", workspaces::workspaced_service())
|
||||
.nest("/apps", apps::workspaced_service())
|
||||
.nest("/flows", flows::workspaced_service())
|
||||
.nest("/capture", capture::workspaced_service())
|
||||
.nest("/favorites", favorite::workspaced_service())
|
||||
.nest("/folders", folders::workspaced_service()),
|
||||
)
|
||||
.nest("/workspaces", workspaces::global_service())
|
||||
.nest(
|
||||
"/users",
|
||||
users::global_service().layer(Extension(argon2.clone())),
|
||||
)
|
||||
.nest("/workers", workers::global_service())
|
||||
.nest("/workers", worker_ping::global_service())
|
||||
.nest("/scripts", scripts::global_service())
|
||||
.nest("/flows", flows::global_service())
|
||||
.nest("/apps", apps::global_service().layer(cors.clone()))
|
||||
.nest("/apps", apps::global_service())
|
||||
.nest("/schedules", schedule::global_service())
|
||||
.route_layer(from_extractor::<Authed>())
|
||||
.route_layer(from_extractor::<users::Tokened>())
|
||||
.nest("/scripts_u", scripts::global_unauthed_service())
|
||||
.nest(
|
||||
"/w/:workspace_id/apps_u",
|
||||
apps::unauthed_service()
|
||||
.layer(from_extractor::<OptAuthed>())
|
||||
.layer(cors.clone()),
|
||||
)
|
||||
.nest(
|
||||
"/w/:workspace_id/jobs_u",
|
||||
jobs::global_service().layer(cors.clone()),
|
||||
)
|
||||
.nest(
|
||||
"/w/:workspace_id/capture_u",
|
||||
capture::global_service().layer(cors),
|
||||
apps::unauthed_service().layer(from_extractor::<OptAuthed>()),
|
||||
)
|
||||
.nest("/w/:workspace_id/jobs_u", jobs::global_service())
|
||||
.nest("/w/:workspace_id/capture_u", capture::global_service())
|
||||
.nest(
|
||||
"/auth",
|
||||
users::make_unauthed_service().layer(Extension(argon2)),
|
||||
|
||||
@@ -29,12 +29,10 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use tower_cookies::{Cookie, Cookies};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::jobs::JobPayload;
|
||||
use windmill_common::users::username_to_permissioned_as;
|
||||
use windmill_common::utils::{not_found_if_none, now_from_db};
|
||||
|
||||
use crate::users::{truncate_token, Authed};
|
||||
use crate::webhook_util::{InstanceEvent, WebhookShared};
|
||||
use crate::users::{truncate_token, Authed, NEW_USER_WEBHOOK};
|
||||
use crate::workspaces::invite_user_to_all_auto_invite_worspaces;
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
@@ -45,7 +43,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
|
||||
use windmill_common::error::{self, to_anyhow, Error};
|
||||
use windmill_common::oauth2::*;
|
||||
|
||||
use windmill_queue::QueueTransaction;
|
||||
use windmill_queue::JobPayload;
|
||||
|
||||
use std::{fs, str};
|
||||
|
||||
@@ -730,20 +728,23 @@ where
|
||||
async fn slack_command(
|
||||
SlackSig { sig, ts }: SlackSig,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
body: Bytes,
|
||||
) -> error::Result<String> {
|
||||
let form: SlackCommand = serde_urlencoded::from_bytes(&body)
|
||||
.map_err(|_| error::Error::BadRequest("invalid payload".to_string()))?;
|
||||
|
||||
let body = String::from_utf8_lossy(&body);
|
||||
if let Some(sv) = SLACK_SIGNING_SECRET.as_ref() {
|
||||
if sv.verify(&ts, &body, &sig).ok().is_none() {
|
||||
return Err(error::Error::BadRequest("verification failed".to_owned()));
|
||||
}
|
||||
if SLACK_SIGNING_SECRET
|
||||
.as_ref()
|
||||
.as_ref()
|
||||
.map(|sv| sv.verify(&ts, &body, &sig).ok())
|
||||
.flatten()
|
||||
.is_none()
|
||||
{
|
||||
return Err(error::Error::BadRequest("verification failed".to_owned()));
|
||||
}
|
||||
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
|
||||
let mut tx = db.begin().await?;
|
||||
let settings = sqlx::query_as!(
|
||||
WorkspaceSettings,
|
||||
"SELECT * FROM workspace_settings WHERE slack_team_id = $1",
|
||||
@@ -754,20 +755,17 @@ async fn slack_command(
|
||||
|
||||
if let Some(settings) = settings {
|
||||
if let Some(path) = &settings.slack_command_script {
|
||||
let (payload, tag) = if let Some(path) = path.strip_prefix("flow/") {
|
||||
(JobPayload::Flow(path.to_string()), None)
|
||||
let payload = if let Some(path) = path.strip_prefix("flow/") {
|
||||
JobPayload::Flow(path.to_string())
|
||||
} else {
|
||||
let path = path.strip_prefix("script/").unwrap_or_else(|| path);
|
||||
let (script_hash, tag) = windmill_common::get_latest_deployed_hash_for_path(
|
||||
tx.transaction_mut(),
|
||||
let script_hash = windmill_common::get_latest_hash_for_path(
|
||||
&mut tx,
|
||||
&settings.workspace_id,
|
||||
path,
|
||||
)
|
||||
.await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() },
|
||||
tag,
|
||||
)
|
||||
JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() }
|
||||
};
|
||||
let mut map = serde_json::Map::new();
|
||||
map.insert("text".to_string(), serde_json::Value::String(form.text));
|
||||
@@ -787,23 +785,20 @@ async fn slack_command(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
let url = BASE_URL.to_owned();
|
||||
tx.commit().await?;
|
||||
let url = BASE_URL.to_owned();
|
||||
return Ok(format!(
|
||||
"Job launched. See details at {url}/run/{uuid}?workspace={}",
|
||||
&settings.workspace_id
|
||||
));
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
return Ok(format!(
|
||||
"workspace not properly configured (did you set the script to trigger in the settings?)"
|
||||
@@ -823,7 +818,6 @@ async fn login_callback(
|
||||
Path(client_name): Path<String>,
|
||||
cookies: Cookies,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Json(callback): Json<OAuthCallback>,
|
||||
) -> error::Result<String> {
|
||||
let client_w_config = &OAUTH_CLIENTS
|
||||
@@ -858,8 +852,7 @@ async fn login_callback(
|
||||
_ => user.email.ok_or_else(|| {
|
||||
error::Error::BadRequest("email address not fetchable from user info".to_string())
|
||||
})?,
|
||||
}
|
||||
.to_lowercase();
|
||||
};
|
||||
|
||||
if let Some(domains) = &client_w_config.allowed_domains {
|
||||
if !domains.iter().any(|d| email.ends_with(d)) {
|
||||
@@ -949,7 +942,14 @@ async fn login_callback(
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
webhook.send_instance_event(InstanceEvent::UserSignupOAuth { email: email.clone() });
|
||||
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
|
||||
let _ = HTTP_CLIENT
|
||||
.post(&new_user_webhook)
|
||||
.json(&serde_json::json!({"email" : &email, "event": "oauth_signup"}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
|
||||
}
|
||||
|
||||
Ok("Successfully logged in".to_string())
|
||||
} else {
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{maybe_refresh_folders, require_owner_of_path, Authed},
|
||||
users::{require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
};
|
||||
use axum::{
|
||||
@@ -264,12 +264,9 @@ async fn create_resource(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(resource): Json<CreateResource>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
check_path_conflict(&mut tx, &w_id, &resource.path).await?;
|
||||
@@ -378,7 +375,6 @@ async fn update_resource(
|
||||
}
|
||||
|
||||
sqlb.returning("path");
|
||||
let authed = maybe_refresh_folders(path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
@@ -386,8 +382,9 @@ async fn update_resource(
|
||||
if npath != path {
|
||||
check_path_conflict(&mut tx, &w_id, &npath).await?;
|
||||
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
sqlx::query!(
|
||||
"UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
|
||||
npath,
|
||||
|
||||
@@ -6,26 +6,27 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{maybe_refresh_folders, Authed},
|
||||
users::Authed,
|
||||
};
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
routing::{delete, get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use chrono::{DateTime, Utc};
|
||||
use chrono::{DateTime, FixedOffset};
|
||||
use serde::Deserialize;
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use std::str::FromStr;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
schedule::Schedule,
|
||||
utils::{not_found_if_none, paginate, Pagination, StripPath}, jobs::JobKind,
|
||||
utils::{not_found_if_none, paginate, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, JobKind};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -46,7 +47,7 @@ pub fn global_service() -> Router {
|
||||
pub struct NewSchedule {
|
||||
pub path: String,
|
||||
pub schedule: String,
|
||||
pub timezone: String,
|
||||
pub offset: i32,
|
||||
pub script_path: String,
|
||||
pub is_flow: bool,
|
||||
pub args: Option<serde_json::Value>,
|
||||
@@ -77,34 +78,23 @@ async fn check_path_conflict<'c>(
|
||||
|
||||
async fn create_schedule(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(ns): Json<NewSchedule>,
|
||||
) -> Result<String> {
|
||||
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
cron::Schedule::from_str(&ns.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
|
||||
check_path_conflict(tx.transaction_mut(), &w_id, &ns.path).await?;
|
||||
check_flow_conflict(
|
||||
tx.transaction_mut(),
|
||||
&w_id,
|
||||
&ns.path,
|
||||
ns.is_flow,
|
||||
&ns.script_path,
|
||||
)
|
||||
.await?;
|
||||
check_path_conflict(&mut tx, &w_id, &ns.path).await?;
|
||||
check_flow_conflict(&mut tx, &w_id, &ns.path, ns.is_flow, &ns.script_path).await?;
|
||||
|
||||
let schedule = sqlx::query_as!(
|
||||
Schedule,
|
||||
"INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
|
||||
"INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, \
|
||||
is_flow, args, enabled, email) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *",
|
||||
w_id,
|
||||
ns.path,
|
||||
ns.schedule,
|
||||
ns.timezone,
|
||||
ns.offset,
|
||||
&authed.username,
|
||||
ns.script_path,
|
||||
ns.is_flow,
|
||||
@@ -145,18 +135,13 @@ async fn create_schedule(
|
||||
|
||||
async fn edit_schedule(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(es): Json<EditSchedule>,
|
||||
) -> Result<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let path = path.to_path();
|
||||
|
||||
let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
|
||||
let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
|
||||
(rsmq, user_db.begin(&authed).await?).into();
|
||||
|
||||
cron::Schedule::from_str(&es.schedule).map_err(|e| Error::BadRequest(e.to_string()))?;
|
||||
|
||||
let is_flow = sqlx::query_scalar!(
|
||||
@@ -167,13 +152,12 @@ async fn edit_schedule(
|
||||
.fetch_one(&mut tx)
|
||||
.await?;
|
||||
|
||||
clear_schedule(tx.transaction_mut(), path, is_flow).await?;
|
||||
clear_schedule(&mut tx, path, is_flow).await?;
|
||||
let schedule = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET schedule = $1, timezone = $2, args = $3 WHERE path \
|
||||
= $4 AND workspace_id = $5 RETURNING *",
|
||||
"UPDATE schedule SET schedule = $1, args = $2 WHERE path \
|
||||
= $3 AND workspace_id = $4 RETURNING *",
|
||||
es.schedule,
|
||||
es.timezone,
|
||||
es.args,
|
||||
path,
|
||||
w_id,
|
||||
@@ -182,6 +166,10 @@ async fn edit_schedule(
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
|
||||
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
@@ -197,10 +185,6 @@ async fn edit_schedule(
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(path.to_string())
|
||||
@@ -251,26 +235,15 @@ async fn exists_schedule(
|
||||
Ok(Json(res))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct PreviewPayload {
|
||||
pub schedule: String,
|
||||
pub timezone: String,
|
||||
}
|
||||
|
||||
pub async fn preview_schedule(
|
||||
Json(payload): Json<PreviewPayload>,
|
||||
) -> JsonResult<Vec<DateTime<Utc>>> {
|
||||
) -> JsonResult<Vec<DateTime<chrono::Utc>>> {
|
||||
let schedule = cron::Schedule::from_str(&payload.schedule)
|
||||
.map_err(|e| Error::BadRequest(e.to_string()))?;
|
||||
|
||||
let tz =
|
||||
chrono_tz::Tz::from_str(&payload.timezone).map_err(|e| Error::BadRequest(e.to_string()))?;
|
||||
|
||||
let upcoming: Vec<DateTime<Utc>> = schedule
|
||||
.upcoming(tz)
|
||||
.take(5)
|
||||
// Convert back to UTC for a standardised API response. The client will convert to the local timezone.
|
||||
.map(|x| x.with_timezone(&Utc))
|
||||
let upcoming: Vec<DateTime<chrono::Utc>> = schedule
|
||||
.upcoming(get_offset(payload.offset))
|
||||
.take(10)
|
||||
.map(|x| x.into())
|
||||
.collect();
|
||||
|
||||
Ok(Json(upcoming))
|
||||
@@ -279,12 +252,10 @@ pub async fn preview_schedule(
|
||||
pub async fn set_enabled(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(payload): Json<SetEnabled>,
|
||||
) -> Result<String> {
|
||||
let mut tx: QueueTransaction<'_, rsmq_async::MultiplexedRsmq> =
|
||||
(rsmq, user_db.begin(&authed).await?).into();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let path = path.to_path();
|
||||
let schedule_o = sqlx::query_as!(
|
||||
Schedule,
|
||||
@@ -299,8 +270,11 @@ pub async fn set_enabled(
|
||||
|
||||
let schedule = not_found_if_none(schedule_o, "Schedule", path)?;
|
||||
|
||||
clear_schedule(tx.transaction_mut(), path, schedule.is_flow).await?;
|
||||
clear_schedule(&mut tx, path, schedule.is_flow).await?;
|
||||
|
||||
if payload.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
@@ -311,12 +285,7 @@ pub async fn set_enabled(
|
||||
Some([("enabled", payload.enabled.to_string().as_ref())].into()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if payload.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(format!(
|
||||
"succesfully updated schedule at path {} to status {}",
|
||||
path, payload.enabled
|
||||
@@ -384,7 +353,6 @@ async fn check_flow_conflict<'c>(
|
||||
#[derive(Deserialize)]
|
||||
pub struct EditSchedule {
|
||||
pub schedule: String,
|
||||
pub timezone: String,
|
||||
pub args: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
@@ -408,6 +376,16 @@ pub async fn clear_schedule<'c>(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct PreviewPayload {
|
||||
pub schedule: String,
|
||||
pub offset: Option<i32>,
|
||||
}
|
||||
|
||||
fn get_offset(offset: Option<i32>) -> FixedOffset {
|
||||
FixedOffset::west_opt(offset.unwrap_or(0) * 60).expect("Invalid offset")
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SetEnabled {
|
||||
pub enabled: bool,
|
||||
|
||||
@@ -6,10 +6,13 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use sql_builder::prelude::*;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
schedule::clear_schedule,
|
||||
users::{maybe_refresh_folders, require_owner_of_path, AuthCache, Authed},
|
||||
users::{require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
HTTP_CLIENT,
|
||||
};
|
||||
@@ -21,21 +24,17 @@ use axum::{
|
||||
use hyper::StatusCode;
|
||||
use serde::Serialize;
|
||||
use serde_json::json;
|
||||
use sql_builder::prelude::*;
|
||||
use sql_builder::SqlBuilder;
|
||||
use sqlx::{FromRow, Postgres, Transaction};
|
||||
use std::{
|
||||
collections::hash_map::DefaultHasher,
|
||||
hash::{Hash, Hasher},
|
||||
sync::Arc,
|
||||
};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
jobs::JobPayload,
|
||||
schedule::Schedule,
|
||||
scripts::{
|
||||
to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
|
||||
to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Script, ScriptHash,
|
||||
ScriptKind, ScriptLang,
|
||||
},
|
||||
users::username_to_permissioned_as,
|
||||
@@ -43,35 +42,10 @@ use windmill_common::{
|
||||
list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_parser::MainArgSignature;
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).filter(|x| !x.is_empty()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
}
|
||||
use windmill_queue::{self, schedule::push_scheduled_job};
|
||||
|
||||
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
|
||||
|
||||
#[derive(Serialize, sqlx::FromRow)]
|
||||
pub struct ScriptWDraft {
|
||||
pub hash: ScriptHash,
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub description: String,
|
||||
pub content: String,
|
||||
pub language: ScriptLang,
|
||||
pub kind: ScriptKind,
|
||||
pub tag: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft: Option<serde_json::Value>,
|
||||
pub schema: Option<Schema>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
@@ -86,31 +60,21 @@ pub fn global_service() -> Router {
|
||||
.route("/hub/get_full/*path", get(get_full_hub_script_by_path))
|
||||
}
|
||||
|
||||
pub fn global_unauthed_service() -> Router {
|
||||
Router::new().route(
|
||||
"/tokened_raw/:workspace/:token/*path",
|
||||
get(get_tokened_raw_script_by_path),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_scripts))
|
||||
.route("/create", post(create_script))
|
||||
.route("/archive/p/*path", post(archive_script_by_path))
|
||||
.route("/get/draft/*path", get(get_script_by_path_w_draft))
|
||||
.route("/get/p/*path", get(get_script_by_path))
|
||||
.route("/raw/p/*path", get(raw_script_by_path))
|
||||
.route("/exists/p/*path", get(exists_script_by_path))
|
||||
.route("/archive/h/:hash", post(archive_script_by_hash))
|
||||
.route("/delete/h/:hash", post(delete_script_by_hash))
|
||||
.route("/delete/p/*path", post(delete_script_by_path))
|
||||
.route("/get/h/:hash", get(get_script_by_hash))
|
||||
.route("/raw/h/:hash", get(raw_script_by_hash))
|
||||
.route("/deployment_status/h/:hash", get(get_deployment_status))
|
||||
.route("/list_paths", get(list_paths))
|
||||
}
|
||||
|
||||
async fn list_scripts(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -122,18 +86,22 @@ async fn list_scripts(
|
||||
|
||||
let mut sqlb = SqlBuilder::select_from("script as o")
|
||||
.fields(&[
|
||||
"o.workspace_id",
|
||||
"hash",
|
||||
"o.path",
|
||||
"array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
|
||||
"summary",
|
||||
"COALESCE(draft.created_at, o.created_at) as created_at",
|
||||
"description",
|
||||
"created_by",
|
||||
"created_at",
|
||||
"archived",
|
||||
"deleted",
|
||||
"is_template",
|
||||
"extra_perms",
|
||||
"CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors",
|
||||
"CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
|
||||
"language",
|
||||
"kind",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
"tag",
|
||||
"draft.path IS NOT NULL as has_draft",
|
||||
"draft_only"
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
@@ -141,11 +109,6 @@ async fn list_scripts(
|
||||
"favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
|
||||
.bind(&authed.username),
|
||||
)
|
||||
.left()
|
||||
.join("draft")
|
||||
.on(
|
||||
"draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'"
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("created_at", lq.order_desc.unwrap_or(true))
|
||||
.and_where("o.workspace_id = ?".bind(&w_id))
|
||||
@@ -155,12 +118,11 @@ async fn list_scripts(
|
||||
|
||||
if lq.show_archived.unwrap_or(false) {
|
||||
sqlb.and_where_eq(
|
||||
"o.created_at",
|
||||
"created_at",
|
||||
"(select max(created_at) from script where o.path = path
|
||||
AND workspace_id = ?)"
|
||||
.bind(&w_id),
|
||||
);
|
||||
sqlb.and_where_eq("archived", true);
|
||||
} else {
|
||||
sqlb.and_where_eq("archived", false);
|
||||
}
|
||||
@@ -220,22 +182,20 @@ fn hash_script(ns: &NewScript) -> i64 {
|
||||
async fn create_script(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(ns): Json<NewScript>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let hash = ScriptHash(hash_script(&ns));
|
||||
let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
if sqlx::query_scalar!(
|
||||
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
hash.0,
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(tx.transaction_mut())
|
||||
.fetch_optional(&mut tx)
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
@@ -261,20 +221,10 @@ async fn create_script(
|
||||
}
|
||||
let parent_hashes_and_perms: Option<ParentInfo> = match (&ns.parent_hash, clashing_script) {
|
||||
(None, None) => Ok(None),
|
||||
(None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!(
|
||||
(None, Some(s)) => Err(Error::BadRequest(format!(
|
||||
"Path conflict for {} with non-archived hash {}",
|
||||
&ns.path, &s.hash
|
||||
))),
|
||||
(None, Some(s)) => {
|
||||
sqlx::query!(
|
||||
"DELETE FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
s.hash.0,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
Ok(None)
|
||||
}
|
||||
(Some(p_hash), o) => {
|
||||
if sqlx::query_scalar!(
|
||||
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
@@ -306,10 +256,13 @@ async fn create_script(
|
||||
)));
|
||||
};
|
||||
|
||||
let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
|
||||
let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
|
||||
|
||||
if ps.path != ns.path {
|
||||
require_owner_of_path(&authed, &ps.path)?;
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &ps.path, &db)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
let ph = {
|
||||
@@ -366,8 +319,8 @@ async fn create_script(
|
||||
//::text::json is to ensure we use serde_json with preserve order
|
||||
sqlx::query!(
|
||||
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
|
||||
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)",
|
||||
content, created_by, schema, is_template, extra_perms, lock, language, kind) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
|
||||
&w_id,
|
||||
&hash.0,
|
||||
ns.path,
|
||||
@@ -382,21 +335,11 @@ async fn create_script(
|
||||
lock,
|
||||
ns.language: ScriptLang,
|
||||
ns.kind.unwrap_or(ScriptKind::Script): ScriptKind,
|
||||
ns.tag,
|
||||
ns.draft_only
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
if let Some(p_path) = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()) {
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
|
||||
p_path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
let schedulables = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
|
||||
@@ -408,22 +351,43 @@ async fn create_script(
|
||||
.await?;
|
||||
|
||||
for schedule in schedulables {
|
||||
clear_schedule(tx.transaction_mut(), &schedule.path, false).await?;
|
||||
clear_schedule(&mut tx, &schedule.path, false).await?;
|
||||
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
|
||||
ns.path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let mut tx = if needs_lock_gen {
|
||||
let dependencies = match ns.language {
|
||||
ScriptLang::Python3 => {
|
||||
windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
|
||||
}
|
||||
_ => ns.content,
|
||||
};
|
||||
let (_, tx) = windmill_queue::push(
|
||||
tx,
|
||||
&w_id,
|
||||
windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
|
||||
serde_json::Map::new(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
tx
|
||||
} else {
|
||||
tx
|
||||
};
|
||||
|
||||
if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
|
||||
audit_log(
|
||||
&mut tx,
|
||||
@@ -438,7 +402,7 @@ async fn create_script(
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::UpdateScript {
|
||||
workspace: w_id.clone(),
|
||||
workspace: w_id,
|
||||
path: ns.path.clone(),
|
||||
hash: hash.to_string(),
|
||||
},
|
||||
@@ -463,42 +427,13 @@ async fn create_script(
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::CreateScript {
|
||||
workspace: w_id.clone(),
|
||||
workspace: w_id,
|
||||
path: ns.path.clone(),
|
||||
hash: hash.to_string(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if needs_lock_gen {
|
||||
let dependencies = match ns.language {
|
||||
ScriptLang::Python3 => {
|
||||
windmill_parser_py::parse_python_imports(&ns.content)?.join("\n")
|
||||
}
|
||||
_ => ns.content,
|
||||
};
|
||||
let (_, new_tx) = windmill_queue::push(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::Dependencies { hash, dependencies, language: ns.language },
|
||||
serde_json::Map::new(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
username_to_permissioned_as(&authed.username),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx = new_tx;
|
||||
}
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, format!("{}", hash)))
|
||||
@@ -540,31 +475,6 @@ async fn get_script_by_path(
|
||||
Ok(Json(script))
|
||||
}
|
||||
|
||||
async fn get_script_by_path_w_draft(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<ScriptWDraft> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let script_o = sqlx::query_as::<_, ScriptWDraft>(
|
||||
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON
|
||||
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
|
||||
WHERE script.path = $1 AND script.workspace_id = $2 \
|
||||
AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
|
||||
workspace_id = $2)",
|
||||
)
|
||||
.bind(path)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let script = not_found_if_none(script_o, "Script", path)?;
|
||||
Ok(Json(script))
|
||||
}
|
||||
|
||||
async fn list_paths(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -583,18 +493,6 @@ async fn list_paths(
|
||||
Ok(Json(scripts))
|
||||
}
|
||||
|
||||
async fn get_tokened_raw_script_by_path(
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, token, path)): Path<(String, String, StripPath)>,
|
||||
Extension(cache): Extension<Arc<AuthCache>>,
|
||||
) -> Result<String> {
|
||||
let authed = cache
|
||||
.get_authed(Some(w_id.clone()), &token)
|
||||
.await
|
||||
.ok_or_else(|| Error::NotAuthorized("Invalid token".to_string()))?;
|
||||
return raw_script_by_path(authed, Extension(user_db), Path((w_id, path))).await;
|
||||
}
|
||||
|
||||
async fn raw_script_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -655,10 +553,11 @@ async fn get_script_by_hash_internal<'c>(
|
||||
}
|
||||
|
||||
async fn get_script_by_hash(
|
||||
Extension(db): Extension<DB>,
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, hash)): Path<(String, ScriptHash)>,
|
||||
) -> JsonResult<Script> {
|
||||
let mut tx = db.begin().await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
|
||||
tx.commit().await?;
|
||||
|
||||
@@ -666,10 +565,11 @@ async fn get_script_by_hash(
|
||||
}
|
||||
|
||||
async fn raw_script_by_hash(
|
||||
Extension(db): Extension<DB>,
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, hash_str)): Path<(String, String)>,
|
||||
) -> Result<String> {
|
||||
let mut tx = db.begin().await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| {
|
||||
Error::BadRequest("Raw script path must end with .ts".to_string())
|
||||
})?)?);
|
||||
@@ -685,10 +585,11 @@ struct DeploymentStatus {
|
||||
lock_error_logs: Option<String>,
|
||||
}
|
||||
async fn get_deployment_status(
|
||||
Extension(db): Extension<DB>,
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, hash)): Path<(String, ScriptHash)>,
|
||||
) -> JsonResult<DeploymentStatus> {
|
||||
let mut tx = db.begin().await?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let status_o: Option<DeploymentStatus> = sqlx::query_as!(
|
||||
DeploymentStatus,
|
||||
"SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
@@ -714,8 +615,6 @@ async fn archive_script_by_path(
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
let hash: i64 = sqlx::query_scalar!(
|
||||
"UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash",
|
||||
path,
|
||||
@@ -819,83 +718,25 @@ async fn delete_script_by_hash(
|
||||
Ok(Json(script))
|
||||
}
|
||||
|
||||
async fn delete_script_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let path = path.to_path();
|
||||
|
||||
let draft_only = sqlx::query_scalar!(
|
||||
"SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
if !draft_only {
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
}
|
||||
|
||||
let script = sqlx::query_scalar!(
|
||||
"DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("deleting script by path {w_id}: {e}")))?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
"scripts.delete",
|
||||
ActionKind::Delete,
|
||||
&w_id,
|
||||
Some(&path),
|
||||
Some([("workspace", w_id.as_str())].into()),
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::DeleteScriptPath { workspace: w_id, path: path.to_string() },
|
||||
);
|
||||
|
||||
Ok(Json(script))
|
||||
async fn parse_python_code_to_jsonschema(
|
||||
Json(code): Json<String>,
|
||||
) -> JsonResult<windmill_parser::MainArgSignature> {
|
||||
windmill_parser_py::parse_python_signature(&code).map(Json)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
enum SigParsing {
|
||||
Valid(MainArgSignature),
|
||||
Invalid { error: String },
|
||||
async fn parse_deno_code_to_jsonschema(
|
||||
Json(code): Json<String>,
|
||||
) -> JsonResult<windmill_parser::MainArgSignature> {
|
||||
windmill_parser_ts::parse_deno_signature(&code).map(Json)
|
||||
}
|
||||
async fn parse_go_code_to_jsonschema(
|
||||
Json(code): Json<String>,
|
||||
) -> JsonResult<windmill_parser::MainArgSignature> {
|
||||
windmill_parser_go::parse_go_sig(&code).map(Json)
|
||||
}
|
||||
|
||||
fn result_to_sig_parsing(result: Result<MainArgSignature>) -> Json<SigParsing> {
|
||||
match result {
|
||||
Ok(sig) => Json(SigParsing::Valid(sig)),
|
||||
Err(e) => Json(SigParsing::Invalid { error: e.to_string() }),
|
||||
}
|
||||
}
|
||||
|
||||
async fn parse_python_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
|
||||
result_to_sig_parsing(windmill_parser_py::parse_python_signature(&code))
|
||||
}
|
||||
|
||||
async fn parse_deno_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
|
||||
result_to_sig_parsing(windmill_parser_ts::parse_deno_signature(&code, false))
|
||||
}
|
||||
async fn parse_go_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
|
||||
result_to_sig_parsing(windmill_parser_go::parse_go_sig(&code))
|
||||
}
|
||||
|
||||
async fn parse_bash_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
|
||||
result_to_sig_parsing(windmill_parser_bash::parse_bash_sig(&code))
|
||||
async fn parse_bash_code_to_jsonschema(
|
||||
Json(code): Json<String>,
|
||||
) -> JsonResult<windmill_parser::MainArgSignature> {
|
||||
windmill_parser_bash::parse_bash_sig(&code).map(Json)
|
||||
}
|
||||
|
||||
@@ -12,9 +12,8 @@ use crate::{
|
||||
db::{UserDB, DB},
|
||||
folders::get_folders_for_user,
|
||||
utils::require_super_admin,
|
||||
webhook_util::{InstanceEvent, WebhookShared},
|
||||
workspaces::invite_user_to_all_auto_invite_worspaces,
|
||||
COOKIE_DOMAIN, IS_SECURE,
|
||||
COOKIE_DOMAIN, HTTP_CLIENT, IS_SECURE,
|
||||
};
|
||||
use argon2::{password_hash::SaltString, Argon2, PasswordHash, PasswordHasher, PasswordVerifier};
|
||||
use axum::{
|
||||
@@ -26,9 +25,7 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::{header::LOCATION, StatusCode};
|
||||
use lazy_static::lazy_static;
|
||||
use rand::rngs::OsRng;
|
||||
use regex::Regex;
|
||||
use retainer::Cache;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::FromRow;
|
||||
@@ -38,7 +35,6 @@ use tracing::{Instrument, Span};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{self, Error, JsonResult, Result},
|
||||
users::SUPERADMIN_SECRET_EMAIL,
|
||||
utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
@@ -77,7 +73,6 @@ pub fn global_service() -> Router {
|
||||
.route("/tokens/create", post(create_token))
|
||||
.route("/tokens/delete/:token_prefix", delete(delete_token))
|
||||
.route("/tokens/list", get(list_tokens))
|
||||
.route("/tokens/impersonate", post(impersonate))
|
||||
.route("/usage", get(get_usage))
|
||||
// .route("/list_invite_codes", get(list_invite_codes))
|
||||
// .route("/create_invite_code", post(create_invite_code))
|
||||
@@ -104,10 +99,6 @@ impl AuthCache {
|
||||
AuthCache { cache: Cache::new(), db, superadmin_secret }
|
||||
}
|
||||
|
||||
pub async fn invalidate(&self, w_id: &str, token: String) {
|
||||
self.cache.remove(&(w_id.to_string(), token)).await;
|
||||
}
|
||||
|
||||
pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<Authed> {
|
||||
let key = (
|
||||
w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
|
||||
@@ -275,7 +266,7 @@ impl AuthCache {
|
||||
.unwrap_or(false)
|
||||
{
|
||||
Some(Authed {
|
||||
email: SUPERADMIN_SECRET_EMAIL.to_string(),
|
||||
email: "superadmin_secret@windmill.dev".to_string(),
|
||||
username: "superadmin_secret".to_string(),
|
||||
is_admin: true,
|
||||
groups: Vec::new(),
|
||||
@@ -359,33 +350,7 @@ pub struct Authed {
|
||||
pub username: String,
|
||||
pub is_admin: bool,
|
||||
pub groups: Vec<String>,
|
||||
// (folder name, can write, is owner)
|
||||
pub folders: Vec<(String, bool, bool)>,
|
||||
}
|
||||
|
||||
pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
|
||||
if authed.is_admin {
|
||||
return authed;
|
||||
}
|
||||
let splitted = path.split('/').collect::<Vec<_>>();
|
||||
if splitted.len() >= 2
|
||||
&& splitted[0] == "f"
|
||||
&& !authed.folders.iter().any(|(f, _, _)| f == splitted[1])
|
||||
{
|
||||
let name = &authed.username;
|
||||
let groups = get_groups_for_user(w_id, name, db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_default();
|
||||
|
||||
let folders = get_folders_for_user(w_id, name, &groups, db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_default();
|
||||
Authed { folders, ..authed }
|
||||
} else {
|
||||
authed
|
||||
}
|
||||
pub folders: Vec<(String, bool)>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -510,7 +475,6 @@ pub struct UserInfo {
|
||||
pub disabled: bool,
|
||||
pub role: Option<String>,
|
||||
pub folders: Vec<String>,
|
||||
pub folders_owners: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize)]
|
||||
@@ -597,7 +561,6 @@ pub struct TruncatedToken {
|
||||
pub struct NewToken {
|
||||
pub label: Option<String>,
|
||||
pub expiration: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub impersonate_email: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -810,14 +773,9 @@ async fn whoami(
|
||||
disabled: false,
|
||||
role: Some("superadmin".to_string()),
|
||||
folders: folders
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.1 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
folders_owners: folders
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.2 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -903,14 +861,9 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
|
||||
disabled: usr.disabled,
|
||||
role: usr.role,
|
||||
folders: folders
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.1 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
folders_owners: folders
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.2 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -925,45 +878,47 @@ pub async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<
|
||||
Ok(groups)
|
||||
}
|
||||
pub async fn is_owner_of_path(
|
||||
authed: Authed,
|
||||
Path((_w_id, path)): Path<(String, StripPath)>,
|
||||
Authed { username, is_admin, groups, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<bool> {
|
||||
let path = path.to_path();
|
||||
if authed.is_admin {
|
||||
if is_admin {
|
||||
Ok(Json(true))
|
||||
} else {
|
||||
Ok(Json(require_owner_of_path(&authed, path).is_ok()))
|
||||
Ok(Json(
|
||||
require_owner_of_path(&w_id, &username, &groups, path, &db)
|
||||
.await
|
||||
.is_ok(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
|
||||
if authed.is_admin {
|
||||
return Ok(());
|
||||
}
|
||||
pub async fn require_owner_of_path(
|
||||
w_id: &str,
|
||||
username: &str,
|
||||
groups: &Vec<String>,
|
||||
path: &str,
|
||||
db: &DB,
|
||||
) -> Result<()> {
|
||||
if !path.is_empty() {
|
||||
let splitted = path.split("/").collect::<Vec<&str>>();
|
||||
if splitted[0] == "u" {
|
||||
if splitted[1] == authed.username {
|
||||
Ok(())
|
||||
if splitted[1] == username {
|
||||
return Ok(());
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"only the owner {} is authorized to perform this operation",
|
||||
splitted[1]
|
||||
)))
|
||||
)));
|
||||
}
|
||||
} else if splitted[0] == "g" {
|
||||
return crate::groups::require_is_owner(splitted[1], username, groups, w_id, db).await;
|
||||
} else if splitted[0] == "f" {
|
||||
crate::folders::require_is_owner(authed, splitted[1])
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
"Not recognized path kind: {}",
|
||||
path
|
||||
)))
|
||||
return crate::folders::require_is_owner(splitted[1], username, groups, w_id, db).await;
|
||||
}
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
"Cannot be owner of an empty path"
|
||||
)))
|
||||
}
|
||||
Err(Error::BadRequest(format!("not recognized owner kind")))
|
||||
}
|
||||
|
||||
async fn whois(
|
||||
@@ -1042,27 +997,14 @@ async fn decline_invite(
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
|
||||
|
||||
pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<u32>().ok())
|
||||
.unwrap_or(60 * 60 * 24 * 60); // 60 days
|
||||
}
|
||||
|
||||
async fn accept_invite(
|
||||
Authed { email, .. }: Authed,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(db): Extension<DB>,
|
||||
Json(nu): Json<AcceptInvite>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
if !VALID_USERNAME.is_match(&nu.username) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Usermame can only contain alphanumeric characters and underscores"
|
||||
)));
|
||||
if &nu.username == "bot" {
|
||||
return Err(Error::BadRequest("bot is a reserved username".to_string()));
|
||||
}
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
let r = sqlx::query!(
|
||||
@@ -1099,11 +1041,6 @@ async fn accept_invite(
|
||||
}
|
||||
|
||||
if is_some {
|
||||
webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace {
|
||||
email: email.clone(),
|
||||
workspace: nu.workspace_id.clone(),
|
||||
username: nu.username.clone(),
|
||||
});
|
||||
Ok((
|
||||
StatusCode::CREATED,
|
||||
format!(
|
||||
@@ -1140,12 +1077,6 @@ async fn add_user_to_workspace<'c>(
|
||||
)));
|
||||
}
|
||||
|
||||
if !VALID_USERNAME.is_match(username) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Usermame can only contain alphanumeric characters and underscores"
|
||||
)));
|
||||
}
|
||||
|
||||
let already_exists_email = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)",
|
||||
&w_id,
|
||||
@@ -1344,20 +1275,12 @@ lazy_static::lazy_static! {
|
||||
async fn create_user(
|
||||
Authed { email, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(argon2): Extension<Arc<Argon2<'_>>>,
|
||||
Json(mut nu): Json<NewUser>,
|
||||
Json(nu): Json<NewUser>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
require_super_admin(&mut tx, &email).await?;
|
||||
nu.email = nu.email.to_lowercase();
|
||||
|
||||
if nu.email == SUPERADMIN_SECRET_EMAIL {
|
||||
return Err(Error::BadRequest(
|
||||
"The superadmin email is a reserved email".into(),
|
||||
));
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, \
|
||||
@@ -1385,9 +1308,17 @@ async fn create_user(
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
|
||||
let _ = HTTP_CLIENT
|
||||
.post(&new_user_webhook)
|
||||
.json(&serde_json::json!({"email" : &nu.email, "name": &nu.name, "event": "new_user"}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
|
||||
}
|
||||
|
||||
invite_user_to_all_auto_invite_worspaces(&db, &nu.email).await?;
|
||||
|
||||
webhook.send_instance_event(InstanceEvent::UserAdded { email: nu.email.clone() });
|
||||
Ok((StatusCode::CREATED, format!("email {} created", nu.email)))
|
||||
}
|
||||
|
||||
@@ -1633,7 +1564,7 @@ async fn login(
|
||||
Json(Login { email, password }): Json<Login>,
|
||||
) -> Result<String> {
|
||||
let mut tx = db.begin().await?;
|
||||
let email = email.to_lowercase();
|
||||
|
||||
let email_w_h: Option<(String, String, bool, bool)> = sqlx::query_as(
|
||||
"SELECT email, password_hash, super_admin, first_time_user FROM password WHERE email = $1 AND login_type = \
|
||||
'password'",
|
||||
@@ -1755,58 +1686,6 @@ async fn create_token(
|
||||
Ok((StatusCode::CREATED, token))
|
||||
}
|
||||
|
||||
async fn impersonate(
|
||||
Extension(db): Extension<DB>,
|
||||
Authed { email, username, .. }: Authed,
|
||||
Json(new_token): Json<NewToken>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let token = rd_string(30);
|
||||
let mut tx = db.begin().await?;
|
||||
require_super_admin(&mut tx, &email).await?;
|
||||
|
||||
if new_token.impersonate_email.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"impersonate_username is required".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let impersonated = new_token.impersonate_email.unwrap();
|
||||
|
||||
let is_super_admin = sqlx::query_scalar!(
|
||||
"SELECT super_admin FROM password WHERE email = $1",
|
||||
impersonated
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
sqlx::query!(
|
||||
"INSERT INTO token
|
||||
(token, email, label, expiration, super_admin)
|
||||
VALUES ($1, $2, $3, $4, $5)",
|
||||
token,
|
||||
impersonated,
|
||||
new_token.label,
|
||||
new_token.expiration,
|
||||
is_super_admin
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&username,
|
||||
"users.impersonate",
|
||||
ActionKind::Delete,
|
||||
&"global",
|
||||
Some(&token[0..10]),
|
||||
Some([("impersonated", &format!("{impersonated}")[..])].into()),
|
||||
)
|
||||
.instrument(tracing::info_span!("token", email = &impersonated))
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok((StatusCode::CREATED, token))
|
||||
}
|
||||
|
||||
async fn list_tokens(
|
||||
Extension(db): Extension<DB>,
|
||||
Authed { email, .. }: Authed,
|
||||
@@ -1907,35 +1786,16 @@ pub async fn delete_expired_items_perdiodically(
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
|
||||
let pip_resolution_r = sqlx::query_scalar!(
|
||||
"DELETE FROM pip_resolution_cache WHERE expiration <= now() RETURNING hash",
|
||||
let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
|
||||
"DELETE FROM magic_link WHERE expiration <= now()
|
||||
RETURNING concat(substring(token for 10), '*****')",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match pip_resolution_r {
|
||||
Ok(res) => tracing::debug!("deleted {} pip_resolution: {:?}", res.len(), res),
|
||||
Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
|
||||
}
|
||||
|
||||
if *JOB_RETENTION_SECS > 0 {
|
||||
let deleted_jobs = sqlx::query_scalar!(
|
||||
"DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
|
||||
*JOB_RETENTION_SECS as i64
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_jobs {
|
||||
Ok(deleted_jobs) => {
|
||||
tracing::info!(
|
||||
"deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
|
||||
deleted_jobs.len(),
|
||||
deleted_jobs
|
||||
)
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
|
||||
}
|
||||
match magic_links_deleted_r {
|
||||
Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
|
||||
@@ -7,24 +7,16 @@
|
||||
*/
|
||||
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
users::SUPERADMIN_SECRET_EMAIL,
|
||||
};
|
||||
use windmill_common::error::{self, Error};
|
||||
|
||||
pub async fn require_super_admin<'c>(
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
email: &str,
|
||||
) -> error::Result<()> {
|
||||
if email == SUPERADMIN_SECRET_EMAIL {
|
||||
return Ok(());
|
||||
}
|
||||
let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
|
||||
.fetch_optional(db)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?
|
||||
.unwrap_or(false);
|
||||
|
||||
.map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
|
||||
if !is_admin {
|
||||
Err(Error::NotAuthorized(
|
||||
"This endpoint require caller to be a super admin".to_owned(),
|
||||
|
||||
@@ -9,9 +9,16 @@
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
oauth2::_refresh_token,
|
||||
users::{maybe_refresh_folders, require_owner_of_path, Authed},
|
||||
users::{require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
};
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2022
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
@@ -19,7 +26,6 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde_json::Value;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
@@ -199,15 +205,12 @@ async fn check_path_conflict<'c>(
|
||||
|
||||
async fn create_variable(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(AlreadyEncrypted { already_encrypted }): Query<AlreadyEncrypted>,
|
||||
Json(variable): Json<CreateVariable>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
check_path_conflict(&mut tx, &w_id, &variable.path).await?;
|
||||
@@ -326,7 +329,6 @@ async fn update_variable(
|
||||
use sql_builder::prelude::*;
|
||||
|
||||
let path = path.to_path();
|
||||
let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
@@ -373,29 +375,12 @@ async fn update_variable(
|
||||
if let Some(npath) = ns.path {
|
||||
if npath != path {
|
||||
check_path_conflict(&mut tx, &w_id, &npath).await?;
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
let mut v = sqlx::query_scalar!(
|
||||
"SELECT value FROM resource WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?
|
||||
.flatten();
|
||||
|
||||
if let Some(old_v) = v {
|
||||
v = Some(replace_path(
|
||||
old_v,
|
||||
&format!("$var:{path}"),
|
||||
&format!("$var:{npath}"),
|
||||
))
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE resource SET path = $1, value = $2 WHERE path = $3 AND workspace_id = $4",
|
||||
"UPDATE resource SET path = $1 WHERE path = $2 AND workspace_id = $3",
|
||||
npath,
|
||||
v,
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
@@ -434,23 +419,6 @@ async fn update_variable(
|
||||
Ok(format!("variable {} updated (npath: {:?})", path, npath))
|
||||
}
|
||||
|
||||
fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value {
|
||||
match v {
|
||||
Value::Object(v) => Value::Object(
|
||||
v.into_iter()
|
||||
.map(|(k, v)| (k, replace_path(v, path, npath)))
|
||||
.collect(),
|
||||
),
|
||||
Value::Array(arr) => Value::Array(
|
||||
arr.into_iter()
|
||||
.map(|v| replace_path(v, path, npath))
|
||||
.collect(),
|
||||
),
|
||||
Value::String(s) if s == path => Value::String(npath.to_owned()),
|
||||
_ => v,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn build_crypt<'c>(
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
|
||||
@@ -2,7 +2,6 @@ use std::time::Duration;
|
||||
|
||||
use serde::Serialize;
|
||||
use tokio::{select, sync::mpsc, time::interval};
|
||||
use windmill_common::METRICS_ENABLED;
|
||||
|
||||
use crate::db::DB;
|
||||
|
||||
@@ -13,26 +12,6 @@ lazy_static::lazy_static! {
|
||||
"Histogram of webhook requests made"
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
pub static ref INSTANCE_EVENTS_WEBHOOK: Option<String> = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok();
|
||||
|
||||
}
|
||||
|
||||
pub enum WebhookPayload {
|
||||
WorkspaceEvent(String, WebhookMessage),
|
||||
InstanceEvent(InstanceEvent),
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum InstanceEvent {
|
||||
UserSignupOAuth { email: String },
|
||||
UserAdded { email: String },
|
||||
// UserDeleted { email: String },
|
||||
// UserDeletedWorkspace { workspace: String, email: String },
|
||||
UserAddedWorkspace { workspace: String, email: String },
|
||||
UserInvitedWorkspace { workspace: String, email: String },
|
||||
UserJoinedWorkspace { workspace: String, email: String, username: String },
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -58,7 +37,6 @@ pub enum WebhookMessage {
|
||||
CreateScript { workspace: String, path: String, hash: String },
|
||||
UpdateScript { workspace: String, path: String, hash: String },
|
||||
DeleteScript { workspace: String, hash: String },
|
||||
DeleteScriptPath { workspace: String, path: String },
|
||||
CreateVariable { workspace: String, path: String },
|
||||
UpdateVariable { workspace: String, old_path: String, new_path: String },
|
||||
DeleteVariable { workspace: String, path: String },
|
||||
@@ -66,12 +44,12 @@ pub enum WebhookMessage {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct WebhookShared {
|
||||
pub channel: mpsc::UnboundedSender<WebhookPayload>,
|
||||
pub channel: mpsc::UnboundedSender<(String, WebhookMessage)>,
|
||||
}
|
||||
|
||||
impl WebhookShared {
|
||||
pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self {
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<WebhookPayload>();
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<(String, WebhookMessage)>();
|
||||
let _process = tokio::spawn(async move {
|
||||
let client = reqwest::Client::builder()
|
||||
// TODO: investigate pool timeouts and such if TCP load is high
|
||||
@@ -86,7 +64,7 @@ impl WebhookShared {
|
||||
biased;
|
||||
_ = shutdown_rx.recv() => break,
|
||||
r = rx.recv() => match r {
|
||||
Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => {
|
||||
Some((workspace_id, message)) => {
|
||||
let url_guard = match cache.get(&workspace_id).await {
|
||||
Some(guard) => {
|
||||
guard
|
||||
@@ -110,19 +88,12 @@ impl WebhookShared {
|
||||
};
|
||||
let webook_opt = url_guard.value();
|
||||
if let Some(url) = webook_opt {
|
||||
let timer = if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
|
||||
let timer = WEBHOOK_REQUEST_COUNT.start_timer();
|
||||
let _ = client.post(url).json(&message).send().await;
|
||||
timer.map(|x| x.stop_and_record());
|
||||
timer.stop_and_record();
|
||||
drop(url_guard);
|
||||
}
|
||||
},
|
||||
Some(WebhookPayload::InstanceEvent(event)) => {
|
||||
if *METRICS_ENABLED { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None };
|
||||
let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await;
|
||||
if let Err(e) = r {
|
||||
tracing::error!("Error sending instance event: {}", e);
|
||||
}
|
||||
},
|
||||
None => break,
|
||||
},
|
||||
_ = futures::future::poll_fn(|cx| cache_purge_interval.poll_tick(cx)) => {
|
||||
@@ -137,16 +108,6 @@ impl WebhookShared {
|
||||
}
|
||||
|
||||
pub fn send_message(&self, workspace_id: String, message: WebhookMessage) {
|
||||
let _ = self.channel.send(WebhookPayload::WorkspaceEvent(
|
||||
workspace_id.clone(),
|
||||
message,
|
||||
));
|
||||
}
|
||||
|
||||
pub fn send_instance_event(&self, event: InstanceEvent) {
|
||||
if INSTANCE_EVENTS_WEBHOOK.is_none() {
|
||||
return;
|
||||
}
|
||||
let _ = self.channel.send(WebhookPayload::InstanceEvent(event));
|
||||
let _ = self.channel.send((workspace_id.clone(), message));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,23 +21,14 @@ use windmill_common::{
|
||||
};
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_worker_pings))
|
||||
.route("/custom_tags", get(get_custom_tags))
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
Router::new().route("/list", get(list_worker_pings))
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize)]
|
||||
struct WorkerPing {
|
||||
worker: String,
|
||||
worker_instance: String,
|
||||
last_ping: Option<i32>,
|
||||
ping_at: chrono::DateTime<chrono::Utc>,
|
||||
started_at: chrono::DateTime<chrono::Utc>,
|
||||
ip: String,
|
||||
jobs_executed: i32,
|
||||
@@ -54,7 +45,7 @@ async fn list_worker_pings(
|
||||
|
||||
let rows = sqlx::query_as!(
|
||||
WorkerPing,
|
||||
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
"SELECT * FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
|
||||
per_page as i64,
|
||||
offset as i64
|
||||
)
|
||||
@@ -63,7 +54,3 @@ async fn list_worker_pings(
|
||||
tx.commit().await?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
async fn get_custom_tags() -> Json<Vec<String>> {
|
||||
Json(CUSTOM_TAGS.clone())
|
||||
}
|
||||
@@ -16,10 +16,9 @@ use crate::{
|
||||
db::{UserDB, DB},
|
||||
folders::Folder,
|
||||
resources::{Resource, ResourceType},
|
||||
users::{Authed, WorkspaceInvite, VALID_USERNAME},
|
||||
users::{Authed, WorkspaceInvite, NEW_USER_WEBHOOK},
|
||||
utils::require_super_admin,
|
||||
variables::build_crypt,
|
||||
webhook_util::{InstanceEvent, WebhookShared},
|
||||
HTTP_CLIENT,
|
||||
};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use axum::response::Redirect;
|
||||
@@ -31,7 +30,6 @@ use axum::{
|
||||
routing::{delete, get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use magic_crypt::MagicCryptTrait;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use stripe::CustomerId;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
@@ -943,14 +941,11 @@ pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> R
|
||||
async fn invite_user(
|
||||
Authed { username, is_admin, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(mut nu): Json<NewWorkspaceInvite>,
|
||||
Json(nu): Json<NewWorkspaceInvite>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
require_admin(is_admin, &username)?;
|
||||
|
||||
nu.email = nu.email.to_lowercase();
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
sqlx::query!(
|
||||
@@ -967,10 +962,14 @@ async fn invite_user(
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace {
|
||||
email: nu.email.clone(),
|
||||
workspace: w_id,
|
||||
});
|
||||
if let Some(new_user_webhook) = NEW_USER_WEBHOOK.clone() {
|
||||
let _ = &HTTP_CLIENT
|
||||
.post(&new_user_webhook)
|
||||
.json(&serde_json::json!({"email" : &nu.email, "event": "new_invite"}))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| tracing::error!("Error sending new user webhook: {}", e.to_string()));
|
||||
}
|
||||
|
||||
Ok((
|
||||
StatusCode::CREATED,
|
||||
@@ -981,19 +980,12 @@ async fn invite_user(
|
||||
async fn add_user(
|
||||
Authed { username, is_admin, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(mut nu): Json<NewWorkspaceUser>,
|
||||
Json(nu): Json<NewWorkspaceUser>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
require_admin(is_admin, &username)?;
|
||||
nu.email = nu.email.to_lowercase();
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
if !VALID_USERNAME.is_match(&nu.username) {
|
||||
return Err(windmill_common::error::Error::BadRequest(format!(
|
||||
"Usermame can only contain alphanumeric characters and underscores"
|
||||
)));
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO usr
|
||||
@@ -1008,23 +1000,8 @@ async fn add_user(
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query_as!(
|
||||
Group,
|
||||
"INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
|
||||
&w_id,
|
||||
nu.username,
|
||||
"all",
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
webhook.send_instance_event(InstanceEvent::UserAddedWorkspace {
|
||||
workspace: w_id.clone(),
|
||||
email: nu.email.clone(),
|
||||
});
|
||||
|
||||
Ok((
|
||||
StatusCode::CREATED,
|
||||
format!("user with email {} added", nu.email),
|
||||
@@ -1087,7 +1064,6 @@ struct ScriptMetadata {
|
||||
schema: Option<Schema>,
|
||||
is_template: bool,
|
||||
lock: Vec<String>,
|
||||
kind: String,
|
||||
}
|
||||
|
||||
enum ArchiveImpl {
|
||||
@@ -1139,7 +1115,6 @@ impl ArchiveImpl {
|
||||
#[derive(Deserialize)]
|
||||
struct ArchiveQueryParams {
|
||||
archive_type: Option<String>,
|
||||
plain_secret: Option<bool>,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -1165,8 +1140,6 @@ where
|
||||
"edited_at",
|
||||
"edited_by",
|
||||
"archived",
|
||||
"has_draft",
|
||||
"draft_only",
|
||||
] {
|
||||
if obj.contains_key(key) {
|
||||
obj.remove(key);
|
||||
@@ -1187,7 +1160,7 @@ async fn tarball_workspace(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(ArchiveQueryParams { archive_type, plain_secret }): Query<ArchiveQueryParams>,
|
||||
Query(ArchiveQueryParams { archive_type }): Query<ArchiveQueryParams>,
|
||||
) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
|
||||
@@ -1253,7 +1226,6 @@ async fn tarball_workspace(
|
||||
description: script.description,
|
||||
schema: script.schema,
|
||||
is_template: script.is_template,
|
||||
kind: script.kind.to_string(),
|
||||
lock,
|
||||
};
|
||||
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
|
||||
@@ -1324,15 +1296,7 @@ async fn tarball_workspace(
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
|
||||
let mc = build_crypt(&mut db.begin().await?, &w_id).await?;
|
||||
|
||||
for mut var in variables {
|
||||
if plain_secret.unwrap_or(false) && var.value.is_some() && var.is_secret {
|
||||
var.value = Some(
|
||||
mc.decrypt_base64_to_string(var.value.unwrap())
|
||||
.map_err(|e| Error::InternalErr(e.to_string()))?,
|
||||
);
|
||||
}
|
||||
for var in variables {
|
||||
let var_str = &to_string_without_metadata(&var, false).unwrap();
|
||||
archive
|
||||
.write_to_archive(&var_str, &format!("{}.variable.json", var.path))
|
||||
|
||||
@@ -41,8 +41,8 @@ pub struct AuditLog {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
|
||||
db: E,
|
||||
pub async fn audit_log<'c>(
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
username: &str,
|
||||
operation: &str,
|
||||
action_kind: ActionKind,
|
||||
|
||||
@@ -11,16 +11,10 @@
|
||||
//! For users writing scripts to access their infrastructure with firewalls requiring incoming
|
||||
//! connections to be from whitelisted IP addresses.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::Result;
|
||||
|
||||
pub async fn get_ip() -> Result<String> {
|
||||
reqwest::ClientBuilder::new()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()?
|
||||
.get("https://hub.windmill.dev/getip")
|
||||
.send()
|
||||
reqwest::get("https://hub.windmill.dev/getip")
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
|
||||
@@ -125,7 +125,7 @@ pub enum FlowStatusModule {
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobResult {
|
||||
SingleJob(Uuid),
|
||||
ListJob(Vec<Uuid>),
|
||||
|
||||
@@ -6,12 +6,9 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, HashMap},
|
||||
time::Duration,
|
||||
};
|
||||
use std::{collections::HashMap, time::Duration};
|
||||
|
||||
use serde::{self, Deserialize, Serialize, Serializer};
|
||||
use serde::{self, Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
more_serde::{
|
||||
@@ -33,8 +30,6 @@ pub struct Flow {
|
||||
pub archived: bool,
|
||||
pub schema: Option<Schema>,
|
||||
pub extra_perms: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -49,9 +44,6 @@ pub struct ListableFlow {
|
||||
pub archived: bool,
|
||||
pub extra_perms: serde_json::Value,
|
||||
pub starred: bool,
|
||||
pub has_draft: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -62,7 +54,6 @@ pub struct NewFlow {
|
||||
pub description: String,
|
||||
pub value: serde_json::Value,
|
||||
pub schema: Option<Schema>,
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
|
||||
@@ -158,6 +149,9 @@ pub struct Suspend {
|
||||
pub struct FlowModule {
|
||||
#[serde(default = "default_id")]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
#[serde(alias = "input_transform")]
|
||||
pub input_transforms: HashMap<String, InputTransform>,
|
||||
pub value: FlowModuleValue,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_after_if: Option<StopAfterIf>,
|
||||
@@ -251,28 +245,18 @@ pub enum FlowModuleValue {
|
||||
},
|
||||
RawScript {
|
||||
#[serde(default)]
|
||||
#[serde(alias = "input_transform", serialize_with = "ordered_map")]
|
||||
#[serde(alias = "input_transform")]
|
||||
input_transforms: HashMap<String, InputTransform>,
|
||||
content: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
lock: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
path: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tag: Option<String>,
|
||||
language: ScriptLang,
|
||||
},
|
||||
Identity,
|
||||
}
|
||||
|
||||
fn ordered_map<S>(value: &HashMap<String, InputTransform>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
let ordered: BTreeMap<_, _> = value.iter().collect();
|
||||
ordered.serialize(serializer)
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ListFlowQuery {
|
||||
pub path_start: Option<String>,
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
error,
|
||||
flow_status::FlowStatus,
|
||||
flows::FlowValue,
|
||||
get_latest_deployed_hash_for_path,
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
};
|
||||
|
||||
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
|
||||
#[serde(rename_all(serialize = "lowercase"))]
|
||||
pub enum JobKind {
|
||||
Script,
|
||||
#[allow(non_camel_case_types)]
|
||||
Script_Hub,
|
||||
Preview,
|
||||
Dependencies,
|
||||
Flow,
|
||||
FlowPreview,
|
||||
Identity,
|
||||
FlowDependencies,
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
|
||||
pub struct QueuedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub scheduled_for: chrono::DateTime<chrono::Utc>,
|
||||
pub running: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logs: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_code: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_lock: Option<String>,
|
||||
pub canceled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_by: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub job_kind: JobKind,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<ScriptLang>,
|
||||
pub same_worker: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pre_run_error: Option<String>,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suspend: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub root_job: Option<Uuid>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub leaf_jobs: Option<serde_json::Value>,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn script_path(&self) -> &str {
|
||||
self.script_path
|
||||
.as_ref()
|
||||
.map(String::as_str)
|
||||
.unwrap_or("tmp/main")
|
||||
}
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
|
||||
self.raw_flow
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
|
||||
}
|
||||
|
||||
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
|
||||
self.flow_status
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for QueuedJob {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
workspace_id: "".to_string(),
|
||||
id: Uuid::default(),
|
||||
parent_job: None,
|
||||
created_by: "".to_string(),
|
||||
created_at: chrono::Utc::now(),
|
||||
started_at: None,
|
||||
scheduled_for: chrono::Utc::now(),
|
||||
running: false,
|
||||
script_hash: None,
|
||||
script_path: None,
|
||||
args: None,
|
||||
logs: None,
|
||||
raw_code: None,
|
||||
raw_lock: None,
|
||||
canceled: false,
|
||||
canceled_by: None,
|
||||
canceled_reason: None,
|
||||
last_ping: None,
|
||||
job_kind: JobKind::Identity,
|
||||
schedule_path: None,
|
||||
permissioned_as: "".to_string(),
|
||||
flow_status: None,
|
||||
raw_flow: None,
|
||||
is_flow_step: false,
|
||||
language: None,
|
||||
same_worker: false,
|
||||
pre_run_error: None,
|
||||
email: "".to_string(),
|
||||
visible_to_owner: false,
|
||||
suspend: None,
|
||||
mem_peak: None,
|
||||
root_job: None,
|
||||
leaf_jobs: None,
|
||||
tag: "deno".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobPayload {
|
||||
ScriptHub { path: String },
|
||||
ScriptHash { hash: ScriptHash, path: String },
|
||||
Code(RawCode),
|
||||
Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
|
||||
FlowDependencies { path: String },
|
||||
Flow(String),
|
||||
RawFlow { value: FlowValue, path: Option<String> },
|
||||
Identity,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, Debug)]
|
||||
pub struct RawCode {
|
||||
pub content: String,
|
||||
pub path: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub lock: Option<String>,
|
||||
}
|
||||
|
||||
type Tag = String;
|
||||
|
||||
pub async fn script_path_to_payload<'c>(
|
||||
script_path: &str,
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &String,
|
||||
) -> error::Result<(JobPayload, Option<Tag>)> {
|
||||
let (job_payload, tag) = if script_path.starts_with("hub/") {
|
||||
(JobPayload::ScriptHub { path: script_path.to_owned() }, None)
|
||||
} else {
|
||||
let (script_hash, tag) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() },
|
||||
tag,
|
||||
)
|
||||
};
|
||||
Ok((job_payload, tag))
|
||||
}
|
||||
|
||||
pub async fn script_hash_to_tag<'c>(
|
||||
script_hash: &ScriptHash,
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &String,
|
||||
) -> error::Result<Option<Tag>> {
|
||||
Ok(sqlx::query_scalar!(
|
||||
"select tag from script where hash = $1 AND workspace_id = $2",
|
||||
script_hash.0,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.flatten())
|
||||
}
|
||||
@@ -15,7 +15,6 @@ pub mod error;
|
||||
pub mod external_ip;
|
||||
pub mod flow_status;
|
||||
pub mod flows;
|
||||
pub mod jobs;
|
||||
pub mod more_serde;
|
||||
pub mod oauth2;
|
||||
pub mod schedule;
|
||||
@@ -31,26 +30,11 @@ pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
|
||||
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 3;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref METRICS_ADDR: Option<SocketAddr> = std::env::var("METRICS_ADDR")
|
||||
.ok()
|
||||
.map(|s| {
|
||||
s.parse::<bool>()
|
||||
.map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
|
||||
.or_else(|_| s.parse::<SocketAddr>().map(Some))
|
||||
})
|
||||
.transpose().ok()
|
||||
.flatten()
|
||||
.flatten();
|
||||
pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
|
||||
pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
|
||||
pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
}
|
||||
|
||||
#[cfg(feature = "tokio")]
|
||||
pub async fn shutdown_signal(
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> anyhow::Result<()> {
|
||||
pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
|
||||
use std::io;
|
||||
use tokio::signal::unix::SignalKind;
|
||||
|
||||
@@ -64,9 +48,6 @@ pub async fn shutdown_signal(
|
||||
tokio::select! {
|
||||
_ = terminate() => {},
|
||||
_ = tokio::signal::ctrl_c() => {},
|
||||
_ = rx.recv() => {
|
||||
tracing::info!("shutdown monitor received killpill");
|
||||
},
|
||||
}
|
||||
println!("signal received, starting graceful shutdown");
|
||||
let _ = tx.send(());
|
||||
@@ -77,31 +58,14 @@ pub async fn shutdown_signal(
|
||||
pub async fn serve_metrics(
|
||||
addr: SocketAddr,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
ready_worker_endpoint: bool,
|
||||
) -> Result<(), hyper::Error> {
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use axum::{routing::get, Router};
|
||||
use hyper::StatusCode;
|
||||
let router = Router::new().route("/metrics", get(metrics));
|
||||
|
||||
let router = if ready_worker_endpoint {
|
||||
router.route(
|
||||
"/ready",
|
||||
get(|| async {
|
||||
if IS_READY.load(Ordering::Relaxed) {
|
||||
(StatusCode::OK, "ready")
|
||||
} else {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "not ready")
|
||||
}
|
||||
}),
|
||||
)
|
||||
} else {
|
||||
router
|
||||
};
|
||||
|
||||
axum::Server::bind(&addr)
|
||||
.serve(router.into_make_service())
|
||||
.serve(
|
||||
Router::new()
|
||||
.route("/metrics", get(metrics))
|
||||
.into_make_service(),
|
||||
)
|
||||
.with_graceful_shutdown(async {
|
||||
rx.recv().await.ok();
|
||||
println!("Graceful shutdown of metrics");
|
||||
@@ -153,44 +117,23 @@ pub async fn connect(
|
||||
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
|
||||
}
|
||||
|
||||
type Tag = String;
|
||||
|
||||
pub async fn get_latest_deployed_hash_for_path<'c>(
|
||||
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
|
||||
w_id: &str,
|
||||
script_path: &str,
|
||||
) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
|
||||
let r_o = sqlx::query!(
|
||||
"select hash, tag from script where path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
|
||||
deleted = false AND archived = false AND lock IS not NULL AND lock_error_logs IS NULL)",
|
||||
script_path,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let script = utils::not_found_if_none(r_o, "script", script_path)?;
|
||||
|
||||
Ok((scripts::ScriptHash(script.hash), script.tag))
|
||||
}
|
||||
|
||||
// TODO: Move this elsewhere
|
||||
pub async fn get_latest_hash_for_path<'c>(
|
||||
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
|
||||
w_id: &str,
|
||||
script_path: &str,
|
||||
) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
|
||||
let r_o = sqlx::query!(
|
||||
"select hash, tag from script where path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
|
||||
deleted = false AND archived = false)",
|
||||
) -> error::Result<scripts::ScriptHash> {
|
||||
let script_hash_o = sqlx::query_scalar!(
|
||||
"select hash from script where path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND
|
||||
deleted = false",
|
||||
script_path,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let script = utils::not_found_if_none(r_o, "script", script_path)?;
|
||||
let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?;
|
||||
|
||||
Ok((scripts::ScriptHash(script.hash), script.tag))
|
||||
Ok(scripts::ScriptHash(script_hash))
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ pub struct Schedule {
|
||||
pub edited_by: String,
|
||||
pub edited_at: DateTime<chrono::Utc>,
|
||||
pub schedule: String,
|
||||
pub timezone: String,
|
||||
pub offset_: i32,
|
||||
pub enabled: bool,
|
||||
pub script_path: String,
|
||||
pub is_flow: bool,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
*/
|
||||
|
||||
use std::{
|
||||
fmt::{self, Display},
|
||||
fmt::Display,
|
||||
hash::{Hash, Hasher},
|
||||
};
|
||||
|
||||
@@ -17,7 +17,7 @@ use serde_json::to_string_pretty;
|
||||
|
||||
use crate::utils::StripPath;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash, Eq)]
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
|
||||
#[cfg_attr(feature = "sqlx", derive(sqlx::Type))]
|
||||
#[cfg_attr(
|
||||
feature = "sqlx",
|
||||
@@ -103,18 +103,6 @@ pub enum ScriptKind {
|
||||
Approval,
|
||||
}
|
||||
|
||||
impl Display for ScriptKind {
|
||||
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
|
||||
fmt.write_str(match self {
|
||||
ScriptKind::Trigger => "trigger",
|
||||
ScriptKind::Failure => "failure",
|
||||
ScriptKind::Script => "script",
|
||||
ScriptKind::Approval => "approval",
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
|
||||
pub struct Script {
|
||||
@@ -136,28 +124,27 @@ pub struct Script {
|
||||
pub lock_error_logs: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub kind: ScriptKind,
|
||||
pub tag: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
|
||||
pub struct ListableScript {
|
||||
pub workspace_id: String,
|
||||
pub hash: ScriptHash,
|
||||
pub path: String,
|
||||
pub parent_hashes: Option<ScriptHashes>,
|
||||
pub summary: String,
|
||||
pub description: String,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub archived: bool,
|
||||
pub deleted: bool,
|
||||
pub is_template: bool,
|
||||
pub extra_perms: serde_json::Value,
|
||||
pub lock_error_logs: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub kind: ScriptKind,
|
||||
pub starred: bool,
|
||||
pub tag: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub has_draft: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
pub has_deploy_errors: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
@@ -187,8 +174,6 @@ pub struct NewScript {
|
||||
pub lock: Option<Vec<String>>,
|
||||
pub language: ScriptLang,
|
||||
pub kind: Option<ScriptKind>,
|
||||
pub tag: Option<String>,
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
|
||||
|
||||
pub fn username_to_permissioned_as(user: &str) -> String {
|
||||
if user.contains('@') {
|
||||
user.to_string()
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use rand::{distributions::Alphanumeric, thread_rng, Rng};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
pub const MAX_PER_PAGE: usize = 10000;
|
||||
pub const DEFAULT_PER_PAGE: usize = 1000;
|
||||
@@ -19,8 +19,7 @@ pub struct Pagination {
|
||||
pub page: Option<usize>,
|
||||
pub per_page: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[derive(Deserialize)]
|
||||
pub struct StripPath(pub String);
|
||||
|
||||
impl StripPath {
|
||||
@@ -52,8 +51,8 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
|
||||
}
|
||||
|
||||
#[cfg(feature = "sqlx")]
|
||||
pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
|
||||
db: E,
|
||||
pub async fn now_from_db<'c>(
|
||||
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
|
||||
) -> Result<chrono::DateTime<chrono::Utc>> {
|
||||
Ok(sqlx::query_scalar!("SELECT now()")
|
||||
.fetch_one(db)
|
||||
@@ -119,9 +118,3 @@ pub fn rd_string(len: usize) -> String {
|
||||
.map(char::from)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn calculate_hash(s: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(s);
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
@@ -25,13 +25,8 @@ serde_json.workspace = true
|
||||
ulid.workspace = true
|
||||
uuid.workspace = true
|
||||
chrono.workspace = true
|
||||
chrono-tz.workspace = true
|
||||
hex.workspace = true
|
||||
reqwest.workspace = true
|
||||
lazy_static.workspace = true
|
||||
prometheus.workspace = true
|
||||
cron.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
tokio.workspace = true
|
||||
futures-core.workspace = true
|
||||
itertools.workspace = true
|
||||
|
||||
@@ -6,11 +6,11 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::{collections::HashMap, str::FromStr};
|
||||
|
||||
use anyhow::Context;
|
||||
use itertools::Itertools;
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
use tracing::{instrument, Instrument};
|
||||
use ulid::Ulid;
|
||||
@@ -20,14 +20,10 @@ use windmill_common::{
|
||||
error::{self, Error},
|
||||
flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
|
||||
flows::{FlowModule, FlowModuleValue, FlowValue},
|
||||
jobs::{JobKind, JobPayload, QueuedJob, RawCode},
|
||||
scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
|
||||
utils::StripPath,
|
||||
METRICS_ENABLED,
|
||||
};
|
||||
|
||||
use crate::QueueTransaction;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new()
|
||||
.user_agent("windmill/beta")
|
||||
@@ -51,62 +47,36 @@ lazy_static::lazy_static! {
|
||||
.unwrap();
|
||||
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
|
||||
|
||||
pub static ref ACCEPTED_TAGS: Vec<String> = std::env::var("WORKER_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect())
|
||||
.unwrap_or_else(|| vec![
|
||||
"deno".to_string(),
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
"other".to_string()]);
|
||||
|
||||
pub static ref ACCEPTED_TAGS_FILTER: String = format!(" AND ({})",
|
||||
ACCEPTED_TAGS.clone().into_iter().map(|x| format!("(tag = '{x}')")).join(" OR "));
|
||||
}
|
||||
|
||||
const MAX_FREE_EXECS: i32 = 1000;
|
||||
const MAX_FREE_CONCURRENT_RUNS: i32 = 15;
|
||||
const RSMQ_MAIN_QUEUE: &'static str = "main_queue";
|
||||
|
||||
pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
|
||||
pub async fn cancel_job<'c>(
|
||||
username: &str,
|
||||
reason: Option<String>,
|
||||
id: Uuid,
|
||||
w_id: &str,
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
rsmq: Option<R>,
|
||||
force_rerun: bool,
|
||||
) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
|
||||
let job_option = sqlx::query_scalar!(
|
||||
"UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0, running = CASE WHEN $3 THEN false ELSE running END WHERE id = $4 \
|
||||
AND workspace_id = $5 RETURNING id",
|
||||
"UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 \
|
||||
AND workspace_id = $4 RETURNING id",
|
||||
username,
|
||||
reason,
|
||||
force_rerun,
|
||||
id,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
if let Some(mut rsmq) = rsmq {
|
||||
rsmq.change_message_visibility(RSMQ_MAIN_QUEUE, &id.to_string(), 0)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))?;
|
||||
}
|
||||
|
||||
let mut jobs = job_option.map(|j| vec![j]).unwrap_or_default();
|
||||
while !jobs.is_empty() {
|
||||
let p_job = jobs.pop();
|
||||
let new_jobs = sqlx::query_scalar!(
|
||||
"UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, running = CASE WHEN $3 THEN false ELSE running END WHERE parent_job = $4 \
|
||||
AND workspace_id = $5 RETURNING id",
|
||||
"UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2 WHERE parent_job = $3 \
|
||||
AND workspace_id = $4 RETURNING id",
|
||||
username,
|
||||
reason,
|
||||
force_rerun,
|
||||
p_job,
|
||||
w_id
|
||||
)
|
||||
@@ -117,11 +87,10 @@ pub async fn cancel_job<'c, R: rsmq_async::RsmqConnection + Clone>(
|
||||
Ok((tx, job_option))
|
||||
}
|
||||
|
||||
pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
pub async fn pull(
|
||||
db: &Pool<Postgres>,
|
||||
whitelist_workspaces: Option<Vec<String>>,
|
||||
blacklist_workspaces: Option<Vec<String>>,
|
||||
rsmq: Option<R>,
|
||||
) -> windmill_common::error::Result<Option<QueuedJob>> {
|
||||
let mut workspaces_filter = String::new();
|
||||
if let Some(whitelist) = whitelist_workspaces {
|
||||
@@ -133,9 +102,6 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
.collect::<Vec<String>>()
|
||||
.join(",")
|
||||
));
|
||||
if let Some(_rsmq) = rsmq {
|
||||
todo!("REDIS: Implement workspace filters for redis");
|
||||
}
|
||||
}
|
||||
if let Some(blacklist) = blacklist_workspaces {
|
||||
workspaces_filter.push_str(&format!(
|
||||
@@ -146,51 +112,16 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
.collect::<Vec<String>>()
|
||||
.join(",")
|
||||
));
|
||||
if let Some(_rsmq) = rsmq {
|
||||
todo!("REDIS: Implement workspace filters for redis");
|
||||
}
|
||||
}
|
||||
|
||||
let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
|
||||
// TODO: REDIS: Race conditions / replace last_ping
|
||||
let msg = rsmq
|
||||
.pop_message::<Vec<u8>>(RSMQ_MAIN_QUEUE)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))?;
|
||||
|
||||
if let Some(msg) = msg {
|
||||
let uuid = Uuid::from_bytes_le(
|
||||
msg.message
|
||||
.try_into()
|
||||
.map_err(|_| anyhow::anyhow!("Failed to parsed Redis message"))?,
|
||||
);
|
||||
|
||||
sqlx::query_as::<_, QueuedJob>(
|
||||
"UPDATE queue
|
||||
SET running = true
|
||||
, started_at = coalesce(started_at, now())
|
||||
, last_ping = now()
|
||||
, suspend_until = null
|
||||
WHERE id = $1
|
||||
RETURNING *",
|
||||
)
|
||||
.bind(uuid)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
let accepted_tags_filter = &*ACCEPTED_TAGS_FILTER;
|
||||
/* Jobs can be started if they:
|
||||
* - haven't been started before,
|
||||
* running = false
|
||||
* - are flows with a step that needed resume,
|
||||
* suspend_until is non-null
|
||||
* and suspend = 0 when the resume messages are received
|
||||
* or suspend_until <= now() if it has timed out */
|
||||
sqlx::query_as::<_, QueuedJob>(&format!(
|
||||
"UPDATE queue
|
||||
/* Jobs can be started if they:
|
||||
* - haven't been started before,
|
||||
* running = false
|
||||
* - are flows with a step that needed resume,
|
||||
* suspend_until is non-null
|
||||
* and suspend = 0 when the resume messages are received
|
||||
* or suspend_until <= now() if it has timed out */
|
||||
let job: Option<QueuedJob> = sqlx::query_as::<_, QueuedJob>(&format!(
|
||||
"UPDATE queue
|
||||
SET running = true
|
||||
, started_at = coalesce(started_at, now())
|
||||
, last_ping = now()
|
||||
@@ -202,20 +133,17 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
AND scheduled_for <= now())
|
||||
OR (suspend_until IS NOT NULL
|
||||
AND ( suspend <= 0
|
||||
OR suspend_until <= now())))
|
||||
{workspaces_filter}
|
||||
{accepted_tags_filter}
|
||||
OR suspend_until <= now()))) {workspaces_filter}
|
||||
ORDER BY scheduled_for
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING *"
|
||||
))
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
};
|
||||
))
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
if job.is_some() && *METRICS_ENABLED {
|
||||
if job.is_some() {
|
||||
QUEUE_PULL_COUNT.inc();
|
||||
}
|
||||
|
||||
@@ -224,24 +152,55 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
|
||||
pub async fn get_result_by_id(
|
||||
db: Pool<Postgres>,
|
||||
mut skip_direct: bool,
|
||||
w_id: String,
|
||||
flow_id: Uuid,
|
||||
flow_id: String,
|
||||
node_id: String,
|
||||
) -> error::Result<serde_json::Value> {
|
||||
let job_result: Option<JobResult> = sqlx::query_scalar!(
|
||||
"SELECT leaf_jobs->$1::text FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3",
|
||||
node_id,
|
||||
flow_id,
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.flatten()
|
||||
.map(|x| serde_json::from_value(x).ok())
|
||||
.flatten();
|
||||
|
||||
let mut result_id: Option<JobResult> = None;
|
||||
let mut parent_id = Uuid::from_str(&flow_id).ok();
|
||||
while result_id.is_none() && parent_id.is_some() {
|
||||
if !skip_direct {
|
||||
let r = sqlx::query!(
|
||||
"SELECT flow_status, parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT flow_status, parent_job FROM queue WHERE id = $1 AND workspace_id = $2 ",
|
||||
parent_id.unwrap(),
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
if let Some(r) = r {
|
||||
let value = r
|
||||
.flow_status
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::InternalErr(format!("requiring a flow status value")))?
|
||||
.to_owned();
|
||||
parent_id = r.parent_job;
|
||||
let status_o = serde_json::from_value::<FlowStatus>(value).ok();
|
||||
result_id = status_o.and_then(|status| {
|
||||
status
|
||||
.modules
|
||||
.iter()
|
||||
.find(|m| m.id() == node_id)
|
||||
.and_then(|m| m.job_result())
|
||||
});
|
||||
} else {
|
||||
parent_id = None;
|
||||
}
|
||||
} else {
|
||||
let q_parent = sqlx::query_scalar!(
|
||||
"SELECT parent_job FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT parent_job FROM queue WHERE id = $1 AND workspace_id = $2",
|
||||
parent_id.unwrap(),
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.flatten();
|
||||
parent_id = q_parent;
|
||||
skip_direct = false
|
||||
}
|
||||
}
|
||||
let result_id = windmill_common::utils::not_found_if_none(
|
||||
job_result,
|
||||
result_id,
|
||||
"Flow result by id",
|
||||
format!("{}, {}", flow_id, node_id),
|
||||
)?;
|
||||
@@ -275,26 +234,24 @@ pub async fn get_result_by_id(
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn delete_job<'c, R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
mut tx: QueueTransaction<'c, R>,
|
||||
pub async fn delete_job(
|
||||
db: &Pool<Postgres>,
|
||||
w_id: &str,
|
||||
job_id: Uuid,
|
||||
) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
|
||||
if *METRICS_ENABLED {
|
||||
QUEUE_DELETE_COUNT.inc();
|
||||
}
|
||||
) -> windmill_common::error::Result<()> {
|
||||
QUEUE_DELETE_COUNT.inc();
|
||||
let job_removed = sqlx::query_scalar!(
|
||||
"DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1",
|
||||
w_id,
|
||||
job_id
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Error during deletion of job {job_id}: {e}")))?
|
||||
.unwrap_or(0)
|
||||
== 1;
|
||||
tracing::debug!("Job {job_id} deleted: {job_removed}");
|
||||
Ok(tx)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_queued_job<'c>(
|
||||
@@ -313,9 +270,9 @@ pub async fn get_queued_job<'c>(
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
// #[instrument(level = "trace", skip_all)]
|
||||
pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
mut tx: QueueTransaction<'c, R>,
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn push<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
workspace_id: &str,
|
||||
job_payload: JobPayload,
|
||||
args: serde_json::Map<String, serde_json::Value>,
|
||||
@@ -325,19 +282,18 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
|
||||
schedule_path: Option<String>,
|
||||
parent_job: Option<Uuid>,
|
||||
root_job: Option<Uuid>,
|
||||
is_flow_step: bool,
|
||||
mut same_worker: bool,
|
||||
pre_run_error: Option<&windmill_common::error::Error>,
|
||||
visible_to_owner: bool,
|
||||
mut tag: Option<String>,
|
||||
) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
|
||||
) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
|
||||
let scheduled_for = scheduled_for_o.unwrap_or_else(chrono::Utc::now);
|
||||
let args_json = serde_json::Value::Object(args);
|
||||
let job_id: Uuid = Ulid::new().into();
|
||||
|
||||
if cfg!(feature = "enterprise") {
|
||||
let premium_workspace = *CLOUD_HOSTED
|
||||
&& sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
|
||||
let premium_workspace =
|
||||
sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
@@ -560,6 +516,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
modules.push(FlowModule {
|
||||
id: format!("{}-v", flow.modules[flow.modules.len() - 1].id),
|
||||
value: FlowModuleValue::Identity,
|
||||
input_transforms: HashMap::new(),
|
||||
stop_after_if: None,
|
||||
summary: Some(
|
||||
"Virtual module needed for suspend/sleep when last module".to_string(),
|
||||
@@ -577,35 +534,12 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
.unwrap_or_else(|| (None, None));
|
||||
|
||||
let flow_status = raw_flow.as_ref().map(FlowStatus::new);
|
||||
|
||||
let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
|
||||
"dependency".to_string()
|
||||
} else if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
|
||||
"flow".to_string()
|
||||
} else if job_kind == JobKind::Identity {
|
||||
// identity is a light script, deno is too
|
||||
"deno".to_string()
|
||||
} else if job_kind == JobKind::Script_Hub {
|
||||
"hub".to_string()
|
||||
} else {
|
||||
if tag == Some("".to_string()) {
|
||||
tag = None;
|
||||
}
|
||||
tag.unwrap_or_else(|| {
|
||||
language
|
||||
.as_ref()
|
||||
.map(|x| x.as_str())
|
||||
.unwrap_or_else(|| "deno")
|
||||
.to_string()
|
||||
})
|
||||
};
|
||||
|
||||
let uuid = sqlx::query_scalar!(
|
||||
"INSERT INTO queue
|
||||
(workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for,
|
||||
script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
|
||||
flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24) \
|
||||
flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22) \
|
||||
RETURNING id",
|
||||
workspace_id,
|
||||
job_id,
|
||||
@@ -613,7 +547,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
parent_job,
|
||||
user,
|
||||
permissioned_as,
|
||||
scheduled_for_o,
|
||||
scheduled_for,
|
||||
script_hash,
|
||||
script_path.clone(),
|
||||
raw_code,
|
||||
@@ -628,17 +562,13 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
same_worker,
|
||||
pre_run_error.map(|e| e.to_string()),
|
||||
email,
|
||||
visible_to_owner,
|
||||
root_job,
|
||||
tag
|
||||
visible_to_owner
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id}: {e}")))?;
|
||||
// TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction.
|
||||
if *METRICS_ENABLED {
|
||||
QUEUE_PUSH_COUNT.inc();
|
||||
}
|
||||
QUEUE_PUSH_COUNT.inc();
|
||||
|
||||
{
|
||||
let uuid_string = job_id.to_string();
|
||||
@@ -673,10 +603,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
.instrument(tracing::info_span!("job_run", email = &email))
|
||||
.await?;
|
||||
}
|
||||
if let Some(ref mut rsmq) = tx.rsmq {
|
||||
rsmq.send_message(job_id.to_bytes_le().to_vec(), scheduled_for_o);
|
||||
}
|
||||
|
||||
Ok((uuid, tx))
|
||||
}
|
||||
|
||||
@@ -698,3 +624,113 @@ pub async fn get_hub_script(
|
||||
.await
|
||||
.map(|e| e)
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
|
||||
pub struct QueuedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub scheduled_for: chrono::DateTime<chrono::Utc>,
|
||||
pub running: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logs: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_code: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_lock: Option<String>,
|
||||
pub canceled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_by: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub job_kind: JobKind,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<ScriptLang>,
|
||||
pub same_worker: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pre_run_error: Option<String>,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suspend: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn script_path(&self) -> &str {
|
||||
self.script_path
|
||||
.as_ref()
|
||||
.map(String::as_str)
|
||||
.unwrap_or("NO_FLOW_PATH")
|
||||
}
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
|
||||
self.raw_flow
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
|
||||
}
|
||||
|
||||
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
|
||||
self.flow_status
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
|
||||
#[serde(rename_all(serialize = "lowercase"))]
|
||||
pub enum JobKind {
|
||||
Script,
|
||||
#[allow(non_camel_case_types)]
|
||||
Script_Hub,
|
||||
Preview,
|
||||
Dependencies,
|
||||
Flow,
|
||||
FlowPreview,
|
||||
Identity,
|
||||
FlowDependencies,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobPayload {
|
||||
ScriptHub { path: String },
|
||||
ScriptHash { hash: ScriptHash, path: String },
|
||||
Code(RawCode),
|
||||
Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
|
||||
FlowDependencies { path: String },
|
||||
Flow(String),
|
||||
RawFlow { value: FlowValue, path: Option<String> },
|
||||
Identity,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, Debug)]
|
||||
pub struct RawCode {
|
||||
pub content: String,
|
||||
pub path: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub lock: Option<String>,
|
||||
}
|
||||
|
||||
@@ -7,8 +7,6 @@
|
||||
*/
|
||||
|
||||
mod jobs;
|
||||
mod queue_transaction;
|
||||
pub mod schedule;
|
||||
|
||||
pub use jobs::*;
|
||||
pub use queue_transaction::*;
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
use std::fmt::Debug;
|
||||
|
||||
use futures_core::{future::BoxFuture, stream::BoxStream};
|
||||
use rsmq_async::{RedisBytes, RsmqConnection};
|
||||
use sqlx::{Postgres, Transaction};
|
||||
|
||||
pub enum RedisOp {
|
||||
SendMessage(RedisBytes, Option<chrono::DateTime<chrono::Utc>>),
|
||||
DeleteMessage(String),
|
||||
}
|
||||
|
||||
impl RedisOp {
|
||||
pub async fn apply<R: RsmqConnection>(self, rsmq: &mut R) -> Result<(), rsmq_async::RsmqError> {
|
||||
match self {
|
||||
RedisOp::SendMessage(bytes, time) => {
|
||||
rsmq.send_message(
|
||||
"main_queue",
|
||||
bytes,
|
||||
time.map(|t| (t - chrono::Utc::now()).num_seconds())
|
||||
.and_then(|e| e.try_into().ok()),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
RedisOp::DeleteMessage(id) => {
|
||||
rsmq.delete_message("main_queue", &id).await?;
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RedisTransaction<R: RsmqConnection> {
|
||||
rsmq: R,
|
||||
queued_ops: Vec<RedisOp>,
|
||||
}
|
||||
|
||||
impl<R: RsmqConnection> From<R> for RedisTransaction<R> {
|
||||
fn from(value: R) -> Self {
|
||||
Self { rsmq: value, queued_ops: Vec::new() }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: RsmqConnection> RedisTransaction<R> {
|
||||
pub async fn commit(self) -> Result<(), rsmq_async::RsmqError> {
|
||||
let mut rsmq = self.rsmq;
|
||||
for op in self.queued_ops {
|
||||
op.apply(&mut rsmq).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn send_message<E: Into<RedisBytes>>(
|
||||
&mut self,
|
||||
bytes: E,
|
||||
delay_until: Option<chrono::DateTime<chrono::Utc>>,
|
||||
) {
|
||||
self.queued_ops
|
||||
.push(RedisOp::SendMessage(bytes.into(), delay_until))
|
||||
}
|
||||
|
||||
pub fn delete_message(&mut self, id: String) {
|
||||
self.queued_ops.push(RedisOp::DeleteMessage(id))
|
||||
}
|
||||
}
|
||||
|
||||
pub struct QueueTransaction<'c, R: RsmqConnection> {
|
||||
pub rsmq: Option<RedisTransaction<R>>,
|
||||
transaction: Transaction<'c, Postgres>,
|
||||
}
|
||||
|
||||
impl<'c, R: RsmqConnection> From<(Option<R>, Transaction<'c, Postgres>)>
|
||||
for QueueTransaction<'c, R>
|
||||
{
|
||||
fn from(value: (Option<R>, Transaction<'c, Postgres>)) -> Self {
|
||||
Self { rsmq: value.0.map(|e| e.into()), transaction: value.1 }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'c, R: RsmqConnection> Debug for QueueTransaction<'c, R> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("QueueTransaction")
|
||||
.field("rsmq", &self.rsmq.as_ref().map(|_| ())) // do not require R: Debug
|
||||
.field("transaction", &self.transaction)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'c, R: RsmqConnection> QueueTransaction<'c, R> {
|
||||
pub async fn commit(self) -> Result<(), windmill_common::error::Error> {
|
||||
self.transaction.commit().await?;
|
||||
if let Some(rsmq) = self.rsmq {
|
||||
rsmq.commit().await.map_err(|e| anyhow::anyhow!(e))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn transaction_mut<'a>(&'a mut self) -> &'a mut Transaction<'c, Postgres> {
|
||||
&mut self.transaction
|
||||
}
|
||||
}
|
||||
|
||||
impl<'c, 'b, R: RsmqConnection + Send> sqlx::Executor<'b> for &'b mut QueueTransaction<'c, R> {
|
||||
type Database = Postgres;
|
||||
|
||||
fn fetch_many<'e, 'q: 'e, E: 'q>(
|
||||
self,
|
||||
query: E,
|
||||
) -> BoxStream<
|
||||
'e,
|
||||
Result<
|
||||
sqlx::Either<
|
||||
<Self::Database as sqlx::Database>::QueryResult,
|
||||
<Self::Database as sqlx::Database>::Row,
|
||||
>,
|
||||
sqlx::Error,
|
||||
>,
|
||||
>
|
||||
where
|
||||
'b: 'e,
|
||||
E: sqlx::Execute<'q, Self::Database>,
|
||||
{
|
||||
self.transaction.fetch_many(query)
|
||||
}
|
||||
|
||||
fn fetch_optional<'e, 'q: 'e, E: 'q>(
|
||||
self,
|
||||
query: E,
|
||||
) -> BoxFuture<'e, Result<Option<<Self::Database as sqlx::Database>::Row>, sqlx::Error>>
|
||||
where
|
||||
'b: 'e,
|
||||
E: sqlx::Execute<'q, Self::Database>,
|
||||
{
|
||||
self.transaction.fetch_optional(query)
|
||||
}
|
||||
|
||||
fn prepare_with<'e, 'q: 'e>(
|
||||
self,
|
||||
sql: &'q str,
|
||||
parameters: &'e [<Self::Database as sqlx::Database>::TypeInfo],
|
||||
) -> BoxFuture<
|
||||
'e,
|
||||
Result<<Self::Database as sqlx::database::HasStatement<'q>>::Statement, sqlx::Error>,
|
||||
>
|
||||
where
|
||||
'b: 'e,
|
||||
{
|
||||
self.transaction.prepare_with(sql, parameters)
|
||||
}
|
||||
|
||||
fn describe<'e, 'q: 'e>(
|
||||
self,
|
||||
sql: &'q str,
|
||||
) -> BoxFuture<'e, Result<sqlx::Describe<Self::Database>, sqlx::Error>>
|
||||
where
|
||||
'b: 'e,
|
||||
{
|
||||
self.transaction.describe(sql)
|
||||
}
|
||||
}
|
||||
@@ -6,11 +6,10 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::push;
|
||||
use crate::QueueTransaction;
|
||||
use sqlx::{query_scalar, Postgres, Transaction};
|
||||
use std::str::FromStr;
|
||||
use windmill_common::jobs::JobPayload;
|
||||
|
||||
use chrono::Duration;
|
||||
use sqlx::{query_scalar, Postgres, Transaction};
|
||||
use windmill_common::{
|
||||
error::{self, Result},
|
||||
schedule::Schedule,
|
||||
@@ -18,28 +17,22 @@ use windmill_common::{
|
||||
utils::{now_from_db, StripPath},
|
||||
};
|
||||
|
||||
pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
mut tx: QueueTransaction<'c, R>,
|
||||
use crate::{push, JobPayload};
|
||||
|
||||
pub async fn push_scheduled_job<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
schedule: Schedule,
|
||||
) -> Result<QueueTransaction<'c, R>> {
|
||||
) -> Result<Transaction<'c, Postgres>> {
|
||||
let sched = cron::Schedule::from_str(&schedule.schedule)
|
||||
.map_err(|e| error::Error::BadRequest(e.to_string()))?;
|
||||
|
||||
let tz = chrono_tz::Tz::from_str(&schedule.timezone)
|
||||
.map_err(|e| error::Error::BadRequest(e.to_string()))?;
|
||||
|
||||
let now = now_from_db(&mut tx).await?.with_timezone(&tz);
|
||||
|
||||
let offset = Duration::minutes(schedule.offset_.into());
|
||||
let now = now_from_db(&mut tx).await?;
|
||||
let next = sched
|
||||
.after(&now)
|
||||
.after(&(now - offset + Duration::seconds(1)))
|
||||
.next()
|
||||
.expect("a schedule should have a next event");
|
||||
|
||||
// println!("next event ({:?}): {}", tz, next);
|
||||
// println!("next event(UTC): {}", next.with_timezone(&chrono::Utc));
|
||||
|
||||
// Scheduled events must be stored in the database in UTC
|
||||
let next = next.with_timezone(&chrono::Utc);
|
||||
.expect("a schedule should have a next event")
|
||||
+ offset;
|
||||
|
||||
let already_exists: bool = query_scalar!(
|
||||
"SELECT EXISTS (SELECT 1 FROM queue WHERE workspace_id = $1 AND schedule_path = $2 AND scheduled_for = $3)",
|
||||
@@ -67,30 +60,21 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
}
|
||||
}
|
||||
|
||||
let (payload, tag) = if schedule.is_flow {
|
||||
(JobPayload::Flow(schedule.script_path), None)
|
||||
let payload = if schedule.is_flow {
|
||||
JobPayload::Flow(schedule.script_path)
|
||||
} else {
|
||||
let (hash, tag) = windmill_common::get_latest_hash_for_path(
|
||||
tx.transaction_mut(),
|
||||
&schedule.workspace_id,
|
||||
&schedule.script_path,
|
||||
)
|
||||
.await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash, path: schedule.script_path },
|
||||
tag,
|
||||
)
|
||||
JobPayload::ScriptHash {
|
||||
hash: windmill_common::get_latest_hash_for_path(
|
||||
&mut tx,
|
||||
&schedule.workspace_id,
|
||||
&schedule.script_path,
|
||||
)
|
||||
.await?,
|
||||
path: schedule.script_path,
|
||||
}
|
||||
};
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
|
||||
&schedule.workspace_id,
|
||||
&schedule.path
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
let (_, tx) = push(
|
||||
let (_, mut tx) = push(
|
||||
tx,
|
||||
&schedule.workspace_id,
|
||||
payload,
|
||||
@@ -101,15 +85,20 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
Some(next),
|
||||
Some(schedule.path.clone()),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
Ok(tx) // TODO: Bubble up pushed UUID from here
|
||||
sqlx::query!(
|
||||
"UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2",
|
||||
&schedule.workspace_id,
|
||||
&schedule.path
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
Ok(tx)
|
||||
}
|
||||
|
||||
pub async fn get_schedule_opt<'c>(
|
||||
|
||||
@@ -48,5 +48,3 @@ deno_core.workspace = true
|
||||
const_format.workspace = true
|
||||
git-version.workspace = true
|
||||
dyn-iter.workspace = true
|
||||
once_cell.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
|
||||
135
backend/windmill-worker/nsjail/run.deno.config.proto
Normal file
135
backend/windmill-worker/nsjail/run.deno.config.proto
Normal file
@@ -0,0 +1,135 @@
|
||||
name: "deno run script"
|
||||
|
||||
mode: ONCE
|
||||
hostname: "deno"
|
||||
log_level: ERROR
|
||||
|
||||
rlimit_as: 16000
|
||||
rlimit_cpu: 1000
|
||||
rlimit_fsize: 1000
|
||||
rlimit_nofile: 10000
|
||||
|
||||
cwd: "/tmp"
|
||||
|
||||
|
||||
clone_newnet: false
|
||||
clone_newuser: {CLONE_NEWUSER}
|
||||
|
||||
keep_caps: false
|
||||
keep_env: true
|
||||
mount_proc: true
|
||||
|
||||
mount {
|
||||
src: "/bin"
|
||||
dst: "/bin"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/lib"
|
||||
dst: "/lib"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
|
||||
mount {
|
||||
src: "/lib64"
|
||||
dst: "/lib64"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
|
||||
mount {
|
||||
src: "/usr"
|
||||
dst: "/usr"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/dev/null"
|
||||
dst: "/dev/null"
|
||||
is_bind: true
|
||||
rw: true
|
||||
}
|
||||
|
||||
mount {
|
||||
dst: "/tmp"
|
||||
fstype: "tmpfs"
|
||||
rw: true
|
||||
options: "size=500000000"
|
||||
}
|
||||
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/inner.ts"
|
||||
dst: "/tmp/inner.ts"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/main.ts"
|
||||
dst: "/tmp/main.ts"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/import_map.json"
|
||||
dst: "/tmp/import_map.json"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/lock.json"
|
||||
dst: "/tmp/lock.json"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/args.json"
|
||||
dst: "/tmp/args.json"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/result.json"
|
||||
dst: "/tmp/result.json"
|
||||
is_bind: true
|
||||
rw: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/etc"
|
||||
dst: "/etc"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/dev/random"
|
||||
dst: "/dev/random"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "/dev/urandom"
|
||||
dst: "/dev/urandom"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{CACHE_DIR}"
|
||||
dst: "/tmp/.cache/deno"
|
||||
is_bind: true
|
||||
rw: true
|
||||
mandatory: false
|
||||
}
|
||||
|
||||
{SHARED_MOUNT}
|
||||
|
||||
iface_no_lo: true
|
||||
|
||||
envar: "DENO_DIR=/tmp/.cache/deno"
|
||||
envar: "NO_COLOR=true"
|
||||
envar: "HOME=/tmp"
|
||||
|
||||
|
||||
|
||||
@@ -66,8 +66,8 @@ mount {
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/{MAIN}.py"
|
||||
dst: "/tmp/{MAIN}.py"
|
||||
src: "{JOB_DIR}/inner.py"
|
||||
dst: "/tmp/inner.py"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
@@ -79,8 +79,8 @@ mount {
|
||||
}
|
||||
|
||||
mount {
|
||||
src: "{JOB_DIR}/wrapper.py"
|
||||
dst: "/tmp/wrapper.py"
|
||||
src: "{JOB_DIR}/main.py"
|
||||
dst: "/tmp/main.py"
|
||||
is_bind: true
|
||||
}
|
||||
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{fs::File, io::AsyncReadExt};
|
||||
use windmill_common::error::{self, Error};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
use crate::MAX_RESULT_SIZE;
|
||||
|
||||
pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
|
||||
let mut file = File::open(format!("{job_dir}/result.json")).await?;
|
||||
let mut content = "".to_string();
|
||||
file.read_to_string(&mut content).await?;
|
||||
if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
|
||||
return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
|
||||
If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
|
||||
}
|
||||
serde_json::from_str(&content)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn set_logs(logs: &str, id: &uuid::Uuid, db: &Pool<Postgres>) {
|
||||
if sqlx::query!(
|
||||
"UPDATE queue SET logs = $1 WHERE id = $2",
|
||||
logs.to_owned(),
|
||||
id
|
||||
)
|
||||
.execute(db)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::error!(%id, "error updating logs for id {id}")
|
||||
};
|
||||
}
|
||||
|
||||
pub fn capitalize(s: &str) -> String {
|
||||
let mut c = s.chars();
|
||||
match c.next() {
|
||||
None => String::new(),
|
||||
Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
|
||||
}
|
||||
}
|
||||
@@ -1,521 +0,0 @@
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::{ROOT_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_CACHE_RATE, TAR_PIP_TMP_CACHE_DIR, TMP_DIR};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use itertools::Itertools;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use rand::Rng;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use std::process::Stdio;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use tokio::{process::Command, sync::mpsc::Sender, time::Instant};
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_common::error;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
const TAR_CACHE_FILENAME: &str = "denogocache.tar";
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<()> {
|
||||
tracing::info!("Started building and pushing piptar {folder}");
|
||||
let start = Instant::now();
|
||||
let folder_name = folder.split("/").last().unwrap();
|
||||
let tar_path = format!("{TAR_PIP_TMP_CACHE_DIR}/{folder_name}.tar",);
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"tar",
|
||||
vec!["-c", "-f", &tar_path, "-C", &folder, "."],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to tar cache. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
let tar_metadata = tokio::fs::metadata(&tar_path).await;
|
||||
if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
|
||||
tracing::info!("Failed to tar cache: {folder}");
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to tar cache: {folder}"
|
||||
)));
|
||||
}
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&tar_path,
|
||||
&format!(":s3,env_auth=true:{bucket}/tar/pip/{folder_name}.tar"),
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--s3-no-check-bucket",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy piptar {folder} to bucket. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying piptar {folder} to bucket {bucket} as tar, took: {:?}s. Size of tar: {}",
|
||||
start.elapsed().as_secs(),
|
||||
tar_metadata.unwrap().len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> {
|
||||
use tokio::fs::metadata;
|
||||
let folder_name = folder.split("/").last().unwrap();
|
||||
|
||||
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
|
||||
|
||||
let start = Instant::now();
|
||||
let tar_path = format!("tar/pip/{folder_name}.tar");
|
||||
let target = format!("{ROOT_TMP_CACHE_DIR}/{tar_path}.single");
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&format!(":s3,env_auth=true:{bucket}/{tar_path}"),
|
||||
&target,
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!(
|
||||
"Failed to copy tar {folder_name} from bucket. Error: {:?}",
|
||||
e
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if metadata(&target).await.is_err() {
|
||||
tracing::info!(
|
||||
"piptar {folder_name} not found in bucket. Took {:?}ms",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"tar {folder_name} does not exist in bucket"
|
||||
)));
|
||||
}
|
||||
|
||||
extract_pip_tar(&target, &folder).await?;
|
||||
tokio::fs::remove_file(&target).await?;
|
||||
tracing::info!(
|
||||
"Finished pulling and extracting {folder_name}. Took {:?}ms",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn cache_global(bucket: &str, tx: Sender<()>) -> error::Result<()> {
|
||||
copy_cache_from_bucket(bucket, tx).await?;
|
||||
copy_cache_to_bucket(bucket).await?;
|
||||
|
||||
// this is to prevent excessive tar upload. 1/100*15min = each worker sync its tar once per day on average
|
||||
if rand::thread_rng().gen_range(0..*TAR_CACHE_RATE) == 0 {
|
||||
copy_cache_to_bucket_as_tar(bucket).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_from_bucket(bucket: &str, tx: Sender<()>) -> error::Result<()> {
|
||||
tracing::info!("Copying cache from bucket in the background {bucket}");
|
||||
let bucket = bucket.to_string();
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copy",
|
||||
&format!(":s3,env_auth=true:{bucket}"),
|
||||
&ROOT_TMP_CACHE_DIR,
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"+ tar/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy cache from bucket. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying cache from bucket {bucket}, took {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
|
||||
tx.send(()).await.expect("can send copy cache signal");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
|
||||
tracing::info!("Copying cache to bucket {bucket}");
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copy",
|
||||
&ROOT_TMP_CACHE_DIR,
|
||||
&format!(":s3,env_auth=true:{bucket}"),
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy cache to bucket. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
tracing::info!(
|
||||
"Finished copying cache to bucket {bucket}, took: {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
|
||||
tracing::info!("Copying cache to bucket {bucket} as tar");
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"tar",
|
||||
vec![
|
||||
"-c",
|
||||
"-f",
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
"go",
|
||||
"deno",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to tar cache. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
let tar_metadata =
|
||||
tokio::fs::metadata(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await;
|
||||
if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
|
||||
tracing::info!("Failed to tar cache");
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
&format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--s3-no-check-bucket",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy tar to bucket. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
|
||||
{
|
||||
tracing::info!("Failed to remove tar cache. Error: {:?}", e);
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying cache to bucket {bucket} as tar, took: {:?}s. Size of new tar: {}",
|
||||
start.elapsed().as_secs(),
|
||||
tar_metadata.unwrap().len()
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
|
||||
use tokio::fs::metadata;
|
||||
|
||||
tracing::info!("Copying denogo cache from bucket {bucket} as tar");
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed copying denogo tar from cache. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_CACHE_DIR,
|
||||
"tar",
|
||||
vec![
|
||||
"-xpvf",
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to untar denogo. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
let denogen = format!("{ROOT_TMP_CACHE_DIR}deno/gen/file");
|
||||
if metadata(&denogen).await.is_ok() {
|
||||
let _ = tokio::fs::remove_dir_all(denogen).await;
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
|
||||
{
|
||||
tracing::info!("Failed to remove denotar cache. Error: {:?}", e);
|
||||
return;
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying denogotar from bucket {bucket} as tar, took: {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_all_piptars_from_bucket(bucket: &str) {
|
||||
tracing::info!("Copying all piptars cache from bucket {bucket}");
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copy",
|
||||
&format!(":s3,env_auth=true:{bucket}/tar/pip/"),
|
||||
&TAR_PIP_TMP_CACHE_DIR,
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed transferring all piptars from cache. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished transferring piptars from bucket {bucket} as tar, took: {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
}
|
||||
|
||||
// async fn check_if_bucket_syncable(bucket: &str) -> bool {
|
||||
// match Command::new("rclone")
|
||||
// .arg("lsf")
|
||||
// .arg(format!(":s3,env_auth=true:{bucket}/NOSYNC"))
|
||||
|
||||
// .arg("-vv")
|
||||
// .arg("--fast-list")
|
||||
// .stdin(Stdio::null())
|
||||
// .stdout(Stdio::null())
|
||||
// .output()
|
||||
// .await;
|
||||
// return true;
|
||||
// }
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_tmp_cache_to_cache() -> error::Result<()> {
|
||||
let start: Instant = Instant::now();
|
||||
execute_command(
|
||||
TMP_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"sync",
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
ROOT_CACHE_DIR,
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying local tmp cache to local cache. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
|
||||
if let Err(e) = untar_all_piptars().await {
|
||||
tracing::info!("Failed to untar piptars. Error: {:?}", e);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn untar_all_piptars() -> error::Result<()> {
|
||||
use tokio::fs::{self, metadata};
|
||||
|
||||
use crate::PIP_CACHE_DIR;
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
|
||||
let mut entries = fs::read_dir(TAR_PIP_TMP_CACHE_DIR).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
if let Err(e) = {
|
||||
let entry_path = entry.path();
|
||||
let path = entry_path.to_str().expect("Could not convert path to str");
|
||||
let folder = format!(
|
||||
"{PIP_CACHE_DIR}/{}",
|
||||
path.split('/')
|
||||
.last()
|
||||
.unwrap()
|
||||
.strip_suffix(".tar")
|
||||
.unwrap()
|
||||
);
|
||||
if metadata(&folder).await.is_ok() {
|
||||
continue;
|
||||
}
|
||||
extract_pip_tar(&path, &folder).await?;
|
||||
Ok(()) as error::Result<()>
|
||||
} {
|
||||
tracing::info!("Failed to extract pip tar. Error: {:?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished untarring all piptars. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn extract_pip_tar(tar: &str, folder: &str) -> error::Result<()> {
|
||||
use tokio::fs;
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
fs::create_dir_all(&folder).await?;
|
||||
if let Err(e) = execute_command(&folder, "tar", vec!["-xpvf", tar]).await {
|
||||
tracing::info!("Failed to untar piptar. Error: {:?}", e);
|
||||
fs::remove_dir_all(&folder).await?;
|
||||
return Err(e);
|
||||
}
|
||||
tracing::info!(
|
||||
"Finished extracting pip tar {folder}. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {
|
||||
let start: Instant = Instant::now();
|
||||
execute_command(
|
||||
TMP_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"sync",
|
||||
ROOT_CACHE_DIR,
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
"Finished copying local cache to local tmp cache. Took {}ms",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn execute_command(dir: &str, command: &str, args: Vec<&str>) -> error::Result<()> {
|
||||
match Command::new(command)
|
||||
.current_dir(dir)
|
||||
.args(args.clone())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.spawn()
|
||||
{
|
||||
Ok(mut h) => {
|
||||
if !h.wait().await.unwrap().success() {
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to apply {command} with args: {}",
|
||||
args.iter().join(" ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to apply {command} with args: {}. Error: {e:?}",
|
||||
args.iter().join(" ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,373 +0,0 @@
|
||||
use std::process::Stdio;
|
||||
|
||||
use itertools::Itertools;
|
||||
use tokio::{
|
||||
fs::{DirBuilder, File},
|
||||
io::AsyncReadExt,
|
||||
process::Command,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
jobs::QueuedJob,
|
||||
utils::calculate_hash,
|
||||
};
|
||||
use windmill_parser_go::parse_go_imports;
|
||||
|
||||
use crate::{
|
||||
common::{capitalize, read_result, set_logs},
|
||||
create_args_and_out_file, get_reserved_variables, handle_child, write_file,
|
||||
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GO_CACHE_DIR, HOME_ENV,
|
||||
NETRC, NSJAIL_PATH, PATH_ENV,
|
||||
};
|
||||
|
||||
const GO_REQ_SPLITTER: &str = "//go.sum\n";
|
||||
const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref GO_PATH: String = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_go_job(
|
||||
logs: &mut String,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
inner_content: &str,
|
||||
job_dir: &str,
|
||||
requirements_o: Option<String>,
|
||||
shared_mount: &str,
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
//go does not like executing modules at temp root
|
||||
let job_dir = &format!("{job_dir}/go");
|
||||
let (skip_go_mod, skip_tidy) = if let Some(requirements) = requirements_o {
|
||||
gen_go_mod(inner_content, job_dir, &requirements).await?
|
||||
} else {
|
||||
(false, false)
|
||||
};
|
||||
|
||||
logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
|
||||
install_go_dependencies(
|
||||
&job.id,
|
||||
inner_content,
|
||||
logs,
|
||||
job_dir,
|
||||
db,
|
||||
true,
|
||||
skip_go_mod,
|
||||
skip_tidy,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
let client = &client.get_authed().await;
|
||||
create_args_and_out_file(client, job, job_dir).await?;
|
||||
{
|
||||
let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
|
||||
drop(inner_content);
|
||||
|
||||
const WRAPPER_CONTENT: &str = r#"package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"fmt"
|
||||
"mymod/inner"
|
||||
)
|
||||
|
||||
func main() {{
|
||||
|
||||
dat, err := os.ReadFile("args.json")
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
|
||||
var req inner.Req
|
||||
|
||||
if err := json.Unmarshal(dat, &req); err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
|
||||
res, err := inner.Run(req)
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
res_json, err := json.Marshal(res)
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
_, err = f.WriteString(string(res_json))
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
}}"#;
|
||||
|
||||
write_file(job_dir, "main.go", WRAPPER_CONTENT).await?;
|
||||
|
||||
{
|
||||
let spread = &sig
|
||||
.args
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|x| format!("req.{}", capitalize(&x.name)))
|
||||
.join(", ");
|
||||
let req_body = &sig
|
||||
.args
|
||||
.into_iter()
|
||||
.map(|x| {
|
||||
format!(
|
||||
"{} {} `json:\"{}\"`",
|
||||
capitalize(&x.name),
|
||||
windmill_parser_go::otyp_to_string(x.otyp),
|
||||
x.name
|
||||
)
|
||||
})
|
||||
.join("\n");
|
||||
let runner_content: String = format!(
|
||||
r#"package inner
|
||||
type Req struct {{
|
||||
{req_body}
|
||||
}}
|
||||
|
||||
func Run(req Req) (interface{{}}, error){{
|
||||
return main({spread})
|
||||
}}
|
||||
|
||||
"#,
|
||||
);
|
||||
write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?;
|
||||
}
|
||||
}
|
||||
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
|
||||
reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
|
||||
|
||||
let child = if !*DISABLE_NSJAIL {
|
||||
let _ = write_file(
|
||||
job_dir,
|
||||
"run.config.proto",
|
||||
&NSJAIL_CONFIG_RUN_GO_CONTENT
|
||||
.replace("{JOB_DIR}", job_dir)
|
||||
.replace("{CACHE_DIR}", GO_CACHE_DIR)
|
||||
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
|
||||
.replace("{SHARED_MOUNT}", shared_mount),
|
||||
)
|
||||
.await?;
|
||||
let build_go = Command::new(GO_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.env("GOPATH", GO_CACHE_DIR)
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
.args(vec!["build", "main.go"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
build_go,
|
||||
false,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"go build",
|
||||
)
|
||||
.await?;
|
||||
|
||||
Command::new(NSJAIL_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(reserved_variables)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
} else {
|
||||
if let Some(ref netrc) = *NETRC {
|
||||
write_file(&HOME_ENV, ".netrc", netrc).await?;
|
||||
}
|
||||
Command::new(GO_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(reserved_variables)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.env("GOPATH", GO_CACHE_DIR)
|
||||
.env("GOPRIVATE", GOPRIVATE.as_ref().unwrap_or(&String::new()))
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
.args(vec!["run", "main.go"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
};
|
||||
handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"go run",
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
}
|
||||
|
||||
async fn gen_go_mod(
|
||||
inner_content: &str,
|
||||
job_dir: &str,
|
||||
requirements: &str,
|
||||
) -> error::Result<(bool, bool)> {
|
||||
gen_go_mymod(inner_content, job_dir).await?;
|
||||
|
||||
let md = requirements.split_once(GO_REQ_SPLITTER);
|
||||
if let Some((req, sum)) = md {
|
||||
write_file(job_dir, "go.mod", &req).await?;
|
||||
write_file(job_dir, "go.sum", &sum).await?;
|
||||
Ok((true, true))
|
||||
} else {
|
||||
write_file(job_dir, "go.mod", &requirements).await?;
|
||||
Ok((true, false))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn install_go_dependencies(
|
||||
job_id: &Uuid,
|
||||
code: &str,
|
||||
logs: &mut String,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
non_dep_job: bool,
|
||||
skip_go_mod: bool,
|
||||
has_sum: bool,
|
||||
worker_name: &str,
|
||||
w_id: &str,
|
||||
) -> error::Result<String> {
|
||||
if !skip_go_mod {
|
||||
gen_go_mymod(code, job_dir).await?;
|
||||
let child = Command::new("go")
|
||||
.current_dir(job_dir)
|
||||
.args(vec!["mod", "init", "mymod"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
|
||||
handle_child(job_id, db, logs, child, false, worker_name, w_id, "go init").await?;
|
||||
}
|
||||
|
||||
let mut new_lockfile = false;
|
||||
|
||||
let hash = if !has_sum {
|
||||
calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
|
||||
} else {
|
||||
"".to_string()
|
||||
};
|
||||
|
||||
let mut skip_tidy = has_sum;
|
||||
|
||||
if !has_sum {
|
||||
if let Some(cached) = sqlx::query_scalar!(
|
||||
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
|
||||
hash
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
{
|
||||
logs.push_str(&format!("\nfound cached resolution"));
|
||||
gen_go_mod(code, job_dir, &cached).await?;
|
||||
skip_tidy = true;
|
||||
new_lockfile = false;
|
||||
} else {
|
||||
new_lockfile = true;
|
||||
}
|
||||
}
|
||||
|
||||
let mod_command = if skip_tidy { "download" } else { "tidy" };
|
||||
let child = Command::new(GO_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env("GOPATH", GO_CACHE_DIR)
|
||||
.args(vec!["mod", mod_command])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
&format!("go {mod_command}"),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
|
||||
if (!new_lockfile || has_sum) && non_dep_job {
|
||||
return Ok("".to_string());
|
||||
}
|
||||
|
||||
let mut req_content = "".to_string();
|
||||
|
||||
let mut file = File::open(format!("{job_dir}/go.mod")).await?;
|
||||
file.read_to_string(&mut req_content).await?;
|
||||
req_content.push_str(GO_REQ_SPLITTER);
|
||||
let sum_path = format!("{job_dir}/go.sum");
|
||||
if tokio::fs::metadata(&sum_path).await.is_ok() {
|
||||
let mut file = File::open(sum_path).await?;
|
||||
file.read_to_string(&mut req_content).await?;
|
||||
}
|
||||
|
||||
if non_dep_job {
|
||||
sqlx::query!(
|
||||
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
|
||||
hash,
|
||||
req_content
|
||||
).fetch_optional(db).await?;
|
||||
|
||||
return Ok(String::new());
|
||||
} else {
|
||||
Ok(req_content)
|
||||
}
|
||||
}
|
||||
|
||||
async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> {
|
||||
let code = if code.trim_start().starts_with("package") {
|
||||
code.to_string()
|
||||
} else {
|
||||
format!("package inner; {code}")
|
||||
};
|
||||
|
||||
let mymod_dir = format!("{job_dir}/inner");
|
||||
DirBuilder::new()
|
||||
.recursive(true)
|
||||
.create(&mymod_dir)
|
||||
.await
|
||||
.expect("could not create go's mymod dir");
|
||||
|
||||
write_file(&mymod_dir, "inner_main.go", &code).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -6,32 +6,23 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use sqlx::{Pool, Postgres};
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
use tracing::instrument;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error::Error,
|
||||
flow_status::FlowStatusModule,
|
||||
jobs::{JobKind, QueuedJob},
|
||||
schedule::Schedule,
|
||||
METRICS_ENABLED,
|
||||
};
|
||||
use windmill_queue::{delete_job, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED};
|
||||
use windmill_common::{error::Error, flow_status::FlowStatusModule, schedule::Schedule};
|
||||
use windmill_queue::{delete_job, schedule::get_schedule_opt, JobKind, QueuedJob};
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn add_completed_job_error(
|
||||
db: &Pool<Postgres>,
|
||||
queued_job: &QueuedJob,
|
||||
logs: String,
|
||||
e: serde_json::Value,
|
||||
metrics: Option<crate::worker::Metrics>,
|
||||
rsmq: Option<R>,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
if *METRICS_ENABLED {
|
||||
metrics.map(|m| m.worker_execution_failed.inc());
|
||||
}
|
||||
metrics.map(|m| m.worker_execution_failed.inc());
|
||||
let result = serde_json::json!({ "error": e });
|
||||
let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs, rsmq).await?;
|
||||
let _ = add_completed_job(db, &queued_job, false, false, result.clone(), logs).await?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
@@ -54,14 +45,13 @@ fn flatten_jobs(modules: Vec<FlowStatusModule>) -> Vec<Uuid> {
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
pub async fn add_completed_job(
|
||||
db: &Pool<Postgres>,
|
||||
queued_job: &QueuedJob,
|
||||
success: bool,
|
||||
skipped: bool,
|
||||
result: serde_json::Value,
|
||||
logs: String,
|
||||
rsmq: Option<R>,
|
||||
) -> Result<Uuid, Error> {
|
||||
let duration =
|
||||
if queued_job.job_kind == JobKind::Flow || queued_job.job_kind == JobKind::FlowPreview {
|
||||
@@ -93,7 +83,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten();
|
||||
let mut tx: QueueTransaction<'_, R> = (rsmq, db.begin().await?).into();
|
||||
let mut tx = db.begin().await?;
|
||||
let job_id = queued_job.id.clone();
|
||||
sqlx::query!(
|
||||
"INSERT INTO completed_job AS cj
|
||||
@@ -126,10 +116,9 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
, email
|
||||
, visible_to_owner
|
||||
, mem_peak
|
||||
, tag
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
|
||||
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29, $30)
|
||||
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
|
||||
ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
|
||||
queued_job.workspace_id,
|
||||
queued_job.id,
|
||||
@@ -159,13 +148,12 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
duration: Option<i64>,
|
||||
queued_job.email,
|
||||
queued_job.visible_to_owner,
|
||||
mem_peak,
|
||||
queued_job.tag,
|
||||
mem_peak
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e}")))?;
|
||||
tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
|
||||
let _ = delete_job(db, &queued_job.workspace_id, job_id).await?;
|
||||
if !queued_job.is_flow_step
|
||||
&& queued_job.job_kind != JobKind::Flow
|
||||
&& queued_job.job_kind != JobKind::FlowPreview
|
||||
@@ -187,8 +175,8 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
let additional_usage = duration.unwrap() as i32 / 1000;
|
||||
|
||||
let w_id = &queued_job.workspace_id;
|
||||
let premium_workspace = *CLOUD_HOSTED
|
||||
&& sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
|
||||
let premium_workspace =
|
||||
sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", w_id)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("fetching if {w_id} is premium: {e}")))?;
|
||||
@@ -220,24 +208,21 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
|
||||
mut tx: QueueTransaction<'c, R>,
|
||||
pub async fn schedule_again_if_scheduled<'c>(
|
||||
mut tx: Transaction<'c, Postgres>,
|
||||
db: &Pool<Postgres>,
|
||||
schedule_path: &str,
|
||||
script_path: &str,
|
||||
w_id: &str,
|
||||
) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
|
||||
let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;
|
||||
|
||||
if schedule.is_none() {
|
||||
tracing::error!(
|
||||
"Schedule {schedule_path} in {w_id} not found. Impossible to schedule again"
|
||||
);
|
||||
return Ok(tx);
|
||||
}
|
||||
|
||||
let schedule = schedule.unwrap();
|
||||
|
||||
) -> windmill_common::error::Result<Transaction<'c, Postgres>> {
|
||||
let schedule = get_schedule_opt(&mut tx, w_id, schedule_path)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
Error::InternalErr(format!(
|
||||
"Could not find schedule {:?} for workspace {}",
|
||||
schedule_path, w_id
|
||||
))
|
||||
})?;
|
||||
if schedule.enabled && script_path == schedule.script_path {
|
||||
let res = windmill_queue::schedule::push_scheduled_job(
|
||||
tx,
|
||||
@@ -247,7 +232,7 @@ pub async fn schedule_again_if_scheduled<'c, R: rsmq_async::RsmqConnection + Clo
|
||||
edited_by: schedule.edited_by,
|
||||
edited_at: schedule.edited_at,
|
||||
schedule: schedule.schedule,
|
||||
timezone: schedule.timezone,
|
||||
offset_: schedule.offset_,
|
||||
enabled: schedule.enabled,
|
||||
script_path: schedule.script_path,
|
||||
is_flow: schedule.is_flow,
|
||||
|
||||
@@ -6,11 +6,9 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{cell::RefCell, collections::HashMap, rc::Rc};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use deno_core::{
|
||||
op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, OpState, RuntimeOptions,
|
||||
};
|
||||
use deno_core::{op, serde_v8, v8, v8::IsolateHandle, Extension, JsRuntime, RuntimeOptions};
|
||||
use itertools::Itertools;
|
||||
use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
@@ -19,7 +17,10 @@ use tokio::{sync::oneshot, time::timeout};
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{error::Error, flow_status::JobResult};
|
||||
|
||||
use crate::AuthedClient;
|
||||
pub struct EvalCreds {
|
||||
pub workspace: String,
|
||||
pub token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IdContext {
|
||||
@@ -28,23 +29,22 @@ pub struct IdContext {
|
||||
pub previous_id: String,
|
||||
}
|
||||
|
||||
pub struct OptAuthedClient(Option<AuthedClient>);
|
||||
pub async fn eval_timeout(
|
||||
expr: String,
|
||||
env: Vec<(String, serde_json::Value)>,
|
||||
authed_client: Option<&AuthedClient>,
|
||||
creds: Option<EvalCreds>,
|
||||
by_id: Option<IdContext>,
|
||||
base_internal_url: &str,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
let expr2 = expr.clone();
|
||||
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
|
||||
let has_client = authed_client.is_some();
|
||||
let authed_client = authed_client.cloned();
|
||||
let base_internal_url: String = base_internal_url.to_string();
|
||||
timeout(
|
||||
std::time::Duration::from_millis(10000),
|
||||
std::time::Duration::from_millis(2000),
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut ops = vec![];
|
||||
|
||||
if authed_client.is_some() {
|
||||
if creds.is_some() {
|
||||
ops.extend([
|
||||
// An op for summing an array of numbers
|
||||
// The op-layer automatically deserializes inputs
|
||||
@@ -54,12 +54,12 @@ pub async fn eval_timeout(
|
||||
])
|
||||
}
|
||||
|
||||
if by_id.is_some() && authed_client.is_some() {
|
||||
if by_id.is_some() {
|
||||
ops.push(op_get_result::decl());
|
||||
ops.push(op_get_id::decl());
|
||||
}
|
||||
|
||||
let ext = Extension::builder("js_eval").ops(ops).build();
|
||||
let ext = Extension::builder().ops(ops).build();
|
||||
// Use our snapshot to provision our new runtime
|
||||
let options = RuntimeOptions {
|
||||
extensions: vec![ext],
|
||||
@@ -68,11 +68,6 @@ pub async fn eval_timeout(
|
||||
};
|
||||
|
||||
let mut js_runtime = JsRuntime::new(options);
|
||||
{
|
||||
let op_state = js_runtime.op_state();
|
||||
let mut op_state = op_state.borrow_mut();
|
||||
op_state.put(OptAuthedClient(authed_client.clone()));
|
||||
}
|
||||
|
||||
sender
|
||||
.send(js_runtime.v8_isolate().thread_safe_handle())
|
||||
@@ -91,7 +86,14 @@ pub async fn eval_timeout(
|
||||
|
||||
let expr = replace_with_await_result(expr);
|
||||
|
||||
let r = runtime.block_on(eval(&mut js_runtime, &expr, env, by_id, has_client))?;
|
||||
let r = runtime.block_on(eval(
|
||||
&mut js_runtime,
|
||||
&expr,
|
||||
env,
|
||||
creds,
|
||||
by_id,
|
||||
&base_internal_url,
|
||||
))?;
|
||||
|
||||
Ok(r) as anyhow::Result<Value>
|
||||
}),
|
||||
@@ -102,7 +104,7 @@ pub async fn eval_timeout(
|
||||
isolate.terminate_execution();
|
||||
};
|
||||
Error::ExecutionErr(format!(
|
||||
"The expression of evaluation `{expr2}` took too long to execute (>10000ms)"
|
||||
"The expression of evaluation `{expr2}` took too long to execute (>2000ms)"
|
||||
))
|
||||
})??
|
||||
}
|
||||
@@ -143,30 +145,24 @@ fn add_closing_bracket(s: &str) -> String {
|
||||
s
|
||||
}
|
||||
|
||||
const SPLIT_PAT: &str = ";";
|
||||
const SPLIT_PAT: &str = ";\n";
|
||||
async fn eval(
|
||||
context: &mut JsRuntime,
|
||||
expr: &str,
|
||||
env: Vec<(String, serde_json::Value)>,
|
||||
creds: Option<EvalCreds>,
|
||||
by_id: Option<IdContext>,
|
||||
has_client: bool,
|
||||
base_internal_url: &str,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
let exprs = expr
|
||||
.trim()
|
||||
.split(SPLIT_PAT)
|
||||
.map(|x| x.trim())
|
||||
.filter(|x| !x.is_empty())
|
||||
.collect::<Vec<&str>>();
|
||||
let expr = if exprs.is_empty() {
|
||||
"return undefined;".to_string()
|
||||
} else {
|
||||
format!(
|
||||
"{};\n return {};",
|
||||
exprs.iter().take(exprs.len() - 1).join(";\n"),
|
||||
exprs.last().unwrap()
|
||||
)
|
||||
};
|
||||
let (api_code, by_id_code) = if has_client {
|
||||
let expr = expr.trim();
|
||||
let expr = format!(
|
||||
"{}\nreturn {};",
|
||||
expr.split(SPLIT_PAT)
|
||||
.take(expr.split(SPLIT_PAT).count() - 1)
|
||||
.join("\n"),
|
||||
expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
|
||||
);
|
||||
let (api_code, by_id_code) = if let Some(EvalCreds { workspace, token }) = creds {
|
||||
let by_id_code = if let Some(by_id) = by_id {
|
||||
format!(
|
||||
r#"
|
||||
@@ -183,12 +179,12 @@ async function result_by_id(node_id) {{
|
||||
}}
|
||||
}} else {{
|
||||
let flow_job_id = "{}";
|
||||
return await Deno.core.opAsync("op_get_id", [ flow_job_id, node_id]);
|
||||
return await Deno.core.opAsync("op_get_id", [workspace, flow_job_id, token, base_url, node_id]);
|
||||
}}
|
||||
}}
|
||||
|
||||
async function get_result(id) {{
|
||||
return await Deno.core.opAsync("op_get_result", [id]);
|
||||
return await Deno.core.opAsync("op_get_result", [workspace, id, token, base_url]);
|
||||
}}
|
||||
const results = new Proxy({{}}, {{
|
||||
get: function(target, name, receiver) {{
|
||||
@@ -202,12 +198,12 @@ const results = new Proxy({{}}, {{
|
||||
.into_iter()
|
||||
.map(|(k, v)| {
|
||||
let v_str = match v {
|
||||
JobResult::SingleJob(x) => format!("\"{x}\""),
|
||||
JobResult::SingleJob(x) => x.to_string(),
|
||||
JobResult::ListJob(x) => {
|
||||
format!("[{}]", x.iter().map(|x| format!("\"{x}\"")).join(","))
|
||||
format!("[{}]", x.iter().map(|x| x.to_string()).join(","))
|
||||
}
|
||||
};
|
||||
format!("\"{k}\": {v_str}")
|
||||
format!("\"{k}\": \"{v_str}\"")
|
||||
})
|
||||
.join(","),
|
||||
by_id.previous_id,
|
||||
@@ -219,13 +215,17 @@ const results = new Proxy({{}}, {{
|
||||
|
||||
let api_code = format!(
|
||||
r#"
|
||||
let workspace = "{workspace}";
|
||||
let base_url = "{}";
|
||||
let token = "{token}";
|
||||
async function variable(path) {{
|
||||
return await Deno.core.opAsync("op_variable", [path]);
|
||||
return await Deno.core.opAsync("op_variable", [workspace, path, token, base_url]);
|
||||
}}
|
||||
async function resource(path) {{
|
||||
return await Deno.core.opAsync("op_resource", [path]);
|
||||
return await Deno.core.opAsync("op_resource", [workspace, path, token, base_url]);
|
||||
}}
|
||||
"#,
|
||||
base_internal_url,
|
||||
);
|
||||
(api_code, by_id_code)
|
||||
} else {
|
||||
@@ -252,7 +252,7 @@ async function resource(path) {{
|
||||
.join(""),
|
||||
);
|
||||
tracing::debug!("{}", code);
|
||||
let global = context.execute_script("<anon>", code.into())?;
|
||||
let global = context.execute_script("<anon>", &code)?;
|
||||
let global = context.resolve_value(global).await?;
|
||||
|
||||
let scope = &mut context.handle_scope();
|
||||
@@ -274,83 +274,60 @@ async function resource(path) {{
|
||||
|
||||
// TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client?
|
||||
#[op]
|
||||
async fn op_variable(
|
||||
op_state: Rc<RefCell<OpState>>,
|
||||
args: Vec<String>,
|
||||
) -> Result<String, anyhow::Error> {
|
||||
let path = &args[0];
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.get_variable(&client.workspace, path, None)
|
||||
.await?;
|
||||
Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned()))
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
async fn op_variable(args: Vec<String>) -> Result<String, anyhow::Error> {
|
||||
let workspace = &args[0];
|
||||
let path = &args[1];
|
||||
let token = &args[2];
|
||||
let base_url = &args[3];
|
||||
let client = windmill_api_client::create_client(base_url, token.clone());
|
||||
let result = client.get_variable(workspace, path, None).await?;
|
||||
Ok(result.into_inner().value.unwrap_or_else(|| "".to_owned()))
|
||||
}
|
||||
|
||||
#[op]
|
||||
async fn op_get_result(
|
||||
op_state: Rc<RefCell<OpState>>,
|
||||
args: Vec<String>,
|
||||
) -> Result<serde_json::Value, anyhow::Error> {
|
||||
let id = &args[0];
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.get_completed_job_result(&client.workspace, &id.parse()?)
|
||||
.await?
|
||||
.clone();
|
||||
Ok(serde_json::json!(result))
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
async fn op_get_result(args: Vec<String>) -> Result<serde_json::Value, anyhow::Error> {
|
||||
let workspace = &args[0];
|
||||
let id = &args[1];
|
||||
let token = &args[2];
|
||||
let base_url = &args[3];
|
||||
let client = windmill_api_client::create_client(base_url, token.clone());
|
||||
let result = client
|
||||
.get_completed_job(workspace, &id.parse()?)
|
||||
.await?
|
||||
.result
|
||||
.clone();
|
||||
Ok(serde_json::json!(result))
|
||||
}
|
||||
|
||||
#[op]
|
||||
async fn op_get_id(
|
||||
op_state: Rc<RefCell<OpState>>,
|
||||
args: Vec<String>,
|
||||
) -> Result<Option<serde_json::Value>, anyhow::Error> {
|
||||
let flow_job_id = &args[0];
|
||||
let node_id = &args[1];
|
||||
async fn op_get_id(args: Vec<String>) -> Result<Option<serde_json::Value>, anyhow::Error> {
|
||||
let workspace = &args[0];
|
||||
let flow_job_id = &args[1];
|
||||
let token = &args[2];
|
||||
let base_url = &args[3];
|
||||
let node_id = &args[4];
|
||||
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.result_by_id(&client.workspace, flow_job_id, node_id)
|
||||
.await
|
||||
.map_or(None, |e| Some(e.into_inner()));
|
||||
Ok(result)
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
let client = windmill_api_client::create_client(base_url, token.clone());
|
||||
let result = client
|
||||
.result_by_id(workspace, flow_job_id, node_id, Some(true))
|
||||
.await
|
||||
.map_or(None, |e| Some(e.into_inner()));
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[op]
|
||||
async fn op_resource(
|
||||
op_state: Rc<RefCell<OpState>>,
|
||||
args: Vec<String>,
|
||||
) -> Result<serde_json::Value, anyhow::Error> {
|
||||
let path = &args[0];
|
||||
|
||||
let client = op_state.borrow().borrow::<OptAuthedClient>().0.clone();
|
||||
if let Some(client) = client {
|
||||
let result = client
|
||||
.get_client()
|
||||
.get_resource(&client.workspace, path)
|
||||
.await?;
|
||||
Ok(result
|
||||
.into_inner()
|
||||
.value
|
||||
.unwrap_or_else(|| serde_json::json!({})))
|
||||
} else {
|
||||
anyhow::bail!("No client found in op state");
|
||||
}
|
||||
async fn op_resource(args: Vec<String>) -> Result<serde_json::Value, anyhow::Error> {
|
||||
let workspace = &args[0];
|
||||
let path = &args[1];
|
||||
let token = &args[2];
|
||||
let base_url = &args[3];
|
||||
let client = windmill_api_client::create_client(base_url, token.clone());
|
||||
let result = client.get_resource(workspace, path).await?;
|
||||
Ok(result
|
||||
.into_inner()
|
||||
.value
|
||||
.unwrap_or_else(|| serde_json::json!({})))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -370,7 +347,7 @@ mod tests {
|
||||
let code = "value.test + params.test";
|
||||
|
||||
let mut runtime = JsRuntime::new(RuntimeOptions::default());
|
||||
let res = eval(&mut runtime, code, env, None, false).await?;
|
||||
let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
|
||||
assert_eq!(res, json!(4));
|
||||
Ok(())
|
||||
}
|
||||
@@ -383,7 +360,7 @@ mod tests {
|
||||
multiline template`";
|
||||
|
||||
let mut runtime = JsRuntime::new(RuntimeOptions::default());
|
||||
let res = eval(&mut runtime, code, env, None, false).await?;
|
||||
let res = eval(&mut runtime, code, env, None, None, String::new().as_str()).await?;
|
||||
assert_eq!(res, json!("my 5\nmultiline template"));
|
||||
Ok(())
|
||||
}
|
||||
@@ -396,7 +373,7 @@ multiline template`";
|
||||
];
|
||||
let code = r#"params.test"#;
|
||||
|
||||
let res = eval_timeout(code.to_string(), env, None, None).await?;
|
||||
let res = eval_timeout(code.to_string(), env, None, None, String::new().as_str()).await?;
|
||||
assert_eq!(res, json!(2));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user