Compare commits
1 Commits
v1.100.2
...
add-dev-do
| Author | SHA1 | Date | |
|---|---|---|---|
|
5131ee8b89 |
4
.env
4
.env
@@ -3,5 +3,5 @@ DB_PASSWORD=changeme
|
||||
# this is the url that your instance is publicly exposed to
|
||||
WM_BASE_URL=http://localhost
|
||||
|
||||
# this is the addres caddy will listen on
|
||||
CADDY_REVERSE_PROXY=":80"
|
||||
# this is the url that caddy will reverse proxy from. It might be different than WM_BASE_URL if you re using a second proxy/load balancer in front
|
||||
CADDY_REVERSE_PROXY=http://localhost
|
||||
|
||||
5
.github/uffizzi/caddy/Caddyfile
vendored
Normal file
5
.github/uffizzi/caddy/Caddyfile
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
localhost {
|
||||
bind 0.0.0.0
|
||||
reverse_proxy /ws/* http://0.0.0.0:3001
|
||||
reverse_proxy /* http://0.0.0.0:8000
|
||||
}
|
||||
53
.github/uffizzi/docker-compose.uffizzi.yml
vendored
Normal file
53
.github/uffizzi/docker-compose.uffizzi.yml
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
version: '3.7'
|
||||
|
||||
x-uffizzi:
|
||||
ingress:
|
||||
service: windmill
|
||||
port: 8000
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:14
|
||||
environment:
|
||||
POSTGRES_PASSWORD: changeme
|
||||
POSTGRES_DB: windmill
|
||||
|
||||
windmill:
|
||||
image: '${WINDMILL_IMAGE}'
|
||||
ports:
|
||||
- 8000:8000
|
||||
entrypoint: ['/bin/sh', '-c']
|
||||
command: 'echo ${OAUTH_JSON_BASE64} | base64 --decode > /usr/src/app/oauth.json && ./windmill'
|
||||
environment:
|
||||
- DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
|
||||
- BASE_URL=${EXPECTED_URL}
|
||||
- BASE_INTERNAL_URL=http://localhost:8000
|
||||
- RUST_LOG=info
|
||||
- NUM_WORKERS=3
|
||||
- KEEP_JOB_DIR=false
|
||||
- DENO_PATH=/usr/bin/deno
|
||||
- PYTHON_PATH=/usr/local/bin/python3
|
||||
- METRICS_ADDR=false
|
||||
- OAUTH_JSON_BASE64=${OAUTH_JSON_BASE64}
|
||||
volumes:
|
||||
- worker_dependency_cache:/tmp/windmill/cache
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 250M
|
||||
|
||||
lsp:
|
||||
image: '${LSP_IMAGE}'
|
||||
ports:
|
||||
- 3001:3001
|
||||
|
||||
# caddy:
|
||||
# image: caddy:2.5.2-alpine
|
||||
# restart: unless-stopped
|
||||
# volumes:
|
||||
# - ./.github/uffizzi/caddy:/etc/caddy
|
||||
# environment:
|
||||
# - BASE_URL=localhost
|
||||
|
||||
volumes:
|
||||
worker_dependency_cache:
|
||||
2
.github/workflows/backend-test.yml
vendored
2
.github/workflows/backend-test.yml
vendored
@@ -40,4 +40,4 @@ jobs:
|
||||
backend -> target
|
||||
- name: cargo test
|
||||
timeout-minutes: 10
|
||||
run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
|
||||
run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture
|
||||
|
||||
2
.github/workflows/frontend-check.yml
vendored
2
.github/workflows/frontend-check.yml
vendored
@@ -4,8 +4,6 @@ on:
|
||||
types: [opened,synchronize,reopened,closed]
|
||||
paths:
|
||||
- "frontend/**"
|
||||
merge_group:
|
||||
|
||||
jobs:
|
||||
npm_check:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
5
.github/workflows/pypi_on_release.yml
vendored
5
.github/workflows/pypi_on_release.yml
vendored
@@ -28,8 +28,8 @@ jobs:
|
||||
needs: [publish_pypi]
|
||||
runs-on: [self-hosted, new]
|
||||
steps:
|
||||
- name: Sleep for 120 seconds waiting for pypi to update index
|
||||
run: sleep 120
|
||||
- name: Sleep for 30 seconds waiting for pypi to update index
|
||||
run: sleep 30s
|
||||
shell: bash
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
@@ -74,6 +74,7 @@ jobs:
|
||||
${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
|
||||
${{ steps.metalocal.outputs.tags }}
|
||||
${{ steps.meta.outputs.tags }}
|
||||
registry.uffizzi.com/windmill-lsp:60d
|
||||
labels: ${{ steps.metalocal.outputs.labels }}
|
||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
|
||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
|
||||
|
||||
93
.github/workflows/uffizzi-build.yml.archived
vendored
Normal file
93
.github/workflows/uffizzi-build.yml.archived
vendored
Normal file
@@ -0,0 +1,93 @@
|
||||
name: Build PR Image
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened,synchronize,reopened,closed]
|
||||
paths:
|
||||
- "backend/**"
|
||||
- ".github/uffizzi/**"
|
||||
- ".github/workflows/**"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-windmill:
|
||||
name: Build and Push `windmill`
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ (github.event_name != 'pull_request' || github.event.action != 'closed')}}
|
||||
outputs:
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
steps:
|
||||
- name: Checkout git repo
|
||||
uses: actions/checkout@v3
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
- name: Generate UUID image name
|
||||
id: uuid
|
||||
run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
|
||||
- name: Docker metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v4
|
||||
with:
|
||||
images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
|
||||
tags: type=raw,value=60d
|
||||
- name: Build and Push Image to registry.uffizzi.com ephemeral registry
|
||||
uses: docker/build-push-action@v4
|
||||
with:
|
||||
push: true
|
||||
context: ./
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
|
||||
render-compose-file:
|
||||
name: Render Docker Compose File
|
||||
# Pass output of this workflow to another triggered by `workflow_run` event.
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- build-windmill
|
||||
outputs:
|
||||
compose-file-cache-key: ${{ steps.hash.outputs.hash }}
|
||||
steps:
|
||||
- name: Checkout git repo
|
||||
uses: actions/checkout@v3
|
||||
- name: Render Compose File
|
||||
run: |
|
||||
WINDMILL_IMAGE=${{ needs.build-windmill.outputs.tags }}
|
||||
export WINDMILL_IMAGE
|
||||
LSP_IMAGE=registry.uffizzi.com/windmill-lsp:60d
|
||||
export LSP_IMAGE
|
||||
envsubst '${WINDMILL_IMAGE} ${LSP_IMAGE}' < ./.github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
|
||||
cat docker-compose.rendered.yml
|
||||
- name: Upload Rendered Compose File as Artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: preview-spec
|
||||
path: docker-compose.rendered.yml
|
||||
retention-days: 2
|
||||
- name: Serialize PR Event to File
|
||||
run: |
|
||||
cat << EOF > event.json
|
||||
${{ toJSON(github.event) }}
|
||||
EOF
|
||||
- name: Upload PR Event as Artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: preview-spec
|
||||
path: event.json
|
||||
retention-days: 2
|
||||
|
||||
delete-preview:
|
||||
name: Call for Preview Deletion
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event.action == 'closed' }}
|
||||
steps:
|
||||
# If this PR is closing, we will not render a compose file nor pass it to the next workflow.
|
||||
- name: Serialize PR Event to File
|
||||
run: echo '${{ toJSON(github.event) }}' > event.json
|
||||
- name: Upload PR Event as Artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: preview-spec
|
||||
path: event.json
|
||||
retention-days: 2
|
||||
115
.github/workflows/uffizzi-preview.yml.archived
vendored
Normal file
115
.github/workflows/uffizzi-preview.yml.archived
vendored
Normal file
@@ -0,0 +1,115 @@
|
||||
name: Deploy Uffizzi Preview
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- "Build PR Image"
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
cache-compose-file:
|
||||
name: Cache Compose File
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
outputs:
|
||||
compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
|
||||
pr-number: ${{ env.PR_NUMBER }}
|
||||
steps:
|
||||
- name: 'Download artifacts'
|
||||
# Fetch output (zip archive) from the workflow run that triggered this workflow.
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.payload.workflow_run.id,
|
||||
});
|
||||
let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
|
||||
return artifact.name == "preview-spec"
|
||||
})[0];
|
||||
if (matchArtifact === undefined) {
|
||||
throw TypeError('Build Artifact not found!');
|
||||
}
|
||||
let download = await github.rest.actions.downloadArtifact({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
artifact_id: matchArtifact.id,
|
||||
archive_format: 'zip',
|
||||
});
|
||||
let fs = require('fs');
|
||||
fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
|
||||
- name: 'Unzip artifact'
|
||||
run: unzip preview-spec.zip
|
||||
- name: Read Event into ENV
|
||||
run: |
|
||||
echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
|
||||
cat event.json >> $GITHUB_ENV
|
||||
echo 'EOF' >> $GITHUB_ENV
|
||||
- name: Read PR Number From Event Object
|
||||
id: pr
|
||||
run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
|
||||
- name: Predict Deployment URL
|
||||
id: url
|
||||
# Replace dots in the repo name with the plus sign
|
||||
run: |
|
||||
REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
|
||||
echo "EXPECTED_URL=https://app.uffizzi.com/github.com/$REPO/pull/$PR_NUMBER" >> $GITHUB_ENV
|
||||
|
||||
- name: Re-Render Compose File
|
||||
run: |
|
||||
OAUTH_JSON_BASE64=${{ secrets.OAUTH_JSON_BASE64 }}
|
||||
export OAUTH_JSON_BASE64
|
||||
envsubst '${OAUTH_JSON_BASE64} ${EXPECTED_URL}' < docker-compose.rendered.yml > docker-compose.uffizzi.yml
|
||||
# cat docker-compose.uffizzi.yml
|
||||
|
||||
- name: Hash Rendered Compose File
|
||||
id: hash
|
||||
# If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
|
||||
if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
|
||||
run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.uffizzi.yml | awk '{ print $1 }')" >> $GITHUB_ENV
|
||||
- name: Cache Rendered Compose File
|
||||
if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: docker-compose.uffizzi.yml
|
||||
key: ${{ env.COMPOSE_FILE_HASH }}
|
||||
|
||||
- name: DEBUG - Print Job Outputs
|
||||
if: ${{ runner.debug }}
|
||||
run: |
|
||||
echo "PR number: ${{ env.PR_NUMBER }}"
|
||||
echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
|
||||
cat event.json
|
||||
|
||||
deploy-uffizzi-preview:
|
||||
name: Use Remote Workflow to Preview on Uffizzi
|
||||
needs:
|
||||
- cache-compose-file
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
|
||||
with:
|
||||
# If this workflow was triggered by a PR close event, cache-key will be an empty string
|
||||
# and this reusable workflow will delete the preview deployment.
|
||||
compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
|
||||
compose-file-cache-path: docker-compose.uffizzi.yml
|
||||
server: https://app.uffizzi.com
|
||||
pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
id-token: write
|
||||
|
||||
playwright:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- deploy-uffizzi-preview
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 16
|
||||
- name: "Playwright run"
|
||||
timeout-minutes: 2
|
||||
run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=${{ needs.deploy-uffizzi-preview.outputs.url }} && npm run test
|
||||
305
CHANGELOG.md
305
CHANGELOG.md
@@ -1,311 +1,6 @@
|
||||
# Changelog
|
||||
|
||||
|
||||
## [1.100.2](https://github.com/windmill-labs/windmill/compare/v1.100.1...v1.100.2) (2023-05-14)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** update wmill script push ([678b574](https://github.com/windmill-labs/windmill/commit/678b574efcae66801a115d576db9d00aa9e4145d))
|
||||
* discriminate execute apps by component ([908358e](https://github.com/windmill-labs/windmill/commit/908358eb08614d07b5e846630743242b68b9e149))
|
||||
* transform_inputs now only support single line expressions ([c252b76](https://github.com/windmill-labs/windmill/commit/c252b765f1b1fd38f07cbe06548ca5cbe4047ea1))
|
||||
|
||||
## [1.100.1](https://github.com/windmill-labs/windmill/compare/v1.100.0...v1.100.1) (2023-05-12)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* update setup step ([178ed6f](https://github.com/windmill-labs/windmill/commit/178ed6f426020c9966380392088562e27aa77cf3))
|
||||
|
||||
## [1.100.0](https://github.com/windmill-labs/windmill/compare/v1.99.0...v1.100.0) (2023-05-12)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add download button ([9b9730d](https://github.com/windmill-labs/windmill/commit/9b9730d2b7239827fd8dfe8f46b6bd98d535e8d0))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** handle Date type ([5e7e46e](https://github.com/windmill-labs/windmill/commit/5e7e46e0259bfc11e92f2446858ddbe9f1b4b08e))
|
||||
* **frontend:** apps rendering should not depend on local time ([8e785d8](https://github.com/windmill-labs/windmill/commit/8e785d8ba6da16d06816d0379cadfb899be99b06))
|
||||
* **frontend:** only download result for apps ([6bbd937](https://github.com/windmill-labs/windmill/commit/6bbd9374cbd2c516dd3b56551103fcfeba01f80f))
|
||||
|
||||
## [1.99.0](https://github.com/windmill-labs/windmill/compare/v1.98.0...v1.99.0) (2023-05-10)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** run endpoints also support support x-www-form-urlencoded encoded payloads ([2b57418](https://github.com/windmill-labs/windmill/commit/2b57418427e9417599f9f969cb78088c5166958a))
|
||||
* **frontend:** add hide refresh button ([ef089ab](https://github.com/windmill-labs/windmill/commit/ef089ab56c5ef493118574076d9512ae3b6a42bf))
|
||||
* **frontend:** add input library to flow builder ([957fd81](https://github.com/windmill-labs/windmill/commit/957fd81576dfe65326cad2ed8487121e157e0953))
|
||||
* **frontend:** allow copy pasting nested containers ([742ee3a](https://github.com/windmill-labs/windmill/commit/742ee3a5181fdcfba1f59889a8d99347fd0c4610))
|
||||
* **frontend:** app on error ([#1556](https://github.com/windmill-labs/windmill/issues/1556)) ([6c2ba05](https://github.com/windmill-labs/windmill/commit/6c2ba053a1a023e454296a5ebf2842abf90362a8))
|
||||
* **frontend:** App select tabs ([#1557](https://github.com/windmill-labs/windmill/issues/1557)) ([4ad530f](https://github.com/windmill-labs/windmill/commit/4ad530f2f004cb33cbc95d5c3b1591a44f93bdee))
|
||||
* **frontend:** conditional rendering ([#1555](https://github.com/windmill-labs/windmill/issues/1555)) ([3d371d5](https://github.com/windmill-labs/windmill/commit/3d371d5b6524a4ec0152b15d001d8758900de457))
|
||||
* increase timeout to 900 ([018b504](https://github.com/windmill-labs/windmill/commit/018b504986a6c36c1e5ecbc5e92a763a6b6e613b))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** run endpoints also support support x-www-form-urlencoded encoded payloads ([5601d04](https://github.com/windmill-labs/windmill/commit/5601d047fe4736a996d064dc8ff34af5d70706a5))
|
||||
|
||||
## [1.98.0](https://github.com/windmill-labs/windmill/compare/v1.97.0...v1.98.0) (2023-05-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** if member of a single workspace, autoset at login ([2dfb74e](https://github.com/windmill-labs/windmill/commit/2dfb74e7e45b279f5169ac89b483ed336e0bd109))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** grant all on raw_app ([c62670f](https://github.com/windmill-labs/windmill/commit/c62670f735da8378a896b538f3c3afeef100f7ab))
|
||||
|
||||
## [1.97.0](https://github.com/windmill-labs/windmill/compare/v1.96.3...v1.97.0) (2023-05-09)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** add windmill_status_code to run_wait_result ([38ec7d3](https://github.com/windmill-labs/windmill/commit/38ec7d3a857a19e474f0a1b07b73b85aa5f10f41))
|
||||
* **backend:** cache hub scripts in the worker cache ([7537f1a](https://github.com/windmill-labs/windmill/commit/7537f1a1d7162610f78a7e84a53d57f8478a5965))
|
||||
* **backend:** in python, if a value is bytes, it is encoded to base64 automaticaly ([6b5ceed](https://github.com/windmill-labs/windmill/commit/6b5ceed6525d4251517627351a15a4fe604629fc))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **lsp:** handle write_message errors ([9392890](https://github.com/windmill-labs/windmill/commit/939289030ba667afc9b517dfdc90f26378fa44a6))
|
||||
|
||||
## [1.96.3](https://github.com/windmill-labs/windmill/compare/v1.96.2...v1.96.3) (2023-05-08)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** add folder listing ([c598083](https://github.com/windmill-labs/windmill/commit/c5980839251cdc759c5afc688e7084f5d58ad57f))
|
||||
* **cli:** show diffs only with --show-diffs ([d254088](https://github.com/windmill-labs/windmill/commit/d254088fce00091352ad95888deeaf88bc6c9d6f))
|
||||
* **cli:** show diffs only with --show-diffs ([37f08e9](https://github.com/windmill-labs/windmill/commit/37f08e9357c7a74a1c6cdc18bad2d6dc4de5d33d))
|
||||
* **cli:** variable updating ([2639250](https://github.com/windmill-labs/windmill/commit/2639250b43c3c55f5b9a43f4020fc2f0747e792b))
|
||||
|
||||
## [1.96.2](https://github.com/windmill-labs/windmill/compare/v1.96.1...v1.96.2) (2023-05-08)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** add debug mode to CLI + improve error output ([8f1cdf1](https://github.com/windmill-labs/windmill/commit/8f1cdf1d61adf80bf0d7c4a5160fd3085d3814ac))
|
||||
|
||||
## [1.96.1](https://github.com/windmill-labs/windmill/compare/v1.96.0...v1.96.1) (2023-05-08)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** fix cli folder sync ([239f401](https://github.com/windmill-labs/windmill/commit/239f40199955d47e4943be4c72c3d150a58f2dd9))
|
||||
* **cli:** fix cli folder sync ([a90514b](https://github.com/windmill-labs/windmill/commit/a90514b8e99a419c26512eb370895322088b6aa9))
|
||||
|
||||
## [1.96.0](https://github.com/windmill-labs/windmill/compare/v1.95.1...v1.96.0) (2023-05-08)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add support for full fleged apps (react, svelte, vue) ([#1536](https://github.com/windmill-labs/windmill/issues/1536)) ([13242ab](https://github.com/windmill-labs/windmill/commit/13242abff153b021cac1ecaa3cbf65ae9d87fb69))
|
||||
* **frontend:** Add a custom deepEqualWithOrderedArray to handle orde… ([#1537](https://github.com/windmill-labs/windmill/issues/1537)) ([3a291f7](https://github.com/windmill-labs/windmill/commit/3a291f7108623b5c7194f0a7f6a3774499669313))
|
||||
* **frontend:** Add label, description, input style + add displayType… ([#1540](https://github.com/windmill-labs/windmill/issues/1540)) ([bef829d](https://github.com/windmill-labs/windmill/commit/bef829d4805bb6c5330b13dc17c9a89a84ad48ca))
|
||||
* **frontend:** app modal ([#1518](https://github.com/windmill-labs/windmill/issues/1518)) ([686f5bb](https://github.com/windmill-labs/windmill/commit/686f5bbe1847cb2a92678e7cfdbb51ecf6bbe2b6))
|
||||
|
||||
## [1.95.1](https://github.com/windmill-labs/windmill/compare/v1.95.0...v1.95.1) (2023-05-06)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** cli flow sync improvements ([e585e3a](https://github.com/windmill-labs/windmill/commit/e585e3aea2b18b6dc0c9fa7ffa1e6c1dfb2a3ce2))
|
||||
|
||||
## [1.95.0](https://github.com/windmill-labs/windmill/compare/v1.94.0...v1.95.0) (2023-05-05)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** default parameters are used in python if missing from args ([8791a86](https://github.com/windmill-labs/windmill/commit/8791a86a936301d44ae05ea09d26c9815abf8929))
|
||||
* **frontend:** App Schema Form component ([#1533](https://github.com/windmill-labs/windmill/issues/1533)) ([85c0d93](https://github.com/windmill-labs/windmill/commit/85c0d939f59411d023cd4b173ce11224d3cbc9db))
|
||||
* **frontend:** App stepper ([#1529](https://github.com/windmill-labs/windmill/issues/1529)) ([15f1c94](https://github.com/windmill-labs/windmill/commit/15f1c947bb233147f7da261fd32054a51a9c6efa))
|
||||
* **frontend:** Merge run configuration + triggers ([#1530](https://github.com/windmill-labs/windmill/issues/1530)) ([1be4658](https://github.com/windmill-labs/windmill/commit/1be4658150ef20a9f1f0fe57b5f30ba3c2d4d94e))
|
||||
|
||||
## [1.94.0](https://github.com/windmill-labs/windmill/compare/v1.93.1...v1.94.0) (2023-05-04)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add eval badge + alert ([#1522](https://github.com/windmill-labs/windmill/issues/1522)) ([32f04c7](https://github.com/windmill-labs/windmill/commit/32f04c796856fa48ddc1548752ba1e7a8802083a))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** fix python transformers ([a07e3e8](https://github.com/windmill-labs/windmill/commit/a07e3e84386c0895a7209fc87a4b07218271feca))
|
||||
* **frontend:** fix ArrayStaticInputEditor width ([#1528](https://github.com/windmill-labs/windmill/issues/1528)) ([b423eec](https://github.com/windmill-labs/windmill/commit/b423eec019785a62c279db01fc93eb3fe08f7f1f))
|
||||
* **frontend:** fix select width ([#1526](https://github.com/windmill-labs/windmill/issues/1526)) ([f248c09](https://github.com/windmill-labs/windmill/commit/f248c09655889ddace24f451597a56e81443be3c))
|
||||
* **frontend:** preserve customise arguments ([b4867f1](https://github.com/windmill-labs/windmill/commit/b4867f12bb4f595b5b0e8142ab5d720307ecadd3))
|
||||
|
||||
## [1.93.1](https://github.com/windmill-labs/windmill/compare/v1.93.0...v1.93.1) (2023-05-03)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **cli:** add yaml support for cli ([03e6017](https://github.com/windmill-labs/windmill/commit/03e6017860526784f1a8696eceed5750b25f1c5c))
|
||||
|
||||
## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
|
||||
|
||||
## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
|
||||
|
||||
## [1.92.2](https://github.com/windmill-labs/windmill/compare/v1.92.1...v1.92.2) (2023-05-02)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **go-client:** fix go-client gen ([82c4d66](https://github.com/windmill-labs/windmill/commit/82c4d6629e134f00389c87a948c52878e5a3f4f5))
|
||||
|
||||
## [1.92.1](https://github.com/windmill-labs/windmill/compare/v1.92.0...v1.92.1) (2023-05-02)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **go-client:** fix go-client gen ([df333d9](https://github.com/windmill-labs/windmill/commit/df333d9739f601714f7a0124f47422dfb113d320))
|
||||
|
||||
## [1.92.0](https://github.com/windmill-labs/windmill/compare/v1.91.0...v1.92.0) (2023-05-02)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** add labels as table action ([64065c1](https://github.com/windmill-labs/windmill/commit/64065c17f305fb4c7c078c7fa4935d5423da8f66))
|
||||
* **frontend:** add labels as table action ([2ab1714](https://github.com/windmill-labs/windmill/commit/2ab1714dfa7cffc46b4b6aa40dabdd92c5a6270f))
|
||||
* **frontend:** allow running eval in every field ([62acbb5](https://github.com/windmill-labs/windmill/commit/62acbb5ab3a0727b306e25a80b74ac8216619501))
|
||||
* **frontend:** background script can run script and flows ([#1515](https://github.com/windmill-labs/windmill/issues/1515)) ([607c803](https://github.com/windmill-labs/windmill/commit/607c803be91921b53f329a2c2c3c129ce53d6c0c))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** fix small ui issues ([#1513](https://github.com/windmill-labs/windmill/issues/1513)) ([f6ff8ca](https://github.com/windmill-labs/windmill/commit/f6ff8ca232f5725f86a36379956da2731def2580))
|
||||
|
||||
## [1.91.0](https://github.com/windmill-labs/windmill/compare/v1.90.0...v1.91.0) (2023-05-01)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* add drafts for apps ([f7374c8](https://github.com/windmill-labs/windmill/commit/f7374c8204f85b4371e61f34dcd4b66857c0f8ab))
|
||||
* introduce backend persisted draft systems for scripts ([88e37fe](https://github.com/windmill-labs/windmill/commit/88e37fe0bed58f396690622e925d5e078c60140c))
|
||||
* introduce draft for flows ([a196642](https://github.com/windmill-labs/windmill/commit/a1966427e893dc8a58c8f2862ded752884843813))
|
||||
|
||||
## [1.90.0](https://github.com/windmill-labs/windmill/compare/v1.89.0...v1.90.0) (2023-04-28)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** add EXIT_AFTER_NO_JOB_FOR_SECS for ephemeral workers ([de9abd1](https://github.com/windmill-labs/windmill/commit/de9abd129db13dcdf0e69e2c1e2d3aa558fb783a))
|
||||
* **backend:** add JOB_RETENTION_SECS to delete completed jobs completed after expiration period ([0b7bad3](https://github.com/windmill-labs/windmill/commit/0b7bad3816e61841ef4765db7881227274c20b23))
|
||||
* **backend:** expose tag in the job ([#1486](https://github.com/windmill-labs/windmill/issues/1486)) ([324d4f5](https://github.com/windmill-labs/windmill/commit/324d4f5e9e89e6de600e882f23bf545c0b1dc539))
|
||||
* **frontend:** adapt style ([#1488](https://github.com/windmill-labs/windmill/issues/1488)) ([41a24ec](https://github.com/windmill-labs/windmill/commit/41a24ecd36d9cc537cbd1dd0cd1de6f689be1b8c))
|
||||
* **frontend:** add an eval input component for flow ([#1494](https://github.com/windmill-labs/windmill/issues/1494)) ([2815f1e](https://github.com/windmill-labs/windmill/commit/2815f1ec71177bb6e89d0d62a8df89030d37b1fc))
|
||||
* **frontend:** Add new integration icons ([#1479](https://github.com/windmill-labs/windmill/issues/1479)) ([7adacd4](https://github.com/windmill-labs/windmill/commit/7adacd4c9f03d17b47abc515bc391e348a7e6ec1))
|
||||
* **frontend:** refactor inline script ([#1480](https://github.com/windmill-labs/windmill/issues/1480)) ([05c837e](https://github.com/windmill-labs/windmill/commit/05c837e64f61bfb22ae1f80263deb1c879030985))
|
||||
* **frontend:** Schedules run now ([#1475](https://github.com/windmill-labs/windmill/issues/1475)) ([47f0f35](https://github.com/windmill-labs/windmill/commit/47f0f35236e02958f6bc00b5652e06e25eabeaf5))
|
||||
* **frontend:** Small style fix ([#1473](https://github.com/windmill-labs/windmill/issues/1473)) ([7ad496a](https://github.com/windmill-labs/windmill/commit/7ad496ad3f746ffd782856a35c1456999792fa94))
|
||||
* **frontend:** Support TS union type with a select field ([#1457](https://github.com/windmill-labs/windmill/issues/1457)) ([8b76324](https://github.com/windmill-labs/windmill/commit/8b763249cb1360c122cc81d50a1a95d1ad3ddd5b))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** Allow 0 as select default value ([#1474](https://github.com/windmill-labs/windmill/issues/1474)) ([d8529ff](https://github.com/windmill-labs/windmill/commit/d8529ff3ed6168de60bb24626a3e36ab4beae15c))
|
||||
* **frontend:** close the modal before deleting a form modal ([#1484](https://github.com/windmill-labs/windmill/issues/1484)) ([430c733](https://github.com/windmill-labs/windmill/commit/430c73399b7e1524ee81d0ce8d7d2eaf16117f9a))
|
||||
* **frontend:** fix apply connection ([#1487](https://github.com/windmill-labs/windmill/issues/1487)) ([cf59cc0](https://github.com/windmill-labs/windmill/commit/cf59cc04efb853fe06a23042115128689b5d26ee))
|
||||
* **frontend:** Fix frontend script ([#1476](https://github.com/windmill-labs/windmill/issues/1476)) ([b60a7f6](https://github.com/windmill-labs/windmill/commit/b60a7f63d04b1fc851479060e77613c77d20198a))
|
||||
* **frontend:** fix recomputa all ([#1491](https://github.com/windmill-labs/windmill/issues/1491)) ([fb05a09](https://github.com/windmill-labs/windmill/commit/fb05a09955f937000fbba0826c03c52e65aa146e))
|
||||
* **frontend:** Flow editor design updates ([#1477](https://github.com/windmill-labs/windmill/issues/1477)) ([50d814c](https://github.com/windmill-labs/windmill/commit/50d814c3dc55841b16cd1df8ae86d021de4e880c))
|
||||
* **frontend:** Minor app editor updates ([#1458](https://github.com/windmill-labs/windmill/issues/1458)) ([8fd10b1](https://github.com/windmill-labs/windmill/commit/8fd10b1f5813b3f4200a980de4a85e7d701660a7))
|
||||
* **frontend:** register applyConnection as a callback to remove unnecessary reactivit ([#1485](https://github.com/windmill-labs/windmill/issues/1485)) ([d915f6b](https://github.com/windmill-labs/windmill/commit/d915f6b004ea7cc27a3c55b3504df902f5db1aef))
|
||||
* **frontend:** reset ui job loading state when submitting preview job triggers error ([#1483](https://github.com/windmill-labs/windmill/issues/1483)) ([6f8616f](https://github.com/windmill-labs/windmill/commit/6f8616fb273b1f4a1489878a617d239f30ecb1c0))
|
||||
* **frontend:** Update CLI login request styling ([#1454](https://github.com/windmill-labs/windmill/issues/1454)) ([c77393c](https://github.com/windmill-labs/windmill/commit/c77393c15444868308b44a72bee89e49fc23d80f))
|
||||
* **frontend:** Update direct exports ([#1456](https://github.com/windmill-labs/windmill/issues/1456)) ([4a2af13](https://github.com/windmill-labs/windmill/commit/4a2af1359ee29001236591f685cacfa9df6715df))
|
||||
|
||||
## [1.89.0](https://github.com/windmill-labs/windmill/compare/v1.88.1...v1.89.0) (2023-04-23)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** global cache refactor for pip using tar for each dependency ([#1443](https://github.com/windmill-labs/windmill/issues/1443)) ([369dd0d](https://github.com/windmill-labs/windmill/commit/369dd0dac61e5430856ed9abf7129bbad3b75860))
|
||||
* **backend:** only run fully deployed scripts ([3d031c7](https://github.com/windmill-labs/windmill/commit/3d031c701705459f418b11d2ca83e71943e4079b))
|
||||
* **backend:** worker groups ([#1452](https://github.com/windmill-labs/windmill/issues/1452)) ([722783f](https://github.com/windmill-labs/windmill/commit/722783f7f630e3123ffd2605deb21a915188bd20))
|
||||
* **backend:** workers are instantly ready and sync with global cache in background ([670ba51](https://github.com/windmill-labs/windmill/commit/670ba51d9bd9a9a0b07a2ed064c316234bb5819d))
|
||||
* **ee:** sync cache in background ([c919827](https://github.com/windmill-labs/windmill/commit/c919827cf8eb9437b6d9bd57b3d3ad883a66de3b))
|
||||
* **ee:** sync cache in background ([0e77e37](https://github.com/windmill-labs/windmill/commit/0e77e37fbddbb517d5e1b1f07a27a40b63371439))
|
||||
* **frontend:** Add documentation links ([#1399](https://github.com/windmill-labs/windmill/issues/1399)) ([36acbf7](https://github.com/windmill-labs/windmill/commit/36acbf793b6714dffe4dbb0e2501b9438f034858))
|
||||
* **frontend:** Add seconds input ([#1445](https://github.com/windmill-labs/windmill/issues/1445)) ([30bf7ad](https://github.com/windmill-labs/windmill/commit/30bf7ad3e9785420b4dcd814ce3c7a444d23cc9f))
|
||||
* **frontend:** add toast actions ([#1411](https://github.com/windmill-labs/windmill/issues/1411)) ([d173232](https://github.com/windmill-labs/windmill/commit/d17323286a05aa0b1680ef94d7058d2c8902782f))
|
||||
* **frontend:** reorder array items in app editor ([#1426](https://github.com/windmill-labs/windmill/issues/1426)) ([3615fb2](https://github.com/windmill-labs/windmill/commit/3615fb26fb91d64626d82ace6e6275e424ece832))
|
||||
* **frontend:** support showing metadata on script add via query param ([#1438](https://github.com/windmill-labs/windmill/issues/1438)) ([3c98452](https://github.com/windmill-labs/windmill/commit/3c98452f50913ef639eba96996f3a6c80508bd63))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** avoid potential conflict between pull from tar and background sync ([d76e907](https://github.com/windmill-labs/windmill/commit/d76e90757e209263da7f79fa85052969e7efd63d))
|
||||
* **backend:** global cache synco only start if all piptars have been downloaded ([5f8a730](https://github.com/windmill-labs/windmill/commit/5f8a730fdfbb9e3d518555f7272a3bb297725f28))
|
||||
* **frontend:** App color picker overflow issue ([#1449](https://github.com/windmill-labs/windmill/issues/1449)) ([32903d2](https://github.com/windmill-labs/windmill/commit/32903d2839a082d53168bb9177fc88f6ab0ec482))
|
||||
* **frontend:** fix copy content button width ([#1428](https://github.com/windmill-labs/windmill/issues/1428)) ([d96d4a5](https://github.com/windmill-labs/windmill/commit/d96d4a524edebea65bc602194c8f11f5d69e920a))
|
||||
* **frontend:** Minor update of app default codes ([#1440](https://github.com/windmill-labs/windmill/issues/1440)) ([fe75aa1](https://github.com/windmill-labs/windmill/commit/fe75aa18f2f27745db35329aa60938694640a8c6))
|
||||
* **frontend:** Update app default codes ([#1432](https://github.com/windmill-labs/windmill/issues/1432)) ([c8acfbc](https://github.com/windmill-labs/windmill/commit/c8acfbc1ff0f6c23e5a2229ca83a3b09eec826c3))
|
||||
* **frontend:** Update app mobile preview width ([#1431](https://github.com/windmill-labs/windmill/issues/1431)) ([1764613](https://github.com/windmill-labs/windmill/commit/17646130bcf8cf646a4ccdfa39f9a8791876a137))
|
||||
* **frontend:** Update flow tooltip z-indexes ([#1433](https://github.com/windmill-labs/windmill/issues/1433)) ([17cb8fc](https://github.com/windmill-labs/windmill/commit/17cb8fc3fa0b39a9750e61ca2731a15bbda690ec))
|
||||
* **frontend:** Update flow viewer styling ([#1441](https://github.com/windmill-labs/windmill/issues/1441)) ([46a29b5](https://github.com/windmill-labs/windmill/commit/46a29b5d27b8d9f7ea38c1063fc081ed5933db5d))
|
||||
|
||||
## [1.88.1](https://github.com/windmill-labs/windmill/compare/v1.88.0...v1.88.1) (2023-04-18)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **frontend:** fix hub list ([1144329](https://github.com/windmill-labs/windmill/commit/1144329972fb61e2df62873ca1e485c88fabc478))
|
||||
|
||||
## [1.88.0](https://github.com/windmill-labs/windmill/compare/v1.87.0...v1.88.0) (2023-04-17)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **backend:** install python scripts on save ([cb7e686](https://github.com/windmill-labs/windmill/commit/cb7e686dd95397d5b37edd5aac50b6d1429c4a71))
|
||||
* **frontend:** Add runs preview popup ([#1405](https://github.com/windmill-labs/windmill/issues/1405)) ([4ab023f](https://github.com/windmill-labs/windmill/commit/4ab023f95085958ab1ad01dc249d308c7ebf423e))
|
||||
* **frontend:** cancellable inline script editor run ([e828d26](https://github.com/windmill-labs/windmill/commit/e828d2673e62e95e5e1235eeca8107ac7cfb7e45))
|
||||
* **frontend:** Remove gap when button label is empty ([#1402](https://github.com/windmill-labs/windmill/issues/1402)) ([568f59e](https://github.com/windmill-labs/windmill/commit/568f59eefb104047b8ef063f273fe238075d6407))
|
||||
* **frontend:** Unify main lists ([#1406](https://github.com/windmill-labs/windmill/issues/1406)) ([48bbbd0](https://github.com/windmill-labs/windmill/commit/48bbbd0e872a12ed1c562a6d14967a2a0f7c4735))
|
||||
* **frontend:** Update airtable instructions ([#1403](https://github.com/windmill-labs/windmill/issues/1403)) ([7dc7ece](https://github.com/windmill-labs/windmill/commit/7dc7ecef55b465fc096f71fc9de5c8b543136ff7))
|
||||
* inputs library on run page ([92a2934](https://github.com/windmill-labs/windmill/commit/92a293488e8e58350229931ab69f7924d58474be))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **backend:** deno uses --no-check ([a5499c2](https://github.com/windmill-labs/windmill/commit/a5499c26f3ebd8b07541a7e0cbf33a7008a8f476))
|
||||
* **backend:** do not fail on schedule not existing anymore ([a5f6d73](https://github.com/windmill-labs/windmill/commit/a5f6d73f7d53d7af9d285a85460509763263c508))
|
||||
* **frontend:** Fix app file uploads ([#1408](https://github.com/windmill-labs/windmill/issues/1408)) ([ac489ac](https://github.com/windmill-labs/windmill/commit/ac489ac2da0fbf01f5e2877612c14cfaf1ef79c2))
|
||||
* **frontend:** fix buttons width ([#1407](https://github.com/windmill-labs/windmill/issues/1407)) ([75a0482](https://github.com/windmill-labs/windmill/commit/75a0482ef046dd7e30f6d6039dbc66880182dc5e))
|
||||
* **frontend:** fix enum sync ([#1410](https://github.com/windmill-labs/windmill/issues/1410)) ([98060ce](https://github.com/windmill-labs/windmill/commit/98060ce55d5efa59a8989cf9357935976d57650b))
|
||||
* **frontend:** Handle scheduled runs in preview ([#1413](https://github.com/windmill-labs/windmill/issues/1413)) ([accdc1a](https://github.com/windmill-labs/windmill/commit/accdc1ac59ce9611f66567222a73995d3c0a3f9d))
|
||||
* **frontend:** Keep selected tab during renaming ([#1409](https://github.com/windmill-labs/windmill/issues/1409)) ([82cd048](https://github.com/windmill-labs/windmill/commit/82cd048ef4d08f31660f6f31a96940676a28996c))
|
||||
* **frontend:** Queued-running jobs preview ([#1414](https://github.com/windmill-labs/windmill/issues/1414)) ([b2a40a0](https://github.com/windmill-labs/windmill/commit/b2a40a05805344c1c34f2ba917b4cdd52dfffc3f))
|
||||
* **frontend:** Remove output when deleting a component ([#1397](https://github.com/windmill-labs/windmill/issues/1397)) ([6aa1008](https://github.com/windmill-labs/windmill/commit/6aa100893352870d5a99fdd56d7f1425a221a273))
|
||||
|
||||
## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ WORKDIR /windmill
|
||||
ENV SQLX_OFFLINE=true
|
||||
ENV CARGO_INCREMENTAL=1
|
||||
|
||||
FROM node:20-alpine as frontend
|
||||
FROM node:19-alpine as frontend
|
||||
|
||||
# install dependencies
|
||||
WORKDIR /frontend
|
||||
|
||||
117
README.md
117
README.md
@@ -5,7 +5,7 @@
|
||||
<em>.</em>
|
||||
</p>
|
||||
<p align=center>
|
||||
Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
|
||||
Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
@@ -187,9 +187,9 @@ compiling from source or using without a postgres super user, see
|
||||
### Docker compose
|
||||
|
||||
```
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
|
||||
curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
|
||||
curl https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml -o docker-compose.yml
|
||||
curl https://github.com/windmill-labs/windmill/blob/main/CaddyFile -o Caddyfile
|
||||
curl https://github.com/windmill-labs/windmill/blob/main/.env -o .env
|
||||
|
||||
docker compose up -d --pull always
|
||||
```
|
||||
@@ -224,9 +224,9 @@ be AGPLv3 or you must get a commercial license. Contact us at
|
||||
<ruben@windmill.dev> if you have any doubts.
|
||||
|
||||
In addition, a commercial license grants you a dedicated engineer to transition
|
||||
your current infrastructure to Windmill, support with tight SLA, and our global
|
||||
cache sync for high-performance/no dependency cache miss of cluster from 10+
|
||||
nodes to 200+ nodes.
|
||||
your current infrastructure to Windmill, support with tight SLA, audit logs
|
||||
export features, SSO, unlimited users creation, advanced permission managing
|
||||
features such as groups and the ability to create more than one workspace.
|
||||
|
||||
### OAuth for self-hosting
|
||||
|
||||
@@ -288,62 +288,53 @@ it being synced automatically everyday.
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Environment Variable name | Default | Description | Api Server/Worker/All |
|
||||
| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
|
||||
| DATABASE_URL | | The Postgres database url. | All |
|
||||
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
|
||||
| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server |
|
||||
| PORT | 8000 | Exposed port | Server |
|
||||
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
|
||||
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
|
||||
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
|
||||
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
|
||||
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server |
|
||||
| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker |
|
||||
| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker |
|
||||
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
|
||||
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
|
||||
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
|
||||
| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker |
|
||||
| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker |
|
||||
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
|
||||
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
|
||||
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
|
||||
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
|
||||
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
|
||||
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
|
||||
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
|
||||
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
|
||||
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
|
||||
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
|
||||
| NETRC | | The netrc content to use a private go registry | Worker |
|
||||
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
|
||||
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
|
||||
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
|
||||
| PATH | None | The path environment variable, usually inherited | Worker |
|
||||
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
|
||||
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
|
||||
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
|
||||
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
|
||||
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
|
||||
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
|
||||
| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker |
|
||||
| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker |
|
||||
| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | |
|
||||
| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker |
|
||||
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
|
||||
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
|
||||
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
|
||||
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails |
|
||||
| GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker |
|
||||
| WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker |
|
||||
| CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server |
|
||||
| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete |
|
||||
| WAIT_RESULT_FAST_POLL_INTERVAL_MS | 50 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
|
||||
| WAIT_RESULT_SLOW_POLL_INTERVAL_MS | 200 | The time in between polling for the run_wait_result endpoints in fast poll mode | Server |
|
||||
| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2 | The duration of fast poll mode before switching to slow poll | Server |
|
||||
| EXIT_AFTER_NO_JOB_FOR_SECS | None | Exit worker if no job is received after duration in secs if defined | Worker |
|
||||
| OAUTH_JSON_AS_BASE64 | None | Base64 encoded JSON of the OAuth configuration. e.g `OAUTH_JSON_AS_BASE64=$(base64 oauth.json \| tr -d '\n')` to encode it | Server |
|
||||
| Environment Variable name | Default | Description | Api Server/Worker/All |
|
||||
| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
|
||||
| DATABASE_URL | | The Postgres database url. | All |
|
||||
| DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker |
|
||||
| SERVER_BIND_ADDR | 0.0.0.0 | IP Address on which to bind listening socket | Server |
|
||||
| PORT | 8000 | Exposed port | Server |
|
||||
| NUM_WORKERS | 3 | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance) | Worker |
|
||||
| DISABLE_SERVER | false | Binary would operate as a worker only instance | Worker |
|
||||
| METRICS_ADDR | None | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001 | All |
|
||||
| JSON_FMT | false | Output the logs in json format instead of logfmt | All |
|
||||
| BASE_URL | http://localhost:8000 | The base url that is exposed publicly to access your instance | Server |
|
||||
| BASE_INTERNAL_URL | http://localhost:8000 | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests. | Worker |
|
||||
| TIMEOUT | 300 | The maximum time of execution of a script. When reached, the job is failed as having timedout. | Worker |
|
||||
| ZOMBIE_JOB_TIMEOUT | 30 | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s) | Server |
|
||||
| RESTART_ZOMBIE_JOBS | true | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed | Server |
|
||||
| SLEEP_QUEUE | 50 | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker |
|
||||
| MAX_LOG_SIZE | 500000 | The maximum number of characters a job can emit (log + result) | Worker |
|
||||
| DISABLE_NUSER | false | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting | Worker |
|
||||
| KEEP_JOB_DIR | false | Keep the job directory after the job is done. Useful for debugging. | Worker |
|
||||
| LICENSE_KEY (EE only) | None | License key checked at startup for the Enterprise Edition of Windmill | Worker |
|
||||
| S3_CACHE_BUCKET (EE only) | None | The S3 bucket to sync the cache of the workers to | Worker |
|
||||
| TAR_CACHE_RATE (EE only) | 100 | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed). | Worker |
|
||||
| SLACK_SIGNING_SECRET | None | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack) | Server |
|
||||
| COOKIE_DOMAIN | None | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin | Server |
|
||||
| DENO_PATH | /usr/bin/deno | The path to the deno binary. | Worker |
|
||||
| PYTHON_PATH | /usr/local/bin/python3 | The path to the python binary. | Worker |
|
||||
| GO_PATH | /usr/bin/go | The path to the go binary. | Worker |
|
||||
| GOPRIVATE | | The GOPRIVATE env variable to use private go modules | Worker |
|
||||
| NETRC | | The netrc content to use a private go registry | Worker |
|
||||
| PIP_INDEX_URL | None | The index url to pass for pip. | Worker |
|
||||
| PIP_EXTRA_INDEX_URL | None | The extra index url to pass to pip. | Worker |
|
||||
| PIP_TRUSTED_HOST | None | The trusted host to pass to pip. | Worker |
|
||||
| PATH | None | The path environment variable, usually inherited | Worker |
|
||||
| HOME | None | The home directory to use for Go and Bash , usually inherited | Worker |
|
||||
| DATABASE_CONNECTIONS | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool | All |
|
||||
| SUPERADMIN_SECRET | None | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev | Server |
|
||||
| TIMEOUT_WAIT_RESULT | 20 | The number of seconds to wait before timeout on the 'run_wait_result' endpoint | Worker |
|
||||
| QUEUE_LIMIT_WAIT_RESULT | None | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit. | Worker |
|
||||
| DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker |
|
||||
| DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker |
|
||||
| NPM_CONFIG_REGISTRY | None | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry | Worker |
|
||||
| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | |
|
||||
| ADDITIONAL_PYTHON_PATHS | None | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill | Worker |
|
||||
| INCLUDE_HEADERS | None | Whitelist of headers that are passed to jobs as args (separated by a comma) | Server |
|
||||
| WHITELIST_WORKSPACES | None | Whitelist of workspaces this worker takes job from | Worker |
|
||||
| BLACKLIST_WORKSPACES | None | Blacklist of workspaces this worker takes job from | Worker |
|
||||
| INSTANCE_EVENTS_WEBHOOK | None | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails | Server |
|
||||
|
||||
## Run a local dev setup
|
||||
|
||||
|
||||
3
backend/.vscode/settings.json
vendored
3
backend/.vscode/settings.json
vendored
@@ -1,4 +1,3 @@
|
||||
{
|
||||
"python.analysis.typeCheckingMode": "basic",
|
||||
"rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"]
|
||||
"python.analysis.typeCheckingMode": "basic"
|
||||
}
|
||||
|
||||
616
backend/Cargo.lock
generated
616
backend/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.100.2"
|
||||
version = "1.87.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -19,7 +19,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.100.2"
|
||||
version = "1.87.0"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
@@ -50,10 +50,6 @@ base64.workspace = true
|
||||
sha2.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
url.workspace = true
|
||||
lazy_static.workspace = true
|
||||
once_cell.workspace = true
|
||||
prometheus.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
@@ -104,7 +100,7 @@ rand = "0.8.5"
|
||||
rand_core = { version = "^0", features = ["std"] }
|
||||
magic-crypt = "^3"
|
||||
git-version = "^0"
|
||||
rustpython-parser = "0.2.0"
|
||||
rustpython-parser = { git = "https://github.com/RustPython/RustPython" }
|
||||
cron = "^0"
|
||||
lettre = { version = "^0", features = [
|
||||
"rustls-tls",
|
||||
@@ -162,5 +158,5 @@ async_zip = { version = "0.0.11", features = ["full"] }
|
||||
once_cell = "1.17.1"
|
||||
rsmq_async = { version = "5.1.5" }
|
||||
gosyn = "0.2.2"
|
||||
bytes = "1.4.0"
|
||||
|
||||
[patch.crates-io]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# Windmill Backend
|
||||
|
||||
This folder holds all backend components, the [src/](./src/) folder only
|
||||
contains files used to build the "root" binary.
|
||||
This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
|
||||
|
||||
## Components
|
||||
|
||||
@@ -14,9 +13,3 @@ contains files used to build the "root" binary.
|
||||
| [windmill-queue](./windmill-queue/) | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
|
||||
| [windmill-worker](./windmill-worker/) | The worker. Used to process and execute flows & jobs. |
|
||||
| [parsers](./parsers/) | Contains code to parse signatures in different langauges. |
|
||||
|
||||
### Compile sqlx for offline ci
|
||||
|
||||
```
|
||||
cargo sqlx prepare --merged -- --bin windmill --features enterprise
|
||||
```
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
GRANT ALL ON input TO windmill_user;
|
||||
GRANT ALL ON input TO windmill_admin;
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add down migration script here
|
||||
ALTER TABLE script DROP COLUMN tag;
|
||||
ALTER TABLE completed_job DROP COLUMN tag;
|
||||
ALTER TABLE queue DROP COLUMN tag;
|
||||
@@ -1,4 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE script ADD COLUMN tag VARCHAR(50);
|
||||
ALTER TABLE completed_job ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
|
||||
ALTER TABLE queue ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add down migration script here
|
||||
DROP TABLE draft;
|
||||
DROP TYPE DRAFT_TYPE;
|
||||
@@ -1,18 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE TYPE DRAFT_TYPE AS ENUM ('script', 'flow', 'app');
|
||||
|
||||
CREATE TABLE draft (
|
||||
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
|
||||
path VARCHAR(255) NOT NULL,
|
||||
typ DRAFT_TYPE NOT NULL,
|
||||
value JSONB NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (workspace_id, path, typ)
|
||||
);
|
||||
|
||||
GRANT ALL ON draft TO windmill_admin;
|
||||
GRANT ALL ON draft TO windmill_user;
|
||||
|
||||
ALTER TABLE script ADD COLUMN draft_only BOOLEAN;
|
||||
ALTER TABLE flow ADD COLUMN draft_only BOOLEAN;
|
||||
ALTER TABLE app ADD COLUMN draft_only BOOLEAN;
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,33 +0,0 @@
|
||||
-- Add up migration script here
|
||||
CREATE TABLE raw_app (
|
||||
path varchar(255) PRIMARY KEY,
|
||||
version INTEGER NOT NULL DEFAULT 0,
|
||||
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
|
||||
summary VARCHAR(1000) NOT NULL DEFAULT '',
|
||||
edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
||||
data TEXT NOT NULL,
|
||||
extra_perms JSONB NOT NULL DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE POLICY see_own ON raw_app FOR ALL
|
||||
USING (SPLIT_PART(raw_app.path, '/', 1) = 'u' AND SPLIT_PART(raw_app.path, '/', 2) = current_setting('session.user'));
|
||||
|
||||
CREATE POLICY see_member ON raw_app FOR ALL
|
||||
USING (SPLIT_PART(raw_app.path, '/', 1) = 'g' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
|
||||
|
||||
CREATE POLICY see_extra_perms_user ON raw_app FOR ALL
|
||||
USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
|
||||
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
|
||||
|
||||
CREATE POLICY see_extra_perms_groups ON raw_app FOR ALL
|
||||
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
|
||||
WITH CHECK (exists(
|
||||
SELECT key, value FROM jsonb_each_text(extra_perms)
|
||||
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
|
||||
AND value::boolean));
|
||||
|
||||
CREATE POLICY see_folder_extra_perms_user ON raw_app FOR ALL
|
||||
USING (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
|
||||
WITH CHECK (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
|
||||
|
||||
ALTER TYPE FAVORITE_KIND ADD VALUE 'raw_app';
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
GRANT ALL ON raw_app TO windmill_admin;
|
||||
GRANT ALL ON raw_app TO windmill_user;
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE app_version
|
||||
ALTER COLUMN value TYPE json;
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
@@ -1,3 +0,0 @@
|
||||
-- Add up migration script here
|
||||
ALTER TABLE draft
|
||||
ALTER COLUMN value TYPE json;
|
||||
@@ -1 +0,0 @@
|
||||
-- Add down migration script here
|
||||
File diff suppressed because it is too large
Load Diff
@@ -96,7 +96,7 @@ fn parse_go_typ(typ: &Expression) -> (Option<String>, Typ) {
|
||||
.as_ref()
|
||||
.and_then(|x| x.value.strip_prefix("`json:\""))
|
||||
.and_then(|x| x.strip_suffix("\"`"))
|
||||
.and_then(|x| x.split(',').next().map(|x| x.to_string()));
|
||||
.and_then(|x| x.split(',').last().map(|x| x.to_string()));
|
||||
let (otyp, typ) = parse_go_typ(&field.typ);
|
||||
let name = get_name(field);
|
||||
let key = json_tag.unwrap_or_else(|| name.to_string());
|
||||
|
||||
@@ -17,8 +17,8 @@ use serde_json::json;
|
||||
use windmill_common::error;
|
||||
use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ};
|
||||
|
||||
use rustpython_parser as parser;
|
||||
use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind};
|
||||
use rustpython_parser::parser::parse_program;
|
||||
|
||||
const DEF_MAIN: &str = "def main(";
|
||||
const FUNCTION_CALL: &str = "<function call>";
|
||||
@@ -64,7 +64,7 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
|
||||
"No main function found".to_string(),
|
||||
));
|
||||
}
|
||||
let ast = parse_program(&filtered_code, "main.py").map_err(|e| {
|
||||
let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| {
|
||||
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
|
||||
})?;
|
||||
let param = ast.into_iter().find_map(|x| match x {
|
||||
@@ -134,7 +134,10 @@ fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
|
||||
.into_iter()
|
||||
.zip(values)
|
||||
.map(|(k, v)| {
|
||||
let key = to_value(&k.node)
|
||||
let key = k
|
||||
.as_ref()
|
||||
.map(|x| x.node.clone())
|
||||
.and_then(|n| to_value(&n))
|
||||
.and_then(|x| match x {
|
||||
serde_json::Value::String(s) => Some(s),
|
||||
_ => None,
|
||||
@@ -212,7 +215,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
|
||||
Ok(lines)
|
||||
} else {
|
||||
let code = code.split(DEF_MAIN).next().unwrap_or("");
|
||||
let ast = parse_program(code, "main.py").map_err(|e| {
|
||||
let ast = parser::parse_program(code, "main.py").map_err(|e| {
|
||||
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
|
||||
})?;
|
||||
let mut imports: Vec<String> = ast
|
||||
|
||||
@@ -255,7 +255,6 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
|
||||
),
|
||||
false,
|
||||
),
|
||||
"Date" => (Typ::Datetime, false),
|
||||
"Base64" => (Typ::Bytes, false),
|
||||
"Email" => (Typ::Email, false),
|
||||
"Sql" => (Typ::Sql, false),
|
||||
@@ -269,7 +268,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
|
||||
pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
|
||||
let mut context = JsRuntime::new(RuntimeOptions::default());
|
||||
let code = format!("let x = {}; x", code);
|
||||
let res = context.execute_script("<anon>", code.into());
|
||||
let res = context.execute_script("<anon>", code);
|
||||
match res {
|
||||
Ok(global) => {
|
||||
let scope = &mut context.handle_scope();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,24 +6,11 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
sync::Arc,
|
||||
};
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
|
||||
use git_version::git_version;
|
||||
use monitor::handle_zombie_jobs_periodically;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{
|
||||
fs::{metadata, DirBuilder},
|
||||
join,
|
||||
sync::RwLock,
|
||||
};
|
||||
use windmill_common::{utils::rd_string, METRICS_ADDR};
|
||||
use windmill_worker::{
|
||||
DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR,
|
||||
HUB_TMP_CACHE_DIR, PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
|
||||
};
|
||||
|
||||
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
|
||||
const DEFAULT_NUM_WORKERS: usize = 3;
|
||||
@@ -31,7 +18,6 @@ const DEFAULT_PORT: u16 = 8000;
|
||||
const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
|
||||
|
||||
mod ee;
|
||||
mod monitor;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
@@ -102,7 +88,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
|
||||
let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());
|
||||
let shutdown_signal = windmill_common::shutdown_signal(tx);
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
tracing::info!(
|
||||
@@ -162,14 +148,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
"BLACKLIST_WORKSPACES",
|
||||
"INSTANCE_EVENTS_WEBHOOK",
|
||||
"CLOUD_HOSTED",
|
||||
"GLOBAL_CACHE_INTERVAL",
|
||||
"WORKER_TAGS",
|
||||
"CUSTOM_TAGS",
|
||||
"JOB_RETENTION_SECS",
|
||||
"WAIT_RESULT_FAST_POLL_DURATION_SECS",
|
||||
"WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
|
||||
"WAIT_RESULT_FAST_POLL_INTERVAL_MS",
|
||||
"EXIT_AFTER_NO_JOB_FOR_SECS",
|
||||
]);
|
||||
|
||||
if server_mode || num_workers > 0 {
|
||||
@@ -193,8 +171,6 @@ Windmill Community Edition {GIT_VERSION}
|
||||
rsmq.clone(),
|
||||
)
|
||||
.await?;
|
||||
tracing::info!("All workers exited.");
|
||||
tx.send(())?; // signal server to shutdown
|
||||
}
|
||||
Ok(()) as anyhow::Result<()>
|
||||
};
|
||||
@@ -254,11 +230,9 @@ pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>
|
||||
let rx2 = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.to_string();
|
||||
tokio::spawn(async move {
|
||||
join!(
|
||||
handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
|
||||
windmill_api::delete_expired_items_perdiodically(&db2, rx2)
|
||||
);
|
||||
windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq).await
|
||||
});
|
||||
tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
|
||||
}
|
||||
|
||||
pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
|
||||
@@ -289,30 +263,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
|
||||
let mut handles = Vec::with_capacity(num_workers as usize);
|
||||
|
||||
if metadata(&ROOT_TMP_CACHE_DIR).await.is_ok() {
|
||||
if let Err(e) = tokio::fs::remove_dir_all(&ROOT_TMP_CACHE_DIR).await {
|
||||
tracing::info!(error = %e, "Could not remove root tmp cache dir");
|
||||
}
|
||||
}
|
||||
|
||||
for x in [
|
||||
PIP_CACHE_DIR,
|
||||
DENO_CACHE_DIR,
|
||||
GO_CACHE_DIR,
|
||||
HUB_CACHE_DIR,
|
||||
TAR_PIP_TMP_CACHE_DIR,
|
||||
DENO_TMP_CACHE_DIR,
|
||||
GO_TMP_CACHE_DIR,
|
||||
HUB_TMP_CACHE_DIR,
|
||||
] {
|
||||
DirBuilder::new()
|
||||
.recursive(true)
|
||||
.create(x)
|
||||
.await
|
||||
.expect("could not create initial worker dir");
|
||||
}
|
||||
|
||||
let sync_barrier = Arc::new(RwLock::new(None));
|
||||
for i in 1..(num_workers + 1) {
|
||||
let db1 = db.clone();
|
||||
let instance_name = instance_name.clone();
|
||||
@@ -321,7 +271,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
let rx = rx.resubscribe();
|
||||
let base_internal_url = base_internal_url.clone();
|
||||
let rsmq2 = rsmq.clone();
|
||||
let sync_barrier = sync_barrier.clone();
|
||||
handles.push(tokio::spawn(monitor.instrument(async move {
|
||||
tracing::info!(worker = %worker_name, "starting worker");
|
||||
windmill_worker::run_worker(
|
||||
@@ -329,12 +278,10 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
|
||||
&instance_name,
|
||||
worker_name,
|
||||
i as u64,
|
||||
num_workers as u32,
|
||||
&ip,
|
||||
rx,
|
||||
&base_internal_url,
|
||||
rsmq2,
|
||||
sync_barrier,
|
||||
)
|
||||
.await
|
||||
})));
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use once_cell::sync::OnceCell;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{error, jobs::{JobKind, QueuedJob}, METRICS_ENABLED};
|
||||
use windmill_worker::{
|
||||
create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
|
||||
};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<String>().ok())
|
||||
.unwrap_or_else(|| "30".to_string());
|
||||
|
||||
|
||||
pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<bool>().ok())
|
||||
.unwrap_or(true);
|
||||
|
||||
static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
|
||||
"queue_zombie_restart_count",
|
||||
"Total number of jobs restarted due to ping timeout."
|
||||
)
|
||||
.unwrap();
|
||||
static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
|
||||
"queue_zombie_delete_count",
|
||||
"Total number of jobs deleted due to their ping timing out in an unrecoverable state."
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
pub async fn handle_zombie_jobs_periodically<
|
||||
R: rsmq_async::RsmqConnection + Send + Sync + Clone,
|
||||
>(
|
||||
db: &Pool<Postgres>,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) {
|
||||
loop {
|
||||
handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
|
||||
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(Duration::from_secs(30)) => (),
|
||||
_ = rx.recv() => {
|
||||
println!("received killpill for monitor job");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
|
||||
db: &Pool<Postgres>,
|
||||
base_internal_url: &str,
|
||||
rsmq: Option<R>,
|
||||
) {
|
||||
if *RESTART_ZOMBIE_JOBS {
|
||||
let restarted = sqlx::query!(
|
||||
"UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
|
||||
*ZOMBIE_JOB_TIMEOUT,
|
||||
JobKind::Flow: JobKind,
|
||||
JobKind::FlowPreview: JobKind,
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_else(|| vec![]);
|
||||
|
||||
if *METRICS_ENABLED {
|
||||
QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
|
||||
}
|
||||
for r in restarted {
|
||||
tracing::info!(
|
||||
"restarted zombie job {} {} {}",
|
||||
r.id,
|
||||
r.workspace_id,
|
||||
r.last_ping
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut timeout_query = "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2".to_string();
|
||||
if *RESTART_ZOMBIE_JOBS {
|
||||
timeout_query.push_str(" AND same_worker = true");
|
||||
};
|
||||
let timeouts = sqlx::query_as::<_, QueuedJob>(&timeout_query)
|
||||
.bind(ZOMBIE_JOB_TIMEOUT.as_str())
|
||||
.bind(JobKind::Flow)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.ok()
|
||||
.unwrap_or_else(|| vec![]);
|
||||
|
||||
if *METRICS_ENABLED {
|
||||
QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
|
||||
}
|
||||
|
||||
for job in timeouts {
|
||||
tracing::info!("timedout zombie job {} {}", job.id, job.workspace_id,);
|
||||
|
||||
// since the job is unrecoverable, the same worker queue should never be sent anything
|
||||
let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
|
||||
|
||||
let token = create_token_for_owner(
|
||||
&db,
|
||||
&job.workspace_id,
|
||||
&job.permissioned_as,
|
||||
"ephemeral-zombie-jobs",
|
||||
*SESSION_TOKEN_EXPIRY,
|
||||
&job.email,
|
||||
)
|
||||
.await
|
||||
.expect("could not create job token");
|
||||
|
||||
let client = AuthedClient {
|
||||
base_internal_url: base_internal_url.to_string(),
|
||||
token,
|
||||
workspace: job.workspace_id.to_string(),
|
||||
client: OnceCell::new(),
|
||||
};
|
||||
|
||||
let last_ping = job.last_ping.clone();
|
||||
let _ = handle_job_error(
|
||||
db,
|
||||
&client,
|
||||
job,
|
||||
error::Error::ExecutionErr(format!(
|
||||
"Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
|
||||
last_ping
|
||||
.map(|x| x.to_string())
|
||||
.unwrap_or_else(|| "no ping".to_string()),
|
||||
*ZOMBIE_JOB_TIMEOUT
|
||||
)),
|
||||
None,
|
||||
true,
|
||||
same_worker_tx_never_used,
|
||||
"",
|
||||
base_internal_url,
|
||||
rsmq.clone(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,14 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{stream, Stream};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
|
||||
use tokio::sync::RwLock;
|
||||
use windmill_api::jobs::{CompletedJob, Job};
|
||||
use windmill_api_client::types::{CreateFlowBody, RawScript};
|
||||
use windmill_common::{
|
||||
flow_status::{FlowStatus, FlowStatusModule},
|
||||
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
|
||||
jobs::{JobPayload, RawCode},
|
||||
scripts::ScriptLang,
|
||||
};
|
||||
use windmill_queue::get_queued_job;
|
||||
use windmill_queue::{get_queued_job, JobPayload, RawCode};
|
||||
|
||||
async fn initialize_tracing() {
|
||||
use std::sync::Once;
|
||||
@@ -133,7 +128,8 @@ mod suspend_resume {
|
||||
use futures::{Stream, StreamExt};
|
||||
use serde_json::json;
|
||||
use sqlx::{query_scalar, types::Uuid};
|
||||
use windmill_common::{flows::FlowValue, jobs::JobPayload};
|
||||
use windmill_common::flows::FlowValue;
|
||||
use windmill_queue::JobPayload;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -397,6 +393,7 @@ mod retry {
|
||||
use serde_json::json;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use windmill_common::flows::FlowValue;
|
||||
use windmill_queue::JobPayload;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -846,7 +843,6 @@ impl RunJob {
|
||||
/* running */ false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("push has to succeed");
|
||||
@@ -881,7 +877,7 @@ async fn in_test_worker<Fut: std::future::Future>(
|
||||
port: u16,
|
||||
) -> <Fut as std::future::Future>::Output {
|
||||
let (quit, worker) = spawn_test_worker(db, port);
|
||||
let worker = tokio::time::timeout(std::time::Duration::from_secs(45), worker);
|
||||
let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker);
|
||||
tokio::pin!(worker);
|
||||
|
||||
let res = tokio::select! {
|
||||
@@ -916,6 +912,7 @@ fn spawn_test_worker(
|
||||
let db = db.to_owned();
|
||||
let worker_instance: &str = "test worker instance";
|
||||
let worker_name: String = next_worker_name();
|
||||
let i_worker: u64 = Default::default();
|
||||
let ip: &str = Default::default();
|
||||
let future = async move {
|
||||
let base_internal_url = format!("http://localhost:{}", port);
|
||||
@@ -923,13 +920,11 @@ fn spawn_test_worker(
|
||||
&db,
|
||||
worker_instance,
|
||||
worker_name,
|
||||
1,
|
||||
1,
|
||||
i_worker,
|
||||
ip,
|
||||
rx,
|
||||
&base_internal_url,
|
||||
None,
|
||||
Arc::new(RwLock::new(None)),
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -1007,7 +1002,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
content: numbers.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -1035,7 +1029,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
|
||||
content: doubles.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -1136,7 +1129,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
content: write_file.clone(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -1175,7 +1167,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
content: write_file,
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -1201,7 +1192,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -1244,7 +1234,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
|
||||
.to_string(),
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Default::default(),
|
||||
summary: Default::default(),
|
||||
@@ -2443,12 +2432,9 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
|
||||
client
|
||||
.create_flow(
|
||||
"test-workspace",
|
||||
&CreateFlowBody {
|
||||
open_flow_w_path: windmill_api_client::types::OpenFlowWPath {
|
||||
open_flow: flow,
|
||||
path: "g/all/flow_lock_all".to_owned(),
|
||||
},
|
||||
draft_only: None,
|
||||
&windmill_api_client::types::OpenFlowWPath {
|
||||
open_flow: flow,
|
||||
path: "g/all/flow_lock_all".to_owned(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2467,21 +2453,20 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
|
||||
.modules
|
||||
.into_iter()
|
||||
.for_each(|m| {
|
||||
tracing::error!("Module: {:?}", m.value);
|
||||
assert!(matches!(
|
||||
m.value,
|
||||
windmill_api_client::types::FlowModuleValue::RawScript(RawScript {
|
||||
windmill_api_client::types::FlowModuleValue::Rawscript {
|
||||
language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
|
||||
lock: Some(ref lock),
|
||||
..
|
||||
}) if lock == "")
|
||||
} if lock == "")
|
||||
|| matches!(
|
||||
m.value,
|
||||
windmill_api_client::types::FlowModuleValue::RawScript(RawScript{
|
||||
windmill_api_client::types::FlowModuleValue::Rawscript {
|
||||
language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
|
||||
lock: Some(ref lock),
|
||||
..
|
||||
}) if lock.len() > 0)
|
||||
} if lock.len() > 0)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ path = "./src/lib.rs"
|
||||
|
||||
|
||||
[dependencies]
|
||||
progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
|
||||
progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" }
|
||||
reqwest = { version = "0.11", features = ["json", "stream"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
chrono.workspace = true
|
||||
@@ -21,7 +21,5 @@ rand.workspace = true
|
||||
base64.workspace = true
|
||||
|
||||
[build-dependencies]
|
||||
prettyplease = "0.1.25"
|
||||
progenitor = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
|
||||
progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
|
||||
serde_json = "1.0"
|
||||
syn = "1.0"
|
||||
@@ -13,9 +13,7 @@ fn main() {
|
||||
let spec = serde_json::from_reader(file).unwrap();
|
||||
let mut generator = progenitor::Generator::default();
|
||||
|
||||
let tokens = generator.generate_tokens(&spec).unwrap();
|
||||
let ast = syn::parse2(tokens).unwrap();
|
||||
let content = prettyplease::unparse(&ast);
|
||||
let content = generator.generate_text(&spec).unwrap();
|
||||
|
||||
let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf();
|
||||
out_file.push("codegen.rs");
|
||||
|
||||
@@ -72,4 +72,3 @@ prometheus.workspace = true
|
||||
async_zip.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
regex.workspace = true
|
||||
bytes.workspace = true
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.100.2
|
||||
version: 1.87.0
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -2125,41 +2125,6 @@ paths:
|
||||
items:
|
||||
type: string
|
||||
|
||||
|
||||
/w/{workspace}/drafts/create:
|
||||
post:
|
||||
summary: create draft
|
||||
operationId: createDraft
|
||||
tags:
|
||||
- draft
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
typ:
|
||||
type: string
|
||||
enum: ["flow", "script", "app"]
|
||||
value: {}
|
||||
required:
|
||||
- path
|
||||
- typ
|
||||
- enum
|
||||
responses:
|
||||
"201":
|
||||
description: draft created
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
|
||||
/w/{workspace}/scripts/create:
|
||||
post:
|
||||
summary: create script
|
||||
@@ -2174,8 +2139,38 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/NewScript"
|
||||
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
parent_hash:
|
||||
type: string
|
||||
summary:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
content:
|
||||
type: string
|
||||
schema:
|
||||
type: object
|
||||
is_template:
|
||||
type: boolean
|
||||
lock:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
language:
|
||||
type: string
|
||||
enum: [python3, deno, go, bash]
|
||||
kind:
|
||||
type: string
|
||||
enum: [script, failure, trigger, command, approval]
|
||||
required:
|
||||
- path
|
||||
- summary
|
||||
- description
|
||||
- content
|
||||
- language
|
||||
responses:
|
||||
"201":
|
||||
description: script created
|
||||
@@ -2184,22 +2179,6 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/workers/custom_tags:
|
||||
get:
|
||||
summary: get all instance custom tags (tags are used to dispatch jobs to different worker groups)
|
||||
operationId: getCustomTags
|
||||
tags:
|
||||
- worker
|
||||
responses:
|
||||
"200":
|
||||
description: list of custom tags
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
|
||||
/scripts/python/tojsonschema:
|
||||
post:
|
||||
summary: inspect python code to infer jsonschema of arguments
|
||||
@@ -2369,25 +2348,6 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Script"
|
||||
|
||||
/w/{workspace}/scripts/get/draft/{path}:
|
||||
get:
|
||||
summary: get script by path with draft
|
||||
operationId: getScriptByPathWithDraft
|
||||
tags:
|
||||
- script
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/ScriptPath"
|
||||
responses:
|
||||
"200":
|
||||
description: script details
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/NewScriptWithDraft"
|
||||
|
||||
|
||||
|
||||
/w/{workspace}/scripts/raw/p/{path}:
|
||||
get:
|
||||
summary: raw script by path
|
||||
@@ -2703,14 +2663,7 @@ paths:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/Flow"
|
||||
- type: object
|
||||
properties:
|
||||
has_draft:
|
||||
type: boolean
|
||||
draft_only:
|
||||
type: boolean
|
||||
$ref: "#/components/schemas/Flow"
|
||||
|
||||
/w/{workspace}/flows/get/{path}:
|
||||
get:
|
||||
@@ -2729,29 +2682,6 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/Flow"
|
||||
|
||||
/w/{workspace}/flows/get/draft/{path}:
|
||||
get:
|
||||
summary: get flow by path with draft
|
||||
operationId: getFlowByPathWithDraft
|
||||
tags:
|
||||
- flow
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/ScriptPath"
|
||||
responses:
|
||||
"200":
|
||||
description: flow details with draft
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/Flow"
|
||||
- type: object
|
||||
properties:
|
||||
draft:
|
||||
$ref: "#/components/schemas/Flow"
|
||||
|
||||
|
||||
/w/{workspace}/flows/exists/{path}:
|
||||
get:
|
||||
summary: exists flow by path
|
||||
@@ -2783,12 +2713,7 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/OpenFlowWPath"
|
||||
- type: object
|
||||
properties:
|
||||
draft_only:
|
||||
type: boolean
|
||||
$ref: "#/components/schemas/OpenFlowWPath"
|
||||
|
||||
responses:
|
||||
"201":
|
||||
@@ -2832,16 +2757,6 @@ paths:
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/ScriptPath"
|
||||
requestBody:
|
||||
description: archiveFlow
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
archived:
|
||||
type: boolean
|
||||
responses:
|
||||
"200":
|
||||
description: flow archived
|
||||
@@ -2888,80 +2803,6 @@ paths:
|
||||
items:
|
||||
$ref: "#/components/schemas/Input"
|
||||
|
||||
/w/{workspace}/raw_apps/list:
|
||||
get:
|
||||
summary: list all available raw apps
|
||||
operationId: listRawApps
|
||||
tags:
|
||||
- raw_app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/Page"
|
||||
- $ref: "#/components/parameters/PerPage"
|
||||
- $ref: "#/components/parameters/OrderDesc"
|
||||
- $ref: "#/components/parameters/CreatedBy"
|
||||
- name: path_start
|
||||
description: mask to filter matching starting path
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- name: path_exact
|
||||
description: mask to filter exact matching path
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
- name: starred_only
|
||||
description: |
|
||||
(default false)
|
||||
show only the starred items
|
||||
in: query
|
||||
schema:
|
||||
type: boolean
|
||||
responses:
|
||||
"200":
|
||||
description: All available raw apps
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/ListableRawApp"
|
||||
|
||||
/w/{workspace}/raw_apps/exists/{path}:
|
||||
get:
|
||||
summary: does an app exisst at path
|
||||
operationId: existsRawApp
|
||||
tags:
|
||||
- raw_app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/Path"
|
||||
responses:
|
||||
"200":
|
||||
description: app exists
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: boolean
|
||||
|
||||
/w/{workspace}/apps/get_data/{version}/{path}:
|
||||
get:
|
||||
summary: get app by path
|
||||
operationId: getRawAppData
|
||||
tags:
|
||||
- raw_app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/VersionId"
|
||||
- $ref: "#/components/parameters/ScriptPath"
|
||||
responses:
|
||||
"200":
|
||||
description: app details
|
||||
content:
|
||||
text/javascript:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/apps/list:
|
||||
get:
|
||||
summary: list all available apps
|
||||
@@ -3001,44 +2842,6 @@ paths:
|
||||
items:
|
||||
$ref: "#/components/schemas/ListableApp"
|
||||
|
||||
/w/{workspace}/apps/create:
|
||||
post:
|
||||
summary: create app
|
||||
operationId: createApp
|
||||
tags:
|
||||
- app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
description: new app
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
value: {}
|
||||
summary:
|
||||
type: string
|
||||
policy:
|
||||
$ref: "#/components/schemas/Policy"
|
||||
draft_only:
|
||||
type: boolean
|
||||
required:
|
||||
- path
|
||||
- value
|
||||
- summary
|
||||
- policy
|
||||
responses:
|
||||
"201":
|
||||
description: app created
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/apps/exists/{path}:
|
||||
get:
|
||||
summary: does an app exisst at path
|
||||
@@ -3073,24 +2876,6 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/AppWithLastVersion"
|
||||
|
||||
/w/{workspace}/apps/get/draft/{path}:
|
||||
get:
|
||||
summary: get app by path with draft
|
||||
operationId: getAppByPathWithDraft
|
||||
tags:
|
||||
- app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/ScriptPath"
|
||||
responses:
|
||||
"200":
|
||||
description: app details with draft
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/AppWithLastVersionWDraft"
|
||||
|
||||
|
||||
/w/{workspace}/apps_u/public_app/{path}:
|
||||
get:
|
||||
summary: get public app by secret
|
||||
@@ -3142,16 +2927,16 @@ paths:
|
||||
schema:
|
||||
$ref: "#/components/schemas/AppWithLastVersion"
|
||||
|
||||
/w/{workspace}/raw_apps/create:
|
||||
/w/{workspace}/apps/create:
|
||||
post:
|
||||
summary: create raw app
|
||||
operationId: createRawApp
|
||||
summary: create app
|
||||
operationId: createApp
|
||||
tags:
|
||||
- raw_app
|
||||
- app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
description: new raw app
|
||||
description: new app
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
@@ -3160,65 +2945,19 @@ paths:
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
value:
|
||||
type: string
|
||||
value: {}
|
||||
summary:
|
||||
type: string
|
||||
policy:
|
||||
$ref: "#/components/schemas/Policy"
|
||||
required:
|
||||
- path
|
||||
- value
|
||||
- summary
|
||||
- policy
|
||||
responses:
|
||||
"201":
|
||||
description: raw app created
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/raw_apps/update/{path}:
|
||||
post:
|
||||
summary: update app
|
||||
operationId: updateRawApp
|
||||
tags:
|
||||
- raw_app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/ScriptPath"
|
||||
requestBody:
|
||||
description: updateraw app
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
summary:
|
||||
type: string
|
||||
value:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: app updated
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/raw_apps/delete/{path}:
|
||||
delete:
|
||||
summary: delete raw app
|
||||
operationId: deleteRawApp
|
||||
tags:
|
||||
- raw_app
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/Path"
|
||||
responses:
|
||||
"200":
|
||||
description: app deleted
|
||||
description: app created
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
@@ -3290,8 +3029,6 @@ paths:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
component:
|
||||
type: string
|
||||
#script: script/<path>
|
||||
#flow: flow/<path>
|
||||
path:
|
||||
@@ -3313,7 +3050,6 @@ paths:
|
||||
type: object
|
||||
required:
|
||||
- args
|
||||
- component
|
||||
|
||||
responses:
|
||||
"200":
|
||||
@@ -3497,7 +3233,6 @@ paths:
|
||||
- $ref: "#/components/parameters/Running"
|
||||
- $ref: "#/components/parameters/ArgsFilter"
|
||||
- $ref: "#/components/parameters/ResultFilter"
|
||||
- $ref: "#/components/parameters/Tag"
|
||||
responses:
|
||||
"200":
|
||||
description: All available queued jobs
|
||||
@@ -3528,7 +3263,6 @@ paths:
|
||||
- $ref: "#/components/parameters/JobKinds"
|
||||
- $ref: "#/components/parameters/ArgsFilter"
|
||||
- $ref: "#/components/parameters/ResultFilter"
|
||||
- $ref: "#/components/parameters/Tag"
|
||||
- name: is_skipped
|
||||
description: is the job skipped
|
||||
in: query
|
||||
@@ -3566,7 +3300,6 @@ paths:
|
||||
- $ref: "#/components/parameters/StartedAfter"
|
||||
- $ref: "#/components/parameters/JobKinds"
|
||||
- $ref: "#/components/parameters/ArgsFilter"
|
||||
- $ref: "#/components/parameters/Tag"
|
||||
- $ref: "#/components/parameters/ResultFilter"
|
||||
- name: is_skipped
|
||||
description: is the job skipped
|
||||
@@ -3678,30 +3411,6 @@ paths:
|
||||
application/json:
|
||||
schema: {}
|
||||
|
||||
/w/{workspace}/jobs/completed/get_result_maybe/{id}:
|
||||
get:
|
||||
summary: get completed job result if job is completed
|
||||
operationId: getCompletedJobResultMaybe
|
||||
tags:
|
||||
- job
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- $ref: "#/components/parameters/JobId"
|
||||
responses:
|
||||
"200":
|
||||
description: result
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
completed:
|
||||
type: boolean
|
||||
result: {}
|
||||
required:
|
||||
- completed
|
||||
- result
|
||||
|
||||
/w/{workspace}/jobs/completed/delete/{id}:
|
||||
post:
|
||||
summary: delete completed job (erase content but keep run id)
|
||||
@@ -4682,7 +4391,7 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
[script, group_, resource, schedule, variable, flow, folder, app, raw_app]
|
||||
[script, group_, resource, schedule, variable, flow, folder, app]
|
||||
responses:
|
||||
"200":
|
||||
description: acls
|
||||
@@ -4708,7 +4417,7 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
[script, group_, resource, schedule, variable, flow, folder, app, raw_app]
|
||||
[script, group_, resource, schedule, variable, flow, folder, app]
|
||||
requestBody:
|
||||
description: acl to add
|
||||
required: true
|
||||
@@ -4745,7 +4454,7 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
enum:
|
||||
[script, group_, resource, schedule, variable, flow, folder, app, raw_app]
|
||||
[script, group_, resource, schedule, variable, flow, folder, app]
|
||||
requestBody:
|
||||
description: acl to add
|
||||
required: true
|
||||
@@ -4825,7 +4534,7 @@ paths:
|
||||
type: string
|
||||
favorite_kind:
|
||||
type: string
|
||||
enum: [flow, app, script, raw_app]
|
||||
enum: [flow, app, script]
|
||||
responses:
|
||||
"200":
|
||||
description: star item
|
||||
@@ -4848,7 +4557,7 @@ paths:
|
||||
type: string
|
||||
favorite_kind:
|
||||
type: string
|
||||
enum: [flow, app, script, raw_app]
|
||||
enum: [flow, app, script]
|
||||
responses:
|
||||
"200":
|
||||
description: unstar item
|
||||
@@ -4982,12 +4691,6 @@ components:
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
VersionId:
|
||||
name: version
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: number
|
||||
Token:
|
||||
name: token
|
||||
in: path
|
||||
@@ -5155,12 +4858,6 @@ components:
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
Tag:
|
||||
name: tag
|
||||
description: filter on jobs with a given tag/worker group
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
ResultFilter:
|
||||
name: result
|
||||
description: filter on jobs containing those result as a json subset (@> in postgres)
|
||||
@@ -5290,12 +4987,6 @@ components:
|
||||
enum: [script, failure, trigger, command, approval]
|
||||
starred:
|
||||
type: boolean
|
||||
tag:
|
||||
type: string
|
||||
has_draft:
|
||||
type: boolean
|
||||
draft_only:
|
||||
type: boolean
|
||||
required:
|
||||
- hash
|
||||
- path
|
||||
@@ -5312,57 +5003,6 @@ components:
|
||||
- kind
|
||||
- starred
|
||||
|
||||
NewScript:
|
||||
type: object
|
||||
properties:
|
||||
path:
|
||||
type: string
|
||||
parent_hash:
|
||||
type: string
|
||||
summary:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
content:
|
||||
type: string
|
||||
schema:
|
||||
type: object
|
||||
is_template:
|
||||
type: boolean
|
||||
lock:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
language:
|
||||
type: string
|
||||
enum: [python3, deno, go, bash]
|
||||
kind:
|
||||
type: string
|
||||
enum: [script, failure, trigger, command, approval]
|
||||
tag:
|
||||
type: string
|
||||
draft_only:
|
||||
type: boolean
|
||||
required:
|
||||
- path
|
||||
- summary
|
||||
- description
|
||||
- content
|
||||
- language
|
||||
|
||||
NewScriptWithDraft:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/NewScript"
|
||||
- type: object
|
||||
properties:
|
||||
draft:
|
||||
$ref: "#/components/schemas/NewScript"
|
||||
hash:
|
||||
type: string
|
||||
required:
|
||||
- hash
|
||||
|
||||
|
||||
ScriptArgs:
|
||||
type: object
|
||||
additionalProperties: {}
|
||||
@@ -5498,8 +5138,6 @@ components:
|
||||
type: boolean
|
||||
mem_peak:
|
||||
type: integer
|
||||
tag:
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
- running
|
||||
@@ -5509,7 +5147,6 @@ components:
|
||||
- is_flow_step
|
||||
- email
|
||||
- visible_to_owner
|
||||
- tag
|
||||
|
||||
CompletedJob:
|
||||
type: object
|
||||
@@ -5589,8 +5226,6 @@ components:
|
||||
type: boolean
|
||||
mem_peak:
|
||||
type: integer
|
||||
tag:
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
- created_by
|
||||
@@ -5605,7 +5240,6 @@ components:
|
||||
- is_skipped
|
||||
- email
|
||||
- visible_to_owner
|
||||
- tag
|
||||
|
||||
Job:
|
||||
allOf:
|
||||
@@ -5646,10 +5280,6 @@ components:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
folders_owners:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
usage:
|
||||
$ref: "#/components/schemas/Usage"
|
||||
required:
|
||||
@@ -5661,7 +5291,6 @@ components:
|
||||
- operator
|
||||
- disabled
|
||||
- folders
|
||||
- folders_owners
|
||||
|
||||
Usage:
|
||||
type: object
|
||||
@@ -6001,8 +5630,6 @@ components:
|
||||
language:
|
||||
type: string
|
||||
enum: [python3, deno, go, bash]
|
||||
tag:
|
||||
type: string
|
||||
|
||||
required:
|
||||
- content
|
||||
@@ -6374,8 +6001,6 @@ components:
|
||||
type: boolean
|
||||
starred:
|
||||
type: boolean
|
||||
draft_only:
|
||||
type: boolean
|
||||
required:
|
||||
- path
|
||||
- edited_by
|
||||
@@ -6458,34 +6083,6 @@ components:
|
||||
- edited_at
|
||||
- execution_mode
|
||||
|
||||
ListableRawApp:
|
||||
type: object
|
||||
properties:
|
||||
workspace_id:
|
||||
type: string
|
||||
path:
|
||||
type: string
|
||||
summary:
|
||||
type: string
|
||||
extra_perms:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: boolean
|
||||
starred:
|
||||
type: boolean
|
||||
version:
|
||||
type: number
|
||||
edited_at:
|
||||
type: string
|
||||
format: date-time
|
||||
required:
|
||||
- workspace_id
|
||||
- path
|
||||
- summary
|
||||
- extra_perms
|
||||
- version
|
||||
- edited_at
|
||||
|
||||
AppWithLastVersion:
|
||||
type: object
|
||||
properties:
|
||||
@@ -6529,15 +6126,6 @@ components:
|
||||
- execution_mode
|
||||
- extra_perms
|
||||
|
||||
AppWithLastVersionWDraft:
|
||||
allOf:
|
||||
- $ref: "#/components/schemas/AppWithLastVersion"
|
||||
- type: object
|
||||
properties:
|
||||
draft_only:
|
||||
type: boolean
|
||||
draft: {}
|
||||
|
||||
SlackToken:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -9,6 +9,7 @@ use std::collections::HashMap;
|
||||
*/
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
jobs::script_path_to_payload,
|
||||
users::{require_owner_of_path, Authed, OptAuthed},
|
||||
variables::build_crypt,
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
@@ -31,19 +32,17 @@ use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
apps::ListAppQuery,
|
||||
error::{to_anyhow, Error, JsonResult, Result},
|
||||
jobs::{script_path_to_payload, JobPayload, RawCode},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{
|
||||
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{push, QueueTransaction};
|
||||
use windmill_queue::{push, JobPayload, QueueTransaction, RawCode};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_apps))
|
||||
.route("/get/p/*path", get(get_app))
|
||||
.route("/get/draft/*path", get(get_app_w_draft))
|
||||
.route("/secret_of/*path", get(get_secret_id))
|
||||
.route("/get/v/*id", get(get_app_by_id))
|
||||
.route("/exists/*path", get(exists_app))
|
||||
@@ -75,9 +74,6 @@ pub struct ListableApp {
|
||||
pub execution_mode: String,
|
||||
pub starred: bool,
|
||||
pub edited_at: chrono::DateTime<chrono::Utc>,
|
||||
pub has_draft: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize)]
|
||||
@@ -102,23 +98,6 @@ pub struct AppWithLastVersion {
|
||||
pub extra_perms: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct AppWithLastVersionAndDraft {
|
||||
pub id: i64,
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub policy: serde_json::Value,
|
||||
pub versions: Vec<i64>,
|
||||
pub value: serde_json::Value,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub extra_perms: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
pub type StaticFields = Map<String, Value>;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
@@ -147,7 +126,6 @@ pub struct CreateApp {
|
||||
pub summary: String,
|
||||
pub value: serde_json::Value,
|
||||
pub policy: Policy,
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -178,8 +156,6 @@ async fn list_apps(
|
||||
"app_version.created_at as edited_at",
|
||||
"app.extra_perms",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
"draft.path IS NOT NULL as has_draft",
|
||||
"draft_only"
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
@@ -192,11 +168,6 @@ async fn list_apps(
|
||||
.on(
|
||||
"app_version.id = versions[array_upper(versions, 1)]"
|
||||
)
|
||||
.left()
|
||||
.join("draft")
|
||||
.on(
|
||||
"draft.path = app.path AND draft.workspace_id = app.workspace_id AND draft.typ = 'app'"
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("app_version.created_at", true)
|
||||
.and_where("app.workspace_id = ?".bind(&w_id))
|
||||
@@ -244,37 +215,6 @@ async fn get_app(
|
||||
Ok(Json(app))
|
||||
}
|
||||
|
||||
async fn get_app_w_draft(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<AppWithLastVersionAndDraft> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let app_o = sqlx::query_as!(
|
||||
AppWithLastVersionAndDraft,
|
||||
r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
|
||||
app.extra_perms, app_version.value,
|
||||
app_version.created_at, app_version.created_by,
|
||||
app.draft_only, draft.value as "draft?"
|
||||
from app
|
||||
INNER JOIN app_version ON
|
||||
app_version.id = app.versions[array_upper(app.versions, 1)]
|
||||
LEFT JOIN draft ON
|
||||
app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app'
|
||||
WHERE app.path = $1 AND app.workspace_id = $2"#,
|
||||
path.to_owned(),
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let app = not_found_if_none(app_o, "App", path)?;
|
||||
Ok(Json(app))
|
||||
}
|
||||
|
||||
async fn get_app_by_id(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -384,39 +324,14 @@ async fn create_app(
|
||||
return Err(Error::BadRequest("App path cannot be empty".to_string()));
|
||||
}
|
||||
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
|
||||
&app.path,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
if exists {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"App with path {} already exists",
|
||||
&app.path
|
||||
)));
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
|
||||
&app.path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
let id = sqlx::query_scalar!(
|
||||
"INSERT INTO app
|
||||
(workspace_id, path, summary, policy, versions, draft_only)
|
||||
VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
|
||||
(workspace_id, path, summary, policy, versions)
|
||||
VALUES ($1, $2, $3, $4, '{}') RETURNING id",
|
||||
w_id,
|
||||
app.path,
|
||||
app.summary,
|
||||
json!(app.policy),
|
||||
app.draft_only,
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await?;
|
||||
@@ -424,10 +339,9 @@ async fn create_app(
|
||||
let v_id = sqlx::query_scalar!(
|
||||
"INSERT INTO app_version
|
||||
(app_id, value, created_by)
|
||||
VALUES ($1, $2::text::json, $3) RETURNING id",
|
||||
VALUES ($1, $2, $3) RETURNING id",
|
||||
id,
|
||||
//to preserve key orders
|
||||
serde_json::to_string(&app.value).unwrap(),
|
||||
app.value,
|
||||
authed.username,
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
@@ -497,14 +411,6 @@ async fn delete_app(
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM app WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
@@ -535,6 +441,7 @@ async fn update_app(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(ns): Json<EditApp>,
|
||||
) -> Result<String> {
|
||||
@@ -549,25 +456,11 @@ async fn update_app(
|
||||
sqlb.and_where_eq("path", "?".bind(&path));
|
||||
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
|
||||
|
||||
sqlb.set("draft_only", "NULL");
|
||||
if let Some(npath) = &ns.path {
|
||||
if npath != path {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
|
||||
npath,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
if exists {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"App with path {} already exists",
|
||||
npath
|
||||
)));
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
sqlb.set_str("path", npath);
|
||||
@@ -611,10 +504,9 @@ async fn update_app(
|
||||
let v_id = sqlx::query_scalar!(
|
||||
"INSERT INTO app_version
|
||||
(app_id, value, created_by)
|
||||
VALUES ($1, $2::text::json, $3) RETURNING id",
|
||||
VALUES ($1, $2, $3) RETURNING id",
|
||||
app_id,
|
||||
//to preserve key orders
|
||||
serde_json::to_string(&nvalue).unwrap(),
|
||||
nvalue,
|
||||
authed.username,
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
@@ -629,15 +521,6 @@ async fn update_app(
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
@@ -667,7 +550,6 @@ pub struct ExecuteApp {
|
||||
// - script: script/<path>
|
||||
// - flow: flow/<path>
|
||||
pub path: Option<String>,
|
||||
pub component: String,
|
||||
pub raw_code: Option<RawCode>,
|
||||
// if set, the app is executed as viewer with the given static fields
|
||||
pub force_viewer_static_fields: Option<StaticFields>,
|
||||
@@ -706,16 +588,11 @@ async fn execute_component(
|
||||
|
||||
let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
|
||||
let mut hm = HashMap::new();
|
||||
|
||||
if let Some(path) = payload.path.clone() {
|
||||
hm.insert(format!("{}:{path}", payload.component), static_fields);
|
||||
hm.insert(path, static_fields);
|
||||
} else {
|
||||
hm.insert(
|
||||
format!(
|
||||
"{}:{}",
|
||||
payload.component,
|
||||
digest(payload.raw_code.clone().unwrap().content.as_str())
|
||||
),
|
||||
digest(payload.raw_code.clone().unwrap().content.as_str()),
|
||||
static_fields,
|
||||
);
|
||||
}
|
||||
@@ -766,16 +643,16 @@ async fn execute_component(
|
||||
}
|
||||
};
|
||||
|
||||
let (job_payload, args, tag) = match &payload {
|
||||
ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
|
||||
let (job_payload, args) = match &payload {
|
||||
ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
|
||||
let content = &raw_code.content;
|
||||
let payload = JobPayload::Code(raw_code.clone());
|
||||
let path = digest(content);
|
||||
let args = build_args(policy, component, path, args)?;
|
||||
(payload, args, None)
|
||||
let args = build_args(policy, path, args)?;
|
||||
(payload, args)
|
||||
}
|
||||
ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
|
||||
let (payload, tag) = if path.starts_with("script/") {
|
||||
ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
|
||||
let payload = if path.starts_with("script/") {
|
||||
script_path_to_payload(
|
||||
path.strip_prefix("script/").unwrap(),
|
||||
tx.transaction_mut(),
|
||||
@@ -783,18 +660,15 @@ async fn execute_component(
|
||||
)
|
||||
.await?
|
||||
} else if path.starts_with("flow/") {
|
||||
(
|
||||
JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
|
||||
None,
|
||||
)
|
||||
JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
|
||||
} else {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"path must start with script/ or flow/ (got {})",
|
||||
path
|
||||
)));
|
||||
};
|
||||
let args = build_args(policy, component, path.to_string(), args)?;
|
||||
(payload, args, tag)
|
||||
let args = build_args(policy, path.to_string(), args)?;
|
||||
(payload, args)
|
||||
}
|
||||
_ => unreachable!(),
|
||||
};
|
||||
@@ -815,7 +689,6 @@ async fn execute_component(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -866,18 +739,15 @@ async fn exists_app(
|
||||
|
||||
fn build_args(
|
||||
policy: Policy,
|
||||
component: &str,
|
||||
path: String,
|
||||
args: &Map<String, Value>,
|
||||
) -> Result<Map<String, Value>> {
|
||||
// disallow var and res access in args coming from the user for security reasons
|
||||
args.into_iter()
|
||||
.try_for_each(|x| disallow_var_res_access(x.1))?;
|
||||
let key = format!("{}:{}", component, &path);
|
||||
let static_args = policy
|
||||
.triggerables
|
||||
.get(&key)
|
||||
.or_else(|| policy.triggerables.get(&path))
|
||||
.get(&path)
|
||||
.map(|x| x.clone())
|
||||
.or_else(|| {
|
||||
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2042
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{maybe_refresh_folders, require_owner_of_path, Authed},
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path},
|
||||
routing::post,
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use windmill_common::error::Result;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new().route("/create", post(create_draft))
|
||||
}
|
||||
|
||||
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[sqlx(type_name = "DRAFT_TYPE", rename_all = "lowercase")]
|
||||
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
|
||||
pub enum DraftType {
|
||||
Script,
|
||||
Flow,
|
||||
App,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug)]
|
||||
pub struct Draft {
|
||||
pub path: String,
|
||||
pub value: serde_json::Value,
|
||||
pub typ: DraftType,
|
||||
}
|
||||
|
||||
async fn create_draft(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(draft): Json<Draft>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let authed = maybe_refresh_folders(&draft.path, &w_id, authed, &db).await;
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_owner_of_path(&authed, &draft.path)?;
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO draft
|
||||
(workspace_id, path, value, typ)
|
||||
VALUES ($1, $2, $3::text::json, $4)
|
||||
ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3::text::json",
|
||||
&w_id,
|
||||
draft.path,
|
||||
//to preserve key orders
|
||||
serde_json::to_string(&draft.value).unwrap(),
|
||||
draft.typ: DraftType,
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok((StatusCode::CREATED, format!("draft {} created", draft.path)))
|
||||
}
|
||||
|
||||
// async fn get_draft(
|
||||
// authed: Authed,
|
||||
// Extension(user_db): Extension<UserDB>,
|
||||
// Path((w_id, path)): Path<(String, StripPath)>,
|
||||
// ) -> JsonResult<Draft> {
|
||||
// let path = path.to_path();
|
||||
// let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
// let script_o = sqlx::query_as!(
|
||||
// Draft,
|
||||
// r#"SELECT path, value, typ as "typ: DraftType" FROM draft WHERE path = $1 AND workspace_id = $2"#,
|
||||
// path,
|
||||
// w_id
|
||||
// )
|
||||
// .fetch_optional(&mut tx)
|
||||
// .await?;
|
||||
// tx.commit().await?;
|
||||
|
||||
// let draft = not_found_if_none(script_o, "draft", path)?;
|
||||
// Ok(Json(draft))
|
||||
// }
|
||||
@@ -29,8 +29,6 @@ pub enum FavoriteKind {
|
||||
Script,
|
||||
Flow,
|
||||
App,
|
||||
#[allow(non_camel_case_types)]
|
||||
Raw_App,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct Favorite {
|
||||
|
||||
@@ -19,7 +19,6 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use hyper::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sql_builder::prelude::*;
|
||||
use sql_builder::SqlBuilder;
|
||||
use sqlx::{Postgres, Transaction};
|
||||
@@ -27,14 +26,12 @@ use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{self, to_anyhow, Error, JsonResult, Result},
|
||||
flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
|
||||
jobs::JobPayload,
|
||||
schedule::Schedule,
|
||||
scripts::Schema,
|
||||
utils::{
|
||||
http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
|
||||
},
|
||||
};
|
||||
use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};
|
||||
use windmill_queue::{push, schedule::push_scheduled_job, JobPayload, QueueTransaction};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -44,7 +41,6 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/archive/*path", post(archive_flow_by_path))
|
||||
.route("/delete/*path", delete(delete_flow_by_path))
|
||||
.route("/get/*path", get(get_flow_by_path))
|
||||
.route("/get/draft/*path", get(get_flow_by_path_w_draft))
|
||||
.route("/exists/*path", get(exists_flow_by_path))
|
||||
.route("/list_paths", get(list_paths))
|
||||
}
|
||||
@@ -75,8 +71,6 @@ async fn list_flows(
|
||||
"archived",
|
||||
"extra_perms",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
"draft.path IS NOT NULL as has_draft",
|
||||
"draft_only"
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
@@ -84,11 +78,6 @@ async fn list_flows(
|
||||
"favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
|
||||
.bind(&authed.username),
|
||||
)
|
||||
.left()
|
||||
.join("draft")
|
||||
.on(
|
||||
"draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'"
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("edited_at", lq.order_desc.unwrap_or(true))
|
||||
.and_where("o.workspace_id = ?".bind(&w_id))
|
||||
@@ -96,8 +85,9 @@ async fn list_flows(
|
||||
.limit(per_page)
|
||||
.clone();
|
||||
|
||||
sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false));
|
||||
|
||||
if !lq.show_archived.unwrap_or(false) {
|
||||
sqlb.and_where_eq("archived", false);
|
||||
}
|
||||
if let Some(ps) = &lq.path_start {
|
||||
sqlb.and_where_like_left("path", "?".bind(ps));
|
||||
}
|
||||
@@ -203,7 +193,7 @@ async fn create_flow(
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
|
||||
schema, dependency_job, draft_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8)",
|
||||
schema, dependency_job) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL)",
|
||||
w_id,
|
||||
nf.path,
|
||||
nf.summary,
|
||||
@@ -211,15 +201,6 @@ async fn create_flow(
|
||||
nf.value,
|
||||
&authed.username,
|
||||
nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
|
||||
nf.draft_only
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
|
||||
nf.path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
@@ -261,7 +242,6 @@ async fn create_flow(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -328,7 +308,7 @@ async fn update_flow(
|
||||
let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
|
||||
sqlx::query!(
|
||||
"UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
|
||||
edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL WHERE path = $7 AND workspace_id = $8",
|
||||
edited_at = now(), schema = $6::text::json, dependency_job = NULL WHERE path = $7 AND workspace_id = $8",
|
||||
nf.path,
|
||||
nf.summary,
|
||||
nf.description,
|
||||
@@ -345,65 +325,58 @@ async fn update_flow(
|
||||
check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&authed, flow_path)?;
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut schedulables: Vec<Schedule> = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
let mut schedulables: Vec<Schedule> = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
nf.path,
|
||||
flow_path,
|
||||
w_id,
|
||||
)
|
||||
.fetch_all(&mut tx)
|
||||
.await?;
|
||||
|
||||
let schedule = sqlx::query_as!(Schedule,
|
||||
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
nf.path,
|
||||
flow_path,
|
||||
w_id,
|
||||
)
|
||||
.fetch_all(&mut tx)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
|
||||
let schedule = sqlx::query_as!(Schedule,
|
||||
"UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
|
||||
nf.path,
|
||||
flow_path,
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
|
||||
if let Some(schedule) = schedule {
|
||||
schedulables.push(schedule);
|
||||
}
|
||||
|
||||
for schedule in schedulables.into_iter() {
|
||||
clear_schedule(tx.transaction_mut(), &schedule.path, true).await?;
|
||||
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
if let Some(schedule) = schedule {
|
||||
schedulables.push(schedule);
|
||||
}
|
||||
|
||||
for schedule in schedulables.into_iter() {
|
||||
// TODO: Why is this in the loop in the first place? Seems like it's just doing nothing after the first iteration? Should this use schedule.path?
|
||||
clear_schedule(tx.transaction_mut(), flow_path, true).await?;
|
||||
|
||||
if schedule.enabled {
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
}
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
"flows.update",
|
||||
ActionKind::Create,
|
||||
&w_id,
|
||||
Some(&nf.path.to_string()),
|
||||
Some(
|
||||
[Some(("flow", nf.path.as_str()))]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
|
||||
flow_path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
"flows.update",
|
||||
ActionKind::Create,
|
||||
&w_id,
|
||||
Some(&nf.path.to_string()),
|
||||
Some(
|
||||
[Some(("flow", nf.path.as_str()))]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::UpdateFlow {
|
||||
@@ -429,7 +402,6 @@ async fn update_flow(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
@@ -472,44 +444,6 @@ async fn get_flow_by_path(
|
||||
Ok(Json(flow))
|
||||
}
|
||||
|
||||
#[derive(Serialize, sqlx::FromRow)]
|
||||
pub struct FlowWDraft {
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub description: String,
|
||||
pub schema: Option<Schema>,
|
||||
pub value: serde_json::Value,
|
||||
pub extra_perms: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
async fn get_flow_by_path_w_draft(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<FlowWDraft> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let flow_o = sqlx::query_as::<_, FlowWDraft>(
|
||||
"SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft FROM flow
|
||||
LEFT JOIN draft ON
|
||||
flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow'
|
||||
WHERE flow.path = $1 AND flow.workspace_id = $2",
|
||||
)
|
||||
.bind(path)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let flow = not_found_if_none(flow_o, "Flow", path)?;
|
||||
Ok(Json(flow))
|
||||
}
|
||||
|
||||
async fn exists_flow_by_path(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
@@ -528,24 +462,17 @@ async fn exists_flow_by_path(
|
||||
Ok(Json(exists))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Archived {
|
||||
archived: Option<bool>,
|
||||
}
|
||||
|
||||
async fn archive_flow_by_path(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(archived): Json<Archived>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3",
|
||||
archived.archived.unwrap_or(true),
|
||||
"UPDATE flow SET archived = true WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
@@ -580,14 +507,6 @@ async fn delete_flow_by_path(
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
|
||||
path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM flow WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
@@ -659,7 +578,6 @@ mod tests {
|
||||
language: scripts::ScriptLang::Deno,
|
||||
path: None,
|
||||
lock: None,
|
||||
tag: None,
|
||||
},
|
||||
stop_after_if: Some(StopAfterIf {
|
||||
expr: "foo = 'bar'".to_string(),
|
||||
|
||||
@@ -22,13 +22,13 @@ use lazy_static::lazy_static;
|
||||
use regex::Regex;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{self, to_anyhow, JsonResult, Result},
|
||||
error::{self, to_anyhow, Error, JsonResult, Result},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, paginate, Pagination},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{FromRow, Postgres, Transaction};
|
||||
use sqlx::{query_scalar, FromRow, Postgres, Transaction};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
@@ -41,7 +41,7 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/delete/:name", delete(delete_folder))
|
||||
.route("/addowner/:name", post(add_owner))
|
||||
.route("/removeowner/:name", post(remove_owner))
|
||||
.route("/is_owner/*path", get(is_owner_api))
|
||||
.route("/is_owner/*path", get(is_owner))
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize, Clone)]
|
||||
@@ -219,29 +219,47 @@ async fn create_folder(
|
||||
Ok(format!("Created folder {}", ng.name))
|
||||
}
|
||||
|
||||
pub async fn is_owner_api(
|
||||
authed: Authed,
|
||||
Path((_w_id, name)): Path<(String, String)>,
|
||||
pub async fn is_owner(
|
||||
Authed { username, is_admin, groups, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, name)): Path<(String, String)>,
|
||||
) -> JsonResult<bool> {
|
||||
Ok(Json(is_owner(&authed, &name)))
|
||||
}
|
||||
|
||||
pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
|
||||
if *is_admin {
|
||||
true
|
||||
if is_admin {
|
||||
Ok(Json(true))
|
||||
} else {
|
||||
folders.into_iter().any(|x| x.0 == name && x.2)
|
||||
Ok(Json(
|
||||
require_is_owner(&name, &username, &groups, &w_id, &db)
|
||||
.await
|
||||
.is_ok(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
|
||||
if is_owner(authed, name) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(windmill_common::error::Error::NotAuthorized(format!(
|
||||
"You are not owner of the folder {}",
|
||||
name
|
||||
pub async fn require_is_owner(
|
||||
folder_name: &str,
|
||||
username: &str,
|
||||
groups: &Vec<String>,
|
||||
w_id: &str,
|
||||
db: &DB,
|
||||
) -> Result<()> {
|
||||
let is_owner = query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
|
||||
SELECT 1 FROM folder, unnest(folder.owners) as o
|
||||
WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
|
||||
username,
|
||||
folder_name,
|
||||
groups,
|
||||
w_id,
|
||||
).fetch_one(db)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
if !is_owner {
|
||||
Err(Error::BadRequest(format!(
|
||||
"{} is not an owner of {} and hence is not authorized to perform this operation",
|
||||
username, folder_name
|
||||
)))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -473,6 +491,7 @@ async fn delete_folder(
|
||||
|
||||
async fn add_owner(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, name)): Path<(String, String)>,
|
||||
@@ -481,7 +500,9 @@ async fn add_owner(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
|
||||
require_is_owner(&authed, &name)?;
|
||||
if !authed.is_admin {
|
||||
require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
|
||||
@@ -517,14 +538,14 @@ pub async fn get_folders_for_user(
|
||||
username: &str,
|
||||
groups: &[String],
|
||||
db: &DB,
|
||||
) -> Result<Vec<(String, bool, bool)>> {
|
||||
) -> Result<Vec<(String, bool)>> {
|
||||
let mut perms = groups
|
||||
.into_iter()
|
||||
.map(|x| format!("g/{}", x))
|
||||
.collect::<Vec<_>>();
|
||||
perms.insert(0, format!("u/{}", username));
|
||||
let folders = sqlx::query!(
|
||||
"SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t WHERE value::boolean IS true)) as write, $1 && owners::text[] as owner FROM folder
|
||||
"SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t WHERE value::boolean IS true)) as write FROM folder
|
||||
WHERE extra_perms ?| $1 AND workspace_id = $2",
|
||||
&perms[..],
|
||||
w_id,
|
||||
@@ -532,7 +553,7 @@ pub async fn get_folders_for_user(
|
||||
.fetch_all(db)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|x| (x.name, x.write.unwrap_or(false), x.owner.unwrap_or(false)))
|
||||
.map(|x| (x.name, x.write.unwrap_or(false)))
|
||||
.collect();
|
||||
|
||||
Ok(folders)
|
||||
@@ -540,6 +561,7 @@ pub async fn get_folders_for_user(
|
||||
|
||||
async fn remove_owner(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, name)): Path<(String, String)>,
|
||||
@@ -548,7 +570,9 @@ async fn remove_owner(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
|
||||
require_is_owner(&authed, &name)?;
|
||||
if !authed.is_admin {
|
||||
require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",
|
||||
|
||||
@@ -43,29 +43,25 @@ async fn add_granular_acl(
|
||||
Json(GranularAcl { owner, write }): Json<GranularAcl>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
|
||||
let (kind, path) = path
|
||||
.split_once('/')
|
||||
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
if !authed.is_admin {
|
||||
if kind == "folder" {
|
||||
crate::folders::require_is_owner(&path, &authed.username, &authed.groups, &w_id, &db)
|
||||
.await?;
|
||||
} else if kind == "group_" {
|
||||
} else {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
}
|
||||
let identifier = if kind == "group_" || kind == "folder" {
|
||||
"name"
|
||||
} else {
|
||||
"path"
|
||||
};
|
||||
|
||||
if !authed.is_admin {
|
||||
if kind == "folder" {
|
||||
crate::folders::require_is_owner(&authed, path)?;
|
||||
} else if kind == "group_" {
|
||||
crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
|
||||
.await?;
|
||||
} else {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
}
|
||||
|
||||
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
|
||||
"UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
|
||||
true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
|
||||
@@ -90,22 +86,12 @@ async fn remove_granular_acl(
|
||||
Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
let (kind, path) = path
|
||||
.split_once('/')
|
||||
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
|
||||
|
||||
if !authed.is_admin {
|
||||
if kind == "folder" {
|
||||
crate::folders::require_is_owner(&authed, path)?;
|
||||
} else if kind == "group_" {
|
||||
crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
|
||||
.await?;
|
||||
} else {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let identifier = if kind == "group_" || kind == "folder" {
|
||||
@@ -113,11 +99,6 @@ async fn remove_granular_acl(
|
||||
} else {
|
||||
"path"
|
||||
};
|
||||
|
||||
if identifier == "path" {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
|
||||
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
|
||||
"UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
|
||||
workspace_id = $3 RETURNING extra_perms"
|
||||
|
||||
@@ -292,12 +292,6 @@ async fn delete_group(
|
||||
) -> Result<String> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
if name == "all" {
|
||||
return Err(Error::BadRequest(
|
||||
"The group 'all' is a special group that contains all users and cannot be deleted".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if !authed.is_admin {
|
||||
require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
|
||||
}
|
||||
|
||||
@@ -22,10 +22,11 @@ use std::{
|
||||
};
|
||||
use windmill_common::{
|
||||
error::JsonResult,
|
||||
jobs::JobKind,
|
||||
scripts::to_i64,
|
||||
utils::{paginate, Pagination},
|
||||
};
|
||||
use windmill_queue::JobKind;
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route("/history", get(get_input_history))
|
||||
@@ -112,9 +113,9 @@ async fn get_input_history(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let sql = &format!(
|
||||
"select * from (select distinct on (args) * from completed_job \
|
||||
"select distinct on (args) * from completed_job \
|
||||
where {} = $1 and job_kind = $2 and workspace_id = $3 \
|
||||
order by args, started_at desc) t ORDER BY started_at desc limit $4 offset $5",
|
||||
order by args, started_at desc limit $4 offset $5",
|
||||
r.runnable_type.column_name()
|
||||
);
|
||||
|
||||
|
||||
@@ -17,11 +17,11 @@ use axum::{
|
||||
extract::{FromRequest, Json, Path, Query},
|
||||
response::{IntoResponse, Response},
|
||||
routing::{get, post},
|
||||
Extension, Form, RequestExt, Router,
|
||||
Extension, Router,
|
||||
};
|
||||
use base64::Engine;
|
||||
use hmac::Mac;
|
||||
use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
|
||||
use hyper::{HeaderMap, Request, StatusCode};
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use sql_builder::{prelude::*, quote, SqlBuilder};
|
||||
use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction};
|
||||
@@ -31,27 +31,22 @@ use windmill_common::{
|
||||
error::{self, to_anyhow, Error},
|
||||
flow_status::{Approval, FlowStatus, FlowStatusModule},
|
||||
flows::FlowValue,
|
||||
jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
|
||||
oauth2::HmacSha256,
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
users::username_to_permissioned_as,
|
||||
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::{get_queued_job, push, QueueTransaction};
|
||||
use windmill_queue::{
|
||||
get_queued_job, push, JobKind, JobPayload, QueueTransaction, QueuedJob, RawCode,
|
||||
};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
"/run/f/*script_path",
|
||||
post(run_flow_by_path).head(|| async { "" }),
|
||||
)
|
||||
.route(
|
||||
"/run/p/*script_path",
|
||||
post(run_job_by_path).head(|| async { "" }),
|
||||
)
|
||||
.route("/run/f/*script_path", post(run_flow_by_path))
|
||||
.route("/run/p/*script_path", post(run_job_by_path))
|
||||
.route(
|
||||
"/run_wait_result/p/*script_path",
|
||||
post(run_wait_result_job_by_path).head(|| async { "" }),
|
||||
post(run_wait_result_job_by_path),
|
||||
)
|
||||
.route(
|
||||
"/run_wait_result/p/*script_path",
|
||||
@@ -59,13 +54,13 @@ pub fn workspaced_service() -> Router {
|
||||
)
|
||||
.route(
|
||||
"/run_wait_result/h/:hash",
|
||||
post(run_wait_result_job_by_hash).head(|| async { "" }),
|
||||
post(run_wait_result_job_by_hash),
|
||||
)
|
||||
.route(
|
||||
"/run_wait_result/f/*script_path",
|
||||
post(run_wait_result_flow_by_path).head(|| async { "" }),
|
||||
post(run_wait_result_flow_by_path),
|
||||
)
|
||||
.route("/run/h/:hash", post(run_job_by_hash).head(|| async { "" }))
|
||||
.route("/run/h/:hash", post(run_job_by_hash))
|
||||
.route("/run/preview", post(run_preview_job))
|
||||
.route("/run/preview_flow", post(run_preview_flow_job))
|
||||
.route("/list", get(list_jobs))
|
||||
@@ -74,10 +69,6 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/completed/list", get(list_completed_jobs))
|
||||
.route("/completed/get/:id", get(get_completed_job))
|
||||
.route("/completed/get_result/:id", get(get_completed_job_result))
|
||||
.route(
|
||||
"/completed/get_result_maybe/:id",
|
||||
get(get_completed_job_result_maybe),
|
||||
)
|
||||
.route("/completed/delete/:id", post(delete_completed_job))
|
||||
.route("/flow/resume/:id", post(resume_suspended_flow_as_owner))
|
||||
.route(
|
||||
@@ -230,26 +221,6 @@ pub async fn get_path_for_hash<'c>(
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub async fn get_path_and_tag_for_hash<'c>(
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &str,
|
||||
hash: i64,
|
||||
) -> error::Result<(String, Option<String>)> {
|
||||
let script = sqlx::query!(
|
||||
"select path, tag from script where hash = $1 AND workspace_id = $2",
|
||||
hash,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
Error::InternalErr(format!(
|
||||
"querying getting path for hash {hash} in {w_id}: {e}"
|
||||
))
|
||||
})?;
|
||||
Ok((script.path, script.tag))
|
||||
}
|
||||
|
||||
async fn get_job(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
@@ -336,7 +307,6 @@ pub struct CompletedJob {
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone)]
|
||||
@@ -421,7 +391,6 @@ pub struct ListQueueQuery {
|
||||
pub suspended: Option<bool>,
|
||||
// filter by matching a subset of the args using base64 encoded json subset
|
||||
pub args: Option<String>,
|
||||
pub tag: Option<String>,
|
||||
}
|
||||
|
||||
fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
|
||||
@@ -444,9 +413,6 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
|
||||
if let Some(cb) = &lq.created_by {
|
||||
sqlb.and_where_eq("created_by", "?".bind(cb));
|
||||
}
|
||||
if let Some(t) = &lq.tag {
|
||||
sqlb.and_where_eq("tag", "?".bind(t));
|
||||
}
|
||||
if let Some(r) = &lq.running {
|
||||
sqlb.and_where_eq("running", &r);
|
||||
}
|
||||
@@ -579,7 +545,6 @@ async fn list_jobs(
|
||||
job_kinds: lq.job_kinds,
|
||||
suspended: lq.suspended,
|
||||
args: lq.args,
|
||||
tag: lq.tag,
|
||||
},
|
||||
&[
|
||||
"'QueuedJob' as typ",
|
||||
@@ -609,7 +574,6 @@ async fn list_jobs(
|
||||
"visible_to_owner",
|
||||
"suspend",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
],
|
||||
);
|
||||
let sqlc = list_completed_jobs_query(
|
||||
@@ -645,7 +609,6 @@ async fn list_jobs(
|
||||
"visible_to_owner",
|
||||
"null as suspend",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
],
|
||||
);
|
||||
let sql = format!(
|
||||
@@ -664,7 +627,7 @@ async fn list_jobs(
|
||||
pub async fn resume_suspended_flow_as_owner(
|
||||
authed: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((_w_id, flow_id)): Path<(String, Uuid)>,
|
||||
Path((w_id, flow_id)): Path<(String, Uuid)>,
|
||||
QueryOrBody(value): QueryOrBody<serde_json::Value>,
|
||||
) -> error::Result<StatusCode> {
|
||||
let value = value.unwrap_or(serde_json::Value::Null);
|
||||
@@ -672,11 +635,16 @@ pub async fn resume_suspended_flow_as_owner(
|
||||
|
||||
let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?;
|
||||
|
||||
require_owner_of_path(
|
||||
&authed,
|
||||
&flow.script_path.clone().unwrap_or_else(|| String::new()),
|
||||
)?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(
|
||||
&w_id,
|
||||
&authed.username,
|
||||
&authed.groups,
|
||||
&flow.script_path.clone().unwrap_or_else(|| String::new()),
|
||||
&db,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
insert_resume_job(0, job_id, &flow, value, Some(authed.username), &mut tx).await?;
|
||||
|
||||
resume_immediately_if_relevant(flow, job_id, &mut tx).await?;
|
||||
@@ -703,7 +671,7 @@ pub async fn resume_suspended_job(
|
||||
}
|
||||
mac.verify_slice(hex::decode(secret)?.as_ref())
|
||||
.map_err(|_| anyhow::anyhow!("Invalid signature"))?;
|
||||
let flow = get_suspended_parent_flow_info(job_id, &mut tx).await?;
|
||||
let flow = get_suspended_prent_flow_info(job_id, &mut tx).await?;
|
||||
|
||||
let exists = sqlx::query_scalar!(
|
||||
r#"
|
||||
@@ -794,7 +762,7 @@ struct FlowInfo {
|
||||
script_path: Option<String>,
|
||||
}
|
||||
|
||||
async fn get_suspended_parent_flow_info<'c>(
|
||||
async fn get_suspended_prent_flow_info<'c>(
|
||||
job_id: Uuid,
|
||||
tx: &mut Transaction<'c, Postgres>,
|
||||
) -> error::Result<FlowInfo> {
|
||||
@@ -865,9 +833,8 @@ pub async fn cancel_suspended_job(
|
||||
.map_err(|_| anyhow::anyhow!("Invalid signature"))?;
|
||||
|
||||
let whom = approver.approver.unwrap_or_else(|| "unknown".to_string());
|
||||
let parent_flow = get_suspended_parent_flow_info(job, &mut tx).await?.id;
|
||||
|
||||
let (mut tx, cjob) = windmill_queue::cancel_job(
|
||||
let parent_flow = get_root_job(db, &w_id, job).await?;
|
||||
let (mut tx, job) = windmill_queue::cancel_job(
|
||||
&whom,
|
||||
Some("approval request disapproved".to_string()),
|
||||
parent_flow,
|
||||
@@ -877,7 +844,7 @@ pub async fn cancel_suspended_job(
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
if cjob.is_some() {
|
||||
if job.is_some() {
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&whom,
|
||||
@@ -888,13 +855,26 @@ pub async fn cancel_suspended_job(
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok("Flow of job cancelled".to_string())
|
||||
}
|
||||
|
||||
Ok(format!("Flow {parent_flow} of job {job} cancelled"))
|
||||
} else {
|
||||
Ok(format!(
|
||||
"Flow {parent_flow} of job {job} was not cancellable"
|
||||
))
|
||||
pub async fn get_root_job(db: DB, w_id: &str, job: Uuid) -> error::Result<Uuid> {
|
||||
let mut tx = db.begin().await?;
|
||||
let mut job_id = job;
|
||||
loop {
|
||||
let (job, ntx) = get_job_by_id(tx, w_id, job_id).await?;
|
||||
tx = ntx;
|
||||
let p_job = job.and_then(|x| match x {
|
||||
Job::QueuedJob(job) => job.parent_job,
|
||||
Job::CompletedJob(job) => job.parent_job,
|
||||
});
|
||||
if let Some(p_job) = p_job {
|
||||
job_id = p_job;
|
||||
} else {
|
||||
return Ok(job_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1110,7 +1090,6 @@ struct UnifiedJob {
|
||||
visible_to_owner: bool,
|
||||
suspend: Option<i32>,
|
||||
mem_peak: Option<i32>,
|
||||
tag: String,
|
||||
}
|
||||
|
||||
impl From<UnifiedJob> for Job {
|
||||
@@ -1146,7 +1125,6 @@ impl From<UnifiedJob> for Job {
|
||||
email: uj.email,
|
||||
visible_to_owner: uj.visible_to_owner,
|
||||
mem_peak: uj.mem_peak,
|
||||
tag: uj.tag,
|
||||
}),
|
||||
"QueuedJob" => Job::QueuedJob(QueuedJob {
|
||||
workspace_id: uj.workspace_id,
|
||||
@@ -1182,7 +1160,6 @@ impl From<UnifiedJob> for Job {
|
||||
mem_peak: uj.mem_peak,
|
||||
root_job: None,
|
||||
leaf_jobs: None,
|
||||
tag: uj.tag,
|
||||
}),
|
||||
t => panic!("job type {} not valid", t),
|
||||
}
|
||||
@@ -1199,7 +1176,6 @@ struct Preview {
|
||||
path: Option<String>,
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
language: ScriptLang,
|
||||
tag: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -1209,40 +1185,6 @@ struct PreviewFlow {
|
||||
args: Option<serde_json::Map<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
pub struct JsonOrForm<T>(T);
|
||||
|
||||
#[axum::async_trait]
|
||||
impl<S> FromRequest<S, axum::body::Body>
|
||||
for JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>
|
||||
where
|
||||
S: Send + Sync,
|
||||
Json<Option<serde_json::Map<String, serde_json::Value>>>: FromRequest<(), axum::body::Body>,
|
||||
Form<serde_json::Map<String, serde_json::Value>>: FromRequest<(), axum::body::Body>,
|
||||
{
|
||||
type Rejection = Response;
|
||||
|
||||
async fn from_request(
|
||||
req: Request<axum::body::Body>,
|
||||
_state: &S,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
let content_type_header = req.headers().get(CONTENT_TYPE);
|
||||
let content_type = content_type_header.and_then(|value| value.to_str().ok());
|
||||
|
||||
if let Some(content_type) = content_type {
|
||||
if content_type.starts_with("application/json") {
|
||||
let Json(payload) = req.extract().await.map_err(IntoResponse::into_response)?;
|
||||
return Ok(Self(payload));
|
||||
}
|
||||
|
||||
if content_type.starts_with("application/x-www-form-urlencoded") {
|
||||
let Form(payload) = req.extract().await.map_err(IntoResponse::into_response)?;
|
||||
return Ok(Self(Some(payload)));
|
||||
}
|
||||
}
|
||||
|
||||
Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
|
||||
}
|
||||
}
|
||||
pub struct QueryOrBody<D>(pub Option<D>);
|
||||
|
||||
#[axum::async_trait]
|
||||
@@ -1296,12 +1238,13 @@ pub async fn run_flow_by_path(
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args): JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let flow_path = flow_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
let (uuid, tx) = push(
|
||||
tx,
|
||||
&w_id,
|
||||
@@ -1318,7 +1261,6 @@ pub async fn run_flow_by_path(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1332,12 +1274,11 @@ pub async fn run_job_by_path(
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args): JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let script_path = script_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let (job_payload, tag) =
|
||||
script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
@@ -1357,7 +1298,6 @@ pub async fn run_job_by_path(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1407,13 +1347,14 @@ async fn run_wait_result<T>(
|
||||
uuid: Uuid,
|
||||
Path((w_id, _)): Path<(String, T)>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
let mut result;
|
||||
let timeout_ms = if timeout <= 0 {
|
||||
2000
|
||||
let mut result = None;
|
||||
let iters = if timeout <= 0 {
|
||||
20
|
||||
} else if timeout <= 1 {
|
||||
timeout * 10
|
||||
} else {
|
||||
(timeout * 1000) as u64
|
||||
10 + ((timeout - 1) * 2)
|
||||
};
|
||||
|
||||
let mut g = Guard {
|
||||
done: false,
|
||||
id: uuid,
|
||||
@@ -1421,10 +1362,7 @@ async fn run_wait_result<T>(
|
||||
db: user_db.clone(),
|
||||
authed: authed.clone(),
|
||||
};
|
||||
|
||||
let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
|
||||
let mut accumulated_delay = 0 as u64;
|
||||
loop {
|
||||
for i in 0..iters {
|
||||
let mut tx = user_db.clone().begin(&authed).await?;
|
||||
result = sqlx::query_scalar!(
|
||||
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
@@ -1439,27 +1377,11 @@ async fn run_wait_result<T>(
|
||||
if result.is_some() {
|
||||
break;
|
||||
}
|
||||
|
||||
let delay = if accumulated_delay <= fast_poll_duration {
|
||||
*WAIT_RESULT_FAST_POLL_INTERVAL_MS
|
||||
} else {
|
||||
*WAIT_RESULT_SLOW_POLL_INTERVAL_MS
|
||||
};
|
||||
accumulated_delay += delay;
|
||||
if accumulated_delay > timeout_ms {
|
||||
break;
|
||||
};
|
||||
let delay = if i < 10 { 100 } else { 500 };
|
||||
tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
|
||||
}
|
||||
if let Some(result) = result {
|
||||
g.done = true;
|
||||
let status_code = result
|
||||
.get("windmill_status_code")
|
||||
.and_then(|x| x.as_i64())
|
||||
.and_then(|x| StatusCode::from_u16(x as u16).ok());
|
||||
if let Some(status_code) = status_code {
|
||||
return Err(Error::CustomStatusCode(status_code, result));
|
||||
}
|
||||
Ok(Json(result))
|
||||
} else {
|
||||
Err(Error::ExecutionErr(format!("timeout after {}s", timeout)))
|
||||
@@ -1495,22 +1417,9 @@ lazy_static::lazy_static! {
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(20);
|
||||
pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(50);
|
||||
pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(2);
|
||||
pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse().ok())
|
||||
.unwrap_or(200);
|
||||
}
|
||||
|
||||
pub async fn run_wait_result_job_by_path_get(
|
||||
method: hyper::http::Method,
|
||||
authed: Authed,
|
||||
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -1518,9 +1427,6 @@ pub async fn run_wait_result_job_by_path_get(
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
if method == http::Method::HEAD {
|
||||
return Ok(Json(serde_json::json!("")));
|
||||
}
|
||||
let payload_r = run_query
|
||||
.payload
|
||||
.map(decode_payload)
|
||||
@@ -1535,8 +1441,7 @@ pub async fn run_wait_result_job_by_path_get(
|
||||
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
|
||||
let script_path = script_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let (job_payload, tag) =
|
||||
script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
|
||||
let (uuid, tx) = push(
|
||||
tx,
|
||||
@@ -1554,7 +1459,6 @@ pub async fn run_wait_result_job_by_path_get(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1577,13 +1481,12 @@ pub async fn run_wait_result_job_by_path(
|
||||
Path((w_id, script_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args): JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
|
||||
let script_path = script_path.to_path();
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let (job_payload, tag) =
|
||||
script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
@@ -1603,7 +1506,6 @@ pub async fn run_wait_result_job_by_path(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1626,13 +1528,13 @@ pub async fn run_wait_result_job_by_hash(
|
||||
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args): JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
check_queue_too_long(db, run_query.queue_limit).await?;
|
||||
|
||||
let hash = script_hash.0;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
|
||||
let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
|
||||
let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
|
||||
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
@@ -1652,7 +1554,6 @@ pub async fn run_wait_result_job_by_hash(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1675,7 +1576,7 @@ pub async fn run_wait_result_flow_by_path(
|
||||
Path((w_id, flow_path)): Path<(String, StripPath)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args): JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::JsonResult<serde_json::Value> {
|
||||
check_queue_too_long(db, run_query.queue_limit).await?;
|
||||
|
||||
@@ -1700,7 +1601,6 @@ pub async fn run_wait_result_flow_by_path(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1715,6 +1615,21 @@ pub async fn run_wait_result_flow_by_path(
|
||||
.await
|
||||
}
|
||||
|
||||
// a similar function exists on the worker
|
||||
pub async fn script_path_to_payload<'c>(
|
||||
script_path: &str,
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &String,
|
||||
) -> std::result::Result<JobPayload, Error> {
|
||||
let job_payload = if script_path.starts_with("hub/") {
|
||||
JobPayload::ScriptHub { path: script_path.to_owned() }
|
||||
} else {
|
||||
let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
|
||||
JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
|
||||
};
|
||||
Ok(job_payload)
|
||||
}
|
||||
|
||||
async fn run_preview_job(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -1749,7 +1664,6 @@ async fn run_preview_job(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
preview.tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1786,7 +1700,6 @@ async fn run_preview_flow_job(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1801,11 +1714,11 @@ pub async fn run_job_by_hash(
|
||||
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
|
||||
Query(run_query): Query<RunJobQuery>,
|
||||
headers: HeaderMap,
|
||||
JsonOrForm(args): JsonOrForm<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
|
||||
) -> error::Result<(StatusCode, String)> {
|
||||
let hash = script_hash.0;
|
||||
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
|
||||
let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
|
||||
let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
|
||||
let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
|
||||
let args = run_query.add_include_headers(headers, args.unwrap_or_default());
|
||||
|
||||
@@ -1825,7 +1738,6 @@ pub async fn run_job_by_hash(
|
||||
false,
|
||||
None,
|
||||
!run_query.invisible_to_owner.unwrap_or(false),
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
@@ -1916,9 +1828,6 @@ fn list_completed_jobs_query(
|
||||
if let Some(h) = &lq.script_hash {
|
||||
sqlb.and_where_eq("script_hash", "?".bind(h));
|
||||
}
|
||||
if let Some(t) = &lq.tag {
|
||||
sqlb.and_where_eq("tag", "?".bind(t));
|
||||
}
|
||||
if let Some(cb) = &lq.created_by {
|
||||
sqlb.and_where_eq("created_by", "?".bind(cb));
|
||||
}
|
||||
@@ -1976,7 +1885,6 @@ pub struct ListCompletedQuery {
|
||||
pub args: Option<String>,
|
||||
// filter by matching a subset of the result using base64 encoded json subset
|
||||
pub result: Option<String>,
|
||||
pub tag: Option<String>,
|
||||
}
|
||||
|
||||
async fn list_completed_jobs(
|
||||
@@ -2022,7 +1930,6 @@ async fn list_completed_jobs(
|
||||
"email",
|
||||
"visible_to_owner",
|
||||
"mem_peak",
|
||||
"tag",
|
||||
],
|
||||
)
|
||||
.sql()?;
|
||||
@@ -2065,31 +1972,6 @@ async fn get_completed_job_result(
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CompletedJobResult {
|
||||
completed: bool,
|
||||
result: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
async fn get_completed_job_result_maybe(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, id)): Path<(String, Uuid)>,
|
||||
) -> error::JsonResult<CompletedJobResult> {
|
||||
let result_o = sqlx::query_scalar!(
|
||||
"SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
|
||||
id,
|
||||
w_id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?;
|
||||
|
||||
if let Some(result) = result_o {
|
||||
Ok(Json(CompletedJobResult { completed: true, result }))
|
||||
} else {
|
||||
Ok(Json(CompletedJobResult { completed: false, result: None }))
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_completed_job(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
|
||||
@@ -33,7 +33,6 @@ mod apps;
|
||||
mod audit;
|
||||
mod capture;
|
||||
mod db;
|
||||
mod drafts;
|
||||
mod favorite;
|
||||
mod flows;
|
||||
mod folders;
|
||||
@@ -42,7 +41,6 @@ mod groups;
|
||||
mod inputs;
|
||||
pub mod jobs;
|
||||
mod oauth2;
|
||||
mod raw_apps;
|
||||
mod resources;
|
||||
mod schedule;
|
||||
mod scripts;
|
||||
@@ -52,7 +50,7 @@ mod users;
|
||||
mod utils;
|
||||
mod variables;
|
||||
mod webhook_util;
|
||||
mod workers;
|
||||
mod worker_ping;
|
||||
mod workspaces;
|
||||
|
||||
pub const GIT_VERSION: &str =
|
||||
@@ -124,7 +122,6 @@ pub async fn run_server(
|
||||
// Reordered alphabetically
|
||||
.nest("/acls", granular_acls::workspaced_service())
|
||||
.nest("/apps", apps::workspaced_service())
|
||||
.nest("/raw_apps", raw_apps::workspaced_service())
|
||||
.nest("/audit", audit::workspaced_service())
|
||||
.nest("/capture", capture::workspaced_service())
|
||||
.nest("/favorites", favorite::workspaced_service())
|
||||
@@ -137,7 +134,6 @@ pub async fn run_server(
|
||||
.nest("/resources", resources::workspaced_service())
|
||||
.nest("/schedules", schedule::workspaced_service())
|
||||
.nest("/scripts", scripts::workspaced_service())
|
||||
.nest("/drafts", drafts::workspaced_service())
|
||||
.nest(
|
||||
"/users",
|
||||
users::workspaced_service().layer(Extension(argon2.clone())),
|
||||
@@ -150,7 +146,7 @@ pub async fn run_server(
|
||||
"/users",
|
||||
users::global_service().layer(Extension(argon2.clone())),
|
||||
)
|
||||
.nest("/workers", workers::global_service())
|
||||
.nest("/workers", worker_ping::global_service())
|
||||
.nest("/scripts", scripts::global_service())
|
||||
.nest("/flows", flows::global_service())
|
||||
.nest("/apps", apps::global_service().layer(cors.clone()))
|
||||
|
||||
@@ -19,7 +19,6 @@ use axum::{
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use base64::Engine;
|
||||
use hmac::Mac;
|
||||
use hyper::StatusCode;
|
||||
use itertools::Itertools;
|
||||
@@ -30,7 +29,6 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use tower_cookies::{Cookie, Cookies};
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::jobs::JobPayload;
|
||||
use windmill_common::users::username_to_permissioned_as;
|
||||
use windmill_common::utils::{not_found_if_none, now_from_db};
|
||||
|
||||
@@ -46,7 +44,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
|
||||
use windmill_common::error::{self, to_anyhow, Error};
|
||||
use windmill_common::oauth2::*;
|
||||
|
||||
use windmill_queue::QueueTransaction;
|
||||
use windmill_queue::{JobPayload, QueueTransaction};
|
||||
|
||||
use std::{fs, str};
|
||||
|
||||
@@ -120,14 +118,7 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
|
||||
))?;
|
||||
|
||||
let path = "./oauth.json";
|
||||
let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
|
||||
str::from_utf8(
|
||||
&base64::engine::general_purpose::STANDARD
|
||||
.decode(e)
|
||||
.map_err(to_anyhow)?,
|
||||
)?
|
||||
.to_string()
|
||||
} else if std::path::Path::new(path).exists() {
|
||||
let content = if std::path::Path::new(path).exists() {
|
||||
fs::read_to_string(path).map_err(to_anyhow)?
|
||||
} else {
|
||||
"{}".to_string()
|
||||
@@ -762,20 +753,17 @@ async fn slack_command(
|
||||
|
||||
if let Some(settings) = settings {
|
||||
if let Some(path) = &settings.slack_command_script {
|
||||
let (payload, tag) = if let Some(path) = path.strip_prefix("flow/") {
|
||||
(JobPayload::Flow(path.to_string()), None)
|
||||
let payload = if let Some(path) = path.strip_prefix("flow/") {
|
||||
JobPayload::Flow(path.to_string())
|
||||
} else {
|
||||
let path = path.strip_prefix("script/").unwrap_or_else(|| path);
|
||||
let (script_hash, tag) = windmill_common::get_latest_deployed_hash_for_path(
|
||||
let script_hash = windmill_common::get_latest_hash_for_path(
|
||||
tx.transaction_mut(),
|
||||
&settings.workspace_id,
|
||||
path,
|
||||
)
|
||||
.await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() },
|
||||
tag,
|
||||
)
|
||||
JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() }
|
||||
};
|
||||
let mut map = serde_json::Map::new();
|
||||
map.insert("text".to_string(), serde_json::Value::String(form.text));
|
||||
@@ -800,7 +788,6 @@ async fn slack_command(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
let url = BASE_URL.to_owned();
|
||||
|
||||
@@ -1,333 +0,0 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2022
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
use crate::{
|
||||
db::{UserDB, DB},
|
||||
users::{require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
};
|
||||
use axum::{
|
||||
body,
|
||||
extract::{Extension, Json, Path, Query},
|
||||
response::Response,
|
||||
routing::{delete, get, post},
|
||||
Router,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use hyper::{header, StatusCode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sql_builder::{bind::Bind, SqlBuilder};
|
||||
use sqlx::FromRow;
|
||||
use std::str;
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
apps::ListAppQuery,
|
||||
error::{Error, JsonResult, Result},
|
||||
utils::{not_found_if_none, paginate, Pagination, StripPath},
|
||||
};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_apps))
|
||||
.route("/get_data/:version/*path", get(get_data))
|
||||
.route("/exists/*path", get(exists_app))
|
||||
.route("/update/*path", post(update_app))
|
||||
.route("/delete/*path", delete(delete_app))
|
||||
.route("/create", post(create_app))
|
||||
}
|
||||
|
||||
#[derive(FromRow, Deserialize, Serialize)]
|
||||
pub struct ListableApp {
|
||||
pub path: String,
|
||||
pub workspace_id: String,
|
||||
pub summary: String,
|
||||
pub edited_at: chrono::DateTime<chrono::Utc>,
|
||||
pub extra_perms: serde_json::Value,
|
||||
pub starred: bool,
|
||||
pub version: i32,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct CreateApp {
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct EditApp {
|
||||
pub path: Option<String>,
|
||||
pub summary: Option<String>,
|
||||
pub value: Option<String>,
|
||||
}
|
||||
|
||||
async fn list_apps(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(pagination): Query<Pagination>,
|
||||
Query(lq): Query<ListAppQuery>,
|
||||
) -> JsonResult<Vec<ListableApp>> {
|
||||
let (per_page, offset) = paginate(pagination);
|
||||
|
||||
let mut sqlb = SqlBuilder::select_from("raw_app as app")
|
||||
.fields(&[
|
||||
"app.workspace_id",
|
||||
"app.path",
|
||||
"app.edited_at",
|
||||
"app.summary",
|
||||
"app.extra_perms",
|
||||
"app.version",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
.on(
|
||||
"favorite.favorite_kind = 'raw_app' AND favorite.workspace_id = app.workspace_id AND favorite.path = app.path AND favorite.usr = ?"
|
||||
.bind(&authed.username),
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("app.edited_at", true)
|
||||
.and_where("app.workspace_id = ?".bind(&w_id))
|
||||
.offset(offset)
|
||||
.limit(per_page)
|
||||
.clone();
|
||||
|
||||
if lq.starred_only.unwrap_or(false) {
|
||||
sqlb.and_where_is_not_null("favorite.path");
|
||||
}
|
||||
|
||||
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let rows = sqlx::query_as::<_, ListableApp>(&sql)
|
||||
.fetch_all(&mut tx)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
async fn get_data(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, _version, path)): Path<(String, u16, StripPath)>,
|
||||
) -> Result<Response> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let app_o = sqlx::query_scalar!(
|
||||
"SELECT data FROM raw_app
|
||||
WHERE path = $1 AND workspace_id = $2",
|
||||
path.to_owned(),
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let app = not_found_if_none(app_o, "App", path)?;
|
||||
let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript");
|
||||
|
||||
Ok(res
|
||||
.body(body::boxed(body::Full::from(Bytes::from(app))))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
async fn create_app(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(app): Json<CreateApp>,
|
||||
) -> Result<(StatusCode, String)> {
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
if &app.path == "" {
|
||||
return Err(Error::BadRequest("App path cannot be empty".to_string()));
|
||||
}
|
||||
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
|
||||
app.path,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
if exists {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"App with path {} already exists",
|
||||
&app.path
|
||||
)));
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO raw_app
|
||||
(workspace_id, path, summary, extra_perms, data)
|
||||
VALUES ($1, $2, $3, '{}', $4)",
|
||||
w_id,
|
||||
app.path,
|
||||
app.summary,
|
||||
app.value,
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
"apps.create",
|
||||
ActionKind::Create,
|
||||
&w_id,
|
||||
Some(&app.path),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
|
||||
);
|
||||
|
||||
Ok((StatusCode::CREATED, app.path))
|
||||
}
|
||||
|
||||
async fn delete_app(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
sqlx::query!(
|
||||
"DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
"apps.delete",
|
||||
ActionKind::Delete,
|
||||
&w_id,
|
||||
Some(path),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
webhook.send_message(
|
||||
w_id.clone().clone(),
|
||||
WebhookMessage::DeleteApp { workspace: w_id, path: path.to_owned() },
|
||||
);
|
||||
|
||||
Ok(format!("app {} deleted", path))
|
||||
}
|
||||
|
||||
async fn update_app(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Extension(webhook): Extension<WebhookShared>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
Json(app): Json<EditApp>,
|
||||
) -> Result<String> {
|
||||
use sql_builder::prelude::*;
|
||||
|
||||
let path = path.to_path();
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let npath = &app.path;
|
||||
let npath = if npath.is_some() || app.summary.is_some() {
|
||||
let mut sqlb = SqlBuilder::update_table("raw_app");
|
||||
sqlb.and_where_eq("path", "?".bind(&path));
|
||||
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
|
||||
|
||||
if let Some(npath) = &app.path {
|
||||
if npath != path {
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
|
||||
npath,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
if exists {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"App with path {} already exists",
|
||||
npath
|
||||
)));
|
||||
}
|
||||
}
|
||||
sqlb.set_str("path", npath);
|
||||
}
|
||||
|
||||
if let Some(nsummary) = &app.summary {
|
||||
sqlb.set_str("summary", nsummary);
|
||||
}
|
||||
|
||||
if let Some(value) = &app.value {
|
||||
sqlb.set_str("data", value);
|
||||
sqlb.set("version", "version + 1");
|
||||
}
|
||||
|
||||
sqlb.returning("path");
|
||||
|
||||
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
|
||||
let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
|
||||
not_found_if_none(npath_o, "Raw App", path)?
|
||||
} else {
|
||||
"".to_string()
|
||||
};
|
||||
|
||||
audit_log(
|
||||
&mut tx,
|
||||
&authed.username,
|
||||
"apps.update",
|
||||
ActionKind::Update,
|
||||
&w_id,
|
||||
Some(&npath),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
webhook.send_message(
|
||||
w_id.clone(),
|
||||
WebhookMessage::UpdateApp {
|
||||
workspace: w_id,
|
||||
old_path: path.to_owned(),
|
||||
new_path: npath.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
Ok(format!("app {} updated (npath: {:?})", path, npath))
|
||||
}
|
||||
|
||||
async fn exists_app(
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<bool> {
|
||||
let path = path.to_path();
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
Ok(Json(exists))
|
||||
}
|
||||
@@ -386,8 +386,9 @@ async fn update_resource(
|
||||
if npath != path {
|
||||
check_path_conflict(&mut tx, &w_id, &npath).await?;
|
||||
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
sqlx::query!(
|
||||
"UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
|
||||
npath,
|
||||
|
||||
@@ -23,9 +23,9 @@ use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
schedule::Schedule,
|
||||
utils::{not_found_if_none, paginate, Pagination, StripPath}, jobs::JobKind,
|
||||
utils::{not_found_if_none, paginate, Pagination, StripPath},
|
||||
};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, JobKind, QueueTransaction};
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
|
||||
@@ -32,10 +32,9 @@ use std::{
|
||||
use windmill_audit::{audit_log, ActionKind};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
jobs::JobPayload,
|
||||
schedule::Schedule,
|
||||
scripts::{
|
||||
to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
|
||||
to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Script, ScriptHash,
|
||||
ScriptKind, ScriptLang,
|
||||
},
|
||||
users::username_to_permissioned_as,
|
||||
@@ -46,32 +45,8 @@ use windmill_common::{
|
||||
use windmill_parser::MainArgSignature;
|
||||
use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).filter(|x| !x.is_empty()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
}
|
||||
|
||||
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
|
||||
|
||||
#[derive(Serialize, sqlx::FromRow)]
|
||||
pub struct ScriptWDraft {
|
||||
pub hash: ScriptHash,
|
||||
pub path: String,
|
||||
pub summary: String,
|
||||
pub description: String,
|
||||
pub content: String,
|
||||
pub language: ScriptLang,
|
||||
pub kind: ScriptKind,
|
||||
pub tag: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft: Option<serde_json::Value>,
|
||||
pub schema: Option<Schema>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route(
|
||||
@@ -98,7 +73,6 @@ pub fn workspaced_service() -> Router {
|
||||
.route("/list", get(list_scripts))
|
||||
.route("/create", post(create_script))
|
||||
.route("/archive/p/*path", post(archive_script_by_path))
|
||||
.route("/get/draft/*path", get(get_script_by_path_w_draft))
|
||||
.route("/get/p/*path", get(get_script_by_path))
|
||||
.route("/raw/p/*path", get(raw_script_by_path))
|
||||
.route("/exists/p/*path", get(exists_script_by_path))
|
||||
@@ -122,18 +96,22 @@ async fn list_scripts(
|
||||
|
||||
let mut sqlb = SqlBuilder::select_from("script as o")
|
||||
.fields(&[
|
||||
"o.workspace_id",
|
||||
"hash",
|
||||
"o.path",
|
||||
"array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
|
||||
"summary",
|
||||
"COALESCE(draft.created_at, o.created_at) as created_at",
|
||||
"description",
|
||||
"created_by",
|
||||
"created_at",
|
||||
"archived",
|
||||
"deleted",
|
||||
"is_template",
|
||||
"extra_perms",
|
||||
"CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors",
|
||||
"CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
|
||||
"language",
|
||||
"kind",
|
||||
"favorite.path IS NOT NULL as starred",
|
||||
"tag",
|
||||
"draft.path IS NOT NULL as has_draft",
|
||||
"draft_only"
|
||||
])
|
||||
.left()
|
||||
.join("favorite")
|
||||
@@ -141,11 +119,6 @@ async fn list_scripts(
|
||||
"favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
|
||||
.bind(&authed.username),
|
||||
)
|
||||
.left()
|
||||
.join("draft")
|
||||
.on(
|
||||
"draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'"
|
||||
)
|
||||
.order_desc("favorite.path IS NOT NULL")
|
||||
.order_by("created_at", lq.order_desc.unwrap_or(true))
|
||||
.and_where("o.workspace_id = ?".bind(&w_id))
|
||||
@@ -155,7 +128,7 @@ async fn list_scripts(
|
||||
|
||||
if lq.show_archived.unwrap_or(false) {
|
||||
sqlb.and_where_eq(
|
||||
"o.created_at",
|
||||
"created_at",
|
||||
"(select max(created_at) from script where o.path = path
|
||||
AND workspace_id = ?)"
|
||||
.bind(&w_id),
|
||||
@@ -261,20 +234,10 @@ async fn create_script(
|
||||
}
|
||||
let parent_hashes_and_perms: Option<ParentInfo> = match (&ns.parent_hash, clashing_script) {
|
||||
(None, None) => Ok(None),
|
||||
(None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!(
|
||||
(None, Some(s)) => Err(Error::BadRequest(format!(
|
||||
"Path conflict for {} with non-archived hash {}",
|
||||
&ns.path, &s.hash
|
||||
))),
|
||||
(None, Some(s)) => {
|
||||
sqlx::query!(
|
||||
"DELETE FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
s.hash.0,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
Ok(None)
|
||||
}
|
||||
(Some(p_hash), o) => {
|
||||
if sqlx::query_scalar!(
|
||||
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
@@ -309,7 +272,10 @@ async fn create_script(
|
||||
let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
|
||||
|
||||
if ps.path != ns.path {
|
||||
require_owner_of_path(&authed, &ps.path)?;
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &ps.path, &db)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
let ph = {
|
||||
@@ -366,8 +332,8 @@ async fn create_script(
|
||||
//::text::json is to ensure we use serde_json with preserve order
|
||||
sqlx::query!(
|
||||
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
|
||||
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)",
|
||||
content, created_by, schema, is_template, extra_perms, lock, language, kind) \
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
|
||||
&w_id,
|
||||
&hash.0,
|
||||
ns.path,
|
||||
@@ -382,21 +348,11 @@ async fn create_script(
|
||||
lock,
|
||||
ns.language: ScriptLang,
|
||||
ns.kind.unwrap_or(ScriptKind::Script): ScriptKind,
|
||||
ns.tag,
|
||||
ns.draft_only
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
if let Some(p_path) = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()) {
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
|
||||
p_path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
|
||||
let schedulables = sqlx::query_as!(
|
||||
Schedule,
|
||||
"UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
|
||||
@@ -414,14 +370,6 @@ async fn create_script(
|
||||
tx = push_scheduled_job(tx, schedule).await?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sqlx::query!(
|
||||
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
|
||||
ns.path,
|
||||
&w_id
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
|
||||
@@ -480,7 +428,7 @@ async fn create_script(
|
||||
let (_, new_tx) = windmill_queue::push(
|
||||
tx,
|
||||
&w_id,
|
||||
JobPayload::Dependencies { hash, dependencies, language: ns.language },
|
||||
windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
|
||||
serde_json::Map::new(),
|
||||
&authed.username,
|
||||
&authed.email,
|
||||
@@ -493,7 +441,6 @@ async fn create_script(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
tx = new_tx;
|
||||
@@ -540,31 +487,6 @@ async fn get_script_by_path(
|
||||
Ok(Json(script))
|
||||
}
|
||||
|
||||
async fn get_script_by_path_w_draft(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<ScriptWDraft> {
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let script_o = sqlx::query_as::<_, ScriptWDraft>(
|
||||
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON
|
||||
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
|
||||
WHERE script.path = $1 AND script.workspace_id = $2 \
|
||||
AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
|
||||
workspace_id = $2)",
|
||||
)
|
||||
.bind(path)
|
||||
.bind(w_id)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
|
||||
let script = not_found_if_none(script_o, "Script", path)?;
|
||||
Ok(Json(script))
|
||||
}
|
||||
|
||||
async fn list_paths(
|
||||
authed: Authed,
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
@@ -600,22 +522,7 @@ async fn raw_script_by_path(
|
||||
Extension(user_db): Extension<UserDB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> Result<String> {
|
||||
let path = path.to_path();
|
||||
if !path.ends_with(".py")
|
||||
&& !path.ends_with(".ts")
|
||||
&& !path.ends_with(".go")
|
||||
&& !path.ends_with(".sh")
|
||||
{
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Path must ends with a .py, .ts, .go. or .sh extension: {}",
|
||||
path
|
||||
)));
|
||||
}
|
||||
let path = path
|
||||
.trim_end_matches(".py")
|
||||
.trim_end_matches(".ts")
|
||||
.trim_end_matches(".go")
|
||||
.trim_end_matches(".sh");
|
||||
let path = path.to_path().split(".").next().unwrap_or_default();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
let content_o = sqlx::query_scalar!(
|
||||
@@ -729,8 +636,6 @@ async fn archive_script_by_path(
|
||||
let path = path.to_path();
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
let hash: i64 = sqlx::query_scalar!(
|
||||
"UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash",
|
||||
path,
|
||||
@@ -844,19 +749,7 @@ async fn delete_script_by_path(
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
let path = path.to_path();
|
||||
|
||||
let draft_only = sqlx::query_scalar!(
|
||||
"SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(&db)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
if !draft_only {
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
}
|
||||
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
let script = sqlx::query_scalar!(
|
||||
"DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
|
||||
path,
|
||||
|
||||
@@ -359,8 +359,7 @@ pub struct Authed {
|
||||
pub username: String,
|
||||
pub is_admin: bool,
|
||||
pub groups: Vec<String>,
|
||||
// (folder name, can write, is owner)
|
||||
pub folders: Vec<(String, bool, bool)>,
|
||||
pub folders: Vec<(String, bool)>,
|
||||
}
|
||||
|
||||
pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
|
||||
@@ -370,7 +369,7 @@ pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &
|
||||
let splitted = path.split('/').collect::<Vec<_>>();
|
||||
if splitted.len() >= 2
|
||||
&& splitted[0] == "f"
|
||||
&& !authed.folders.iter().any(|(f, _, _)| f == splitted[1])
|
||||
&& !authed.folders.iter().any(|(f, _)| f == splitted[1])
|
||||
{
|
||||
let name = &authed.username;
|
||||
let groups = get_groups_for_user(w_id, name, db)
|
||||
@@ -510,7 +509,6 @@ pub struct UserInfo {
|
||||
pub disabled: bool,
|
||||
pub role: Option<String>,
|
||||
pub folders: Vec<String>,
|
||||
pub folders_owners: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize)]
|
||||
@@ -810,14 +808,9 @@ async fn whoami(
|
||||
disabled: false,
|
||||
role: Some("superadmin".to_string()),
|
||||
folders: folders
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.1 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
folders_owners: folders
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.2 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -903,14 +896,9 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
|
||||
disabled: usr.disabled,
|
||||
role: usr.role,
|
||||
folders: folders
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.1 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
folders_owners: folders
|
||||
.into_iter()
|
||||
.filter_map(|x| if x.2 { Some(x.0) } else { None })
|
||||
.collect(),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -925,45 +913,47 @@ pub async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<
|
||||
Ok(groups)
|
||||
}
|
||||
pub async fn is_owner_of_path(
|
||||
authed: Authed,
|
||||
Path((_w_id, path)): Path<(String, StripPath)>,
|
||||
Authed { username, is_admin, groups, .. }: Authed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path((w_id, path)): Path<(String, StripPath)>,
|
||||
) -> JsonResult<bool> {
|
||||
let path = path.to_path();
|
||||
if authed.is_admin {
|
||||
if is_admin {
|
||||
Ok(Json(true))
|
||||
} else {
|
||||
Ok(Json(require_owner_of_path(&authed, path).is_ok()))
|
||||
Ok(Json(
|
||||
require_owner_of_path(&w_id, &username, &groups, path, &db)
|
||||
.await
|
||||
.is_ok(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
|
||||
if authed.is_admin {
|
||||
return Ok(());
|
||||
}
|
||||
pub async fn require_owner_of_path(
|
||||
w_id: &str,
|
||||
username: &str,
|
||||
groups: &Vec<String>,
|
||||
path: &str,
|
||||
db: &DB,
|
||||
) -> Result<()> {
|
||||
if !path.is_empty() {
|
||||
let splitted = path.split("/").collect::<Vec<&str>>();
|
||||
if splitted[0] == "u" {
|
||||
if splitted[1] == authed.username {
|
||||
Ok(())
|
||||
if splitted[1] == username {
|
||||
return Ok(());
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
return Err(Error::BadRequest(format!(
|
||||
"only the owner {} is authorized to perform this operation",
|
||||
splitted[1]
|
||||
)))
|
||||
)));
|
||||
}
|
||||
} else if splitted[0] == "g" {
|
||||
return crate::groups::require_is_owner(splitted[1], username, groups, w_id, db).await;
|
||||
} else if splitted[0] == "f" {
|
||||
crate::folders::require_is_owner(authed, splitted[1])
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
"Not recognized path kind: {}",
|
||||
path
|
||||
)))
|
||||
return crate::folders::require_is_owner(splitted[1], username, groups, w_id, db).await;
|
||||
}
|
||||
} else {
|
||||
Err(Error::BadRequest(format!(
|
||||
"Cannot be owner of an empty path"
|
||||
)))
|
||||
}
|
||||
Err(Error::BadRequest(format!("not recognized owner kind")))
|
||||
}
|
||||
|
||||
async fn whois(
|
||||
@@ -1044,11 +1034,6 @@ async fn decline_invite(
|
||||
|
||||
lazy_static! {
|
||||
pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
|
||||
|
||||
pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
|
||||
.ok()
|
||||
.and_then(|x| x.parse::<u32>().ok())
|
||||
.unwrap_or(60 * 60 * 24 * 60); // 60 days
|
||||
}
|
||||
|
||||
async fn accept_invite(
|
||||
@@ -1918,24 +1903,16 @@ pub async fn delete_expired_items_perdiodically(
|
||||
Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
|
||||
}
|
||||
|
||||
if *JOB_RETENTION_SECS > 0 {
|
||||
let deleted_jobs = sqlx::query_scalar!(
|
||||
"DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
|
||||
*JOB_RETENTION_SECS as i64
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
|
||||
"DELETE FROM magic_link WHERE expiration <= now()
|
||||
RETURNING concat(substring(token for 10), '*****')",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match deleted_jobs {
|
||||
Ok(deleted_jobs) => {
|
||||
tracing::info!(
|
||||
"deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
|
||||
deleted_jobs.len(),
|
||||
deleted_jobs
|
||||
)
|
||||
}
|
||||
Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
|
||||
}
|
||||
match magic_links_deleted_r {
|
||||
Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
|
||||
Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
|
||||
}
|
||||
|
||||
tokio::select! {
|
||||
|
||||
@@ -12,6 +12,13 @@ use crate::{
|
||||
users::{maybe_refresh_folders, require_owner_of_path, Authed},
|
||||
webhook_util::{WebhookMessage, WebhookShared},
|
||||
};
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2022
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use axum::{
|
||||
extract::{Extension, Path, Query},
|
||||
@@ -337,20 +344,15 @@ async fn update_variable(
|
||||
if let Some(npath) = &ns.path {
|
||||
sqlb.set_str("path", npath);
|
||||
}
|
||||
let ns_value_is_none = ns.value.is_none();
|
||||
if let Some(nvalue) = ns.value {
|
||||
let is_secret = if ns.is_secret.is_some() {
|
||||
ns.is_secret.unwrap()
|
||||
} else {
|
||||
sqlx::query_scalar!(
|
||||
"SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
|
||||
&path,
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false)
|
||||
};
|
||||
let is_secret = sqlx::query_scalar!(
|
||||
"SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
|
||||
&path,
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
|
||||
let value = if is_secret && !already_encrypted.unwrap_or(false) {
|
||||
let mc = build_crypt(&mut tx, &w_id).await?;
|
||||
@@ -366,17 +368,9 @@ async fn update_variable(
|
||||
}
|
||||
|
||||
if let Some(nbool) = ns.is_secret {
|
||||
let old_secret = sqlx::query_scalar!(
|
||||
"SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
|
||||
&path,
|
||||
&w_id
|
||||
)
|
||||
.fetch_optional(&mut tx)
|
||||
.await?
|
||||
.unwrap_or(false);
|
||||
if old_secret != nbool && ns_value_is_none {
|
||||
if !nbool {
|
||||
return Err(Error::BadRequest(
|
||||
"cannot change is_secret without updating value too".to_string(),
|
||||
"A variable can not be updated to be non secret".to_owned(),
|
||||
));
|
||||
}
|
||||
sqlb.set_str("is_secret", nbool);
|
||||
@@ -386,8 +380,9 @@ async fn update_variable(
|
||||
if let Some(npath) = ns.path {
|
||||
if npath != path {
|
||||
check_path_conflict(&mut tx, &w_id, &npath).await?;
|
||||
require_owner_of_path(&authed, path)?;
|
||||
|
||||
if !authed.is_admin {
|
||||
require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
|
||||
}
|
||||
let mut v = sqlx::query_scalar!(
|
||||
"SELECT value FROM resource WHERE path = $1 AND workspace_id = $2",
|
||||
path,
|
||||
|
||||
@@ -21,16 +21,7 @@ use windmill_common::{
|
||||
};
|
||||
|
||||
pub fn global_service() -> Router {
|
||||
Router::new()
|
||||
.route("/list", get(list_worker_pings))
|
||||
.route("/custom_tags", get(get_custom_tags))
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
|
||||
|
||||
Router::new().route("/list", get(list_worker_pings))
|
||||
}
|
||||
|
||||
#[derive(FromRow, Serialize, Deserialize)]
|
||||
@@ -63,7 +54,3 @@ async fn list_worker_pings(
|
||||
tx.commit().await?;
|
||||
Ok(Json(rows))
|
||||
}
|
||||
|
||||
async fn get_custom_tags() -> Json<Vec<String>> {
|
||||
Json(CUSTOM_TAGS.clone())
|
||||
}
|
||||
@@ -1165,8 +1165,6 @@ where
|
||||
"edited_at",
|
||||
"edited_by",
|
||||
"archived",
|
||||
"has_draft",
|
||||
"draft_only",
|
||||
] {
|
||||
if obj.contains_key(key) {
|
||||
obj.remove(key);
|
||||
|
||||
@@ -13,7 +13,6 @@ use axum::{
|
||||
response::Json,
|
||||
};
|
||||
|
||||
use hyper::StatusCode;
|
||||
#[cfg(feature = "sqlx")]
|
||||
use sqlx::migrate::MigrateError;
|
||||
use thiserror::Error;
|
||||
@@ -61,8 +60,6 @@ pub enum Error {
|
||||
Anyhow(#[from] anyhow::Error),
|
||||
#[error("Error: {0:#?}")]
|
||||
JsonErr(serde_json::Value),
|
||||
#[error("Custom Status Code: {0:#?}")]
|
||||
CustomStatusCode(StatusCode, serde_json::Value),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
@@ -85,7 +82,6 @@ impl IntoResponse for Error {
|
||||
Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND,
|
||||
Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED,
|
||||
Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
|
||||
Self::CustomStatusCode(code, _) => code,
|
||||
Self::SqlErr(_) | Self::BadRequest(_) => axum::http::StatusCode::BAD_REQUEST,
|
||||
_ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
};
|
||||
|
||||
@@ -11,16 +11,10 @@
|
||||
//! For users writing scripts to access their infrastructure with firewalls requiring incoming
|
||||
//! connections to be from whitelisted IP addresses.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::Result;
|
||||
|
||||
pub async fn get_ip() -> Result<String> {
|
||||
reqwest::ClientBuilder::new()
|
||||
.timeout(Duration::from_secs(3))
|
||||
.build()?
|
||||
.get("https://hub.windmill.dev/getip")
|
||||
.send()
|
||||
reqwest::get("https://hub.windmill.dev/getip")
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.text()
|
||||
|
||||
@@ -33,8 +33,6 @@ pub struct Flow {
|
||||
pub archived: bool,
|
||||
pub schema: Option<Schema>,
|
||||
pub extra_perms: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -49,9 +47,6 @@ pub struct ListableFlow {
|
||||
pub archived: bool,
|
||||
pub extra_perms: serde_json::Value,
|
||||
pub starred: bool,
|
||||
pub has_draft: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -62,7 +57,6 @@ pub struct NewFlow {
|
||||
pub description: String,
|
||||
pub value: serde_json::Value,
|
||||
pub schema: Option<Schema>,
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
|
||||
@@ -258,8 +252,6 @@ pub enum FlowModuleValue {
|
||||
lock: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
path: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tag: Option<String>,
|
||||
language: ScriptLang,
|
||||
},
|
||||
Identity,
|
||||
|
||||
@@ -1,201 +1 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{Postgres, Transaction};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
error,
|
||||
flow_status::FlowStatus,
|
||||
flows::FlowValue,
|
||||
get_latest_deployed_hash_for_path,
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
};
|
||||
|
||||
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
|
||||
#[serde(rename_all(serialize = "lowercase"))]
|
||||
pub enum JobKind {
|
||||
Script,
|
||||
#[allow(non_camel_case_types)]
|
||||
Script_Hub,
|
||||
Preview,
|
||||
Dependencies,
|
||||
Flow,
|
||||
FlowPreview,
|
||||
Identity,
|
||||
FlowDependencies,
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
|
||||
pub struct QueuedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub scheduled_for: chrono::DateTime<chrono::Utc>,
|
||||
pub running: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logs: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_code: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_lock: Option<String>,
|
||||
pub canceled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_by: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub job_kind: JobKind,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<ScriptLang>,
|
||||
pub same_worker: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pre_run_error: Option<String>,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suspend: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub root_job: Option<Uuid>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub leaf_jobs: Option<serde_json::Value>,
|
||||
pub tag: String,
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn script_path(&self) -> &str {
|
||||
self.script_path
|
||||
.as_ref()
|
||||
.map(String::as_str)
|
||||
.unwrap_or("tmp/main")
|
||||
}
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
|
||||
self.raw_flow
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
|
||||
}
|
||||
|
||||
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
|
||||
self.flow_status
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for QueuedJob {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
workspace_id: "".to_string(),
|
||||
id: Uuid::default(),
|
||||
parent_job: None,
|
||||
created_by: "".to_string(),
|
||||
created_at: chrono::Utc::now(),
|
||||
started_at: None,
|
||||
scheduled_for: chrono::Utc::now(),
|
||||
running: false,
|
||||
script_hash: None,
|
||||
script_path: None,
|
||||
args: None,
|
||||
logs: None,
|
||||
raw_code: None,
|
||||
raw_lock: None,
|
||||
canceled: false,
|
||||
canceled_by: None,
|
||||
canceled_reason: None,
|
||||
last_ping: None,
|
||||
job_kind: JobKind::Identity,
|
||||
schedule_path: None,
|
||||
permissioned_as: "".to_string(),
|
||||
flow_status: None,
|
||||
raw_flow: None,
|
||||
is_flow_step: false,
|
||||
language: None,
|
||||
same_worker: false,
|
||||
pre_run_error: None,
|
||||
email: "".to_string(),
|
||||
visible_to_owner: false,
|
||||
suspend: None,
|
||||
mem_peak: None,
|
||||
root_job: None,
|
||||
leaf_jobs: None,
|
||||
tag: "deno".to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobPayload {
|
||||
ScriptHub { path: String },
|
||||
ScriptHash { hash: ScriptHash, path: String },
|
||||
Code(RawCode),
|
||||
Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
|
||||
FlowDependencies { path: String },
|
||||
Flow(String),
|
||||
RawFlow { value: FlowValue, path: Option<String> },
|
||||
Identity,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, Debug)]
|
||||
pub struct RawCode {
|
||||
pub content: String,
|
||||
pub path: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub lock: Option<String>,
|
||||
}
|
||||
|
||||
type Tag = String;
|
||||
|
||||
pub async fn script_path_to_payload<'c>(
|
||||
script_path: &str,
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &String,
|
||||
) -> error::Result<(JobPayload, Option<Tag>)> {
|
||||
let (job_payload, tag) = if script_path.starts_with("hub/") {
|
||||
(JobPayload::ScriptHub { path: script_path.to_owned() }, None)
|
||||
} else {
|
||||
let (script_hash, tag) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() },
|
||||
tag,
|
||||
)
|
||||
};
|
||||
Ok((job_payload, tag))
|
||||
}
|
||||
|
||||
pub async fn script_hash_to_tag<'c>(
|
||||
script_hash: &ScriptHash,
|
||||
db: &mut Transaction<'c, Postgres>,
|
||||
w_id: &String,
|
||||
) -> error::Result<Option<Tag>> {
|
||||
Ok(sqlx::query_scalar!(
|
||||
"select tag from script where hash = $1 AND workspace_id = $2",
|
||||
script_hash.0,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
.flatten())
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::{net::SocketAddr, sync::Arc};
|
||||
|
||||
use error::Error;
|
||||
|
||||
@@ -15,7 +15,6 @@ pub mod error;
|
||||
pub mod external_ip;
|
||||
pub mod flow_status;
|
||||
pub mod flows;
|
||||
pub mod jobs;
|
||||
pub mod more_serde;
|
||||
pub mod oauth2;
|
||||
pub mod schedule;
|
||||
@@ -43,14 +42,11 @@ lazy_static::lazy_static! {
|
||||
.flatten();
|
||||
pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
|
||||
pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
|
||||
pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
pub static ref IS_READY: Arc<std::sync::atomic::AtomicBool> = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
}
|
||||
|
||||
#[cfg(feature = "tokio")]
|
||||
pub async fn shutdown_signal(
|
||||
tx: tokio::sync::broadcast::Sender<()>,
|
||||
mut rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> anyhow::Result<()> {
|
||||
pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
|
||||
use std::io;
|
||||
use tokio::signal::unix::SignalKind;
|
||||
|
||||
@@ -64,9 +60,6 @@ pub async fn shutdown_signal(
|
||||
tokio::select! {
|
||||
_ = terminate() => {},
|
||||
_ = tokio::signal::ctrl_c() => {},
|
||||
_ = rx.recv() => {
|
||||
tracing::info!("shutdown monitor received killpill");
|
||||
},
|
||||
}
|
||||
println!("signal received, starting graceful shutdown");
|
||||
let _ = tx.send(());
|
||||
@@ -153,44 +146,23 @@ pub async fn connect(
|
||||
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
|
||||
}
|
||||
|
||||
type Tag = String;
|
||||
|
||||
pub async fn get_latest_deployed_hash_for_path<'c>(
|
||||
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
|
||||
w_id: &str,
|
||||
script_path: &str,
|
||||
) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
|
||||
let r_o = sqlx::query!(
|
||||
"select hash, tag from script where path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
|
||||
deleted = false AND archived = false AND lock IS not NULL AND lock_error_logs IS NULL)",
|
||||
script_path,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let script = utils::not_found_if_none(r_o, "script", script_path)?;
|
||||
|
||||
Ok((scripts::ScriptHash(script.hash), script.tag))
|
||||
}
|
||||
|
||||
// TODO: Move this elsewhere
|
||||
pub async fn get_latest_hash_for_path<'c>(
|
||||
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
|
||||
w_id: &str,
|
||||
script_path: &str,
|
||||
) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
|
||||
let r_o = sqlx::query!(
|
||||
"select hash, tag from script where path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
|
||||
deleted = false AND archived = false)",
|
||||
) -> error::Result<scripts::ScriptHash> {
|
||||
let script_hash_o = sqlx::query_scalar!(
|
||||
"select hash from script where path = $1 AND workspace_id = $2 AND
|
||||
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND
|
||||
deleted = false",
|
||||
script_path,
|
||||
w_id
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?;
|
||||
|
||||
let script = utils::not_found_if_none(r_o, "script", script_path)?;
|
||||
let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?;
|
||||
|
||||
Ok((scripts::ScriptHash(script.hash), script.tag))
|
||||
Ok(scripts::ScriptHash(script_hash))
|
||||
}
|
||||
|
||||
@@ -136,28 +136,27 @@ pub struct Script {
|
||||
pub lock_error_logs: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub kind: ScriptKind,
|
||||
pub tag: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
|
||||
pub struct ListableScript {
|
||||
pub workspace_id: String,
|
||||
pub hash: ScriptHash,
|
||||
pub path: String,
|
||||
pub parent_hashes: Option<ScriptHashes>,
|
||||
pub summary: String,
|
||||
pub description: String,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
pub archived: bool,
|
||||
pub deleted: bool,
|
||||
pub is_template: bool,
|
||||
pub extra_perms: serde_json::Value,
|
||||
pub lock_error_logs: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub kind: ScriptKind,
|
||||
pub starred: bool,
|
||||
pub tag: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub has_draft: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub draft_only: Option<bool>,
|
||||
pub has_deploy_errors: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
@@ -187,8 +186,6 @@ pub struct NewScript {
|
||||
pub lock: Option<Vec<String>>,
|
||||
pub language: ScriptLang,
|
||||
pub kind: Option<ScriptKind>,
|
||||
pub tag: Option<String>,
|
||||
pub draft_only: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
||||
@@ -34,4 +34,3 @@ cron.workspace = true
|
||||
rsmq_async.workspace = true
|
||||
tokio.workspace = true
|
||||
futures-core.workspace = true
|
||||
itertools.workspace = true
|
||||
|
||||
@@ -8,8 +8,9 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use itertools::Itertools;
|
||||
use anyhow::Context;
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::{Pool, Postgres, Transaction};
|
||||
use tracing::{instrument, Instrument};
|
||||
use ulid::Ulid;
|
||||
@@ -19,8 +20,8 @@ use windmill_common::{
|
||||
error::{self, Error},
|
||||
flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
|
||||
flows::{FlowModule, FlowModuleValue, FlowValue},
|
||||
jobs::{JobKind, JobPayload, QueuedJob, RawCode},
|
||||
scripts::{ScriptHash, ScriptLang},
|
||||
scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
|
||||
utils::StripPath,
|
||||
METRICS_ENABLED,
|
||||
};
|
||||
|
||||
@@ -49,21 +50,6 @@ lazy_static::lazy_static! {
|
||||
.unwrap();
|
||||
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
|
||||
|
||||
pub static ref ACCEPTED_TAGS: Vec<String> = std::env::var("WORKER_TAGS")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect())
|
||||
.unwrap_or_else(|| vec![
|
||||
"deno".to_string(),
|
||||
"python3".to_string(),
|
||||
"go".to_string(),
|
||||
"bash".to_string(),
|
||||
"dependency".to_string(),
|
||||
"flow".to_string(),
|
||||
"hub".to_string(),
|
||||
"other".to_string()]);
|
||||
|
||||
pub static ref ACCEPTED_TAGS_FILTER: String = format!(" AND ({})",
|
||||
ACCEPTED_TAGS.clone().into_iter().map(|x| format!("(tag = '{x}')")).join(" OR "));
|
||||
}
|
||||
|
||||
const MAX_FREE_EXECS: i32 = 1000;
|
||||
@@ -179,7 +165,6 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
None
|
||||
}
|
||||
} else {
|
||||
let accepted_tags_filter = &*ACCEPTED_TAGS_FILTER;
|
||||
/* Jobs can be started if they:
|
||||
* - haven't been started before,
|
||||
* running = false
|
||||
@@ -200,9 +185,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
|
||||
AND scheduled_for <= now())
|
||||
OR (suspend_until IS NOT NULL
|
||||
AND ( suspend <= 0
|
||||
OR suspend_until <= now())))
|
||||
{workspaces_filter}
|
||||
{accepted_tags_filter}
|
||||
OR suspend_until <= now()))) {workspaces_filter}
|
||||
ORDER BY scheduled_for
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
@@ -328,7 +311,6 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
mut same_worker: bool,
|
||||
pre_run_error: Option<&windmill_common::error::Error>,
|
||||
visible_to_owner: bool,
|
||||
mut tag: Option<String>,
|
||||
) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
|
||||
let args_json = serde_json::Value::Object(args);
|
||||
let job_id: Uuid = Ulid::new().into();
|
||||
@@ -455,14 +437,16 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
)
|
||||
}
|
||||
JobPayload::ScriptHub { path } => {
|
||||
let script = get_hub_script(&HTTP_CLIENT, path.clone(), email)
|
||||
.await
|
||||
.context("error fetching hub script")?;
|
||||
(
|
||||
None,
|
||||
Some(path),
|
||||
None,
|
||||
// Some((script.content, script.lockfile)),
|
||||
Some((script.content, script.lockfile)),
|
||||
JobKind::Script_Hub,
|
||||
None,
|
||||
None,
|
||||
Some(script.language),
|
||||
)
|
||||
}
|
||||
JobPayload::Code(RawCode { content, path, language, lock }) => (
|
||||
@@ -574,34 +558,12 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
|
||||
let flow_status = raw_flow.as_ref().map(FlowStatus::new);
|
||||
|
||||
let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
|
||||
"dependency".to_string()
|
||||
} else if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
|
||||
"flow".to_string()
|
||||
} else if job_kind == JobKind::Identity {
|
||||
// identity is a light script, deno is too
|
||||
"deno".to_string()
|
||||
} else if job_kind == JobKind::Script_Hub {
|
||||
"hub".to_string()
|
||||
} else {
|
||||
if tag == Some("".to_string()) {
|
||||
tag = None;
|
||||
}
|
||||
tag.unwrap_or_else(|| {
|
||||
language
|
||||
.as_ref()
|
||||
.map(|x| x.as_str())
|
||||
.unwrap_or_else(|| "deno")
|
||||
.to_string()
|
||||
})
|
||||
};
|
||||
|
||||
let uuid = sqlx::query_scalar!(
|
||||
"INSERT INTO queue
|
||||
(workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for,
|
||||
script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
|
||||
flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24) \
|
||||
flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
|
||||
RETURNING id",
|
||||
workspace_id,
|
||||
job_id,
|
||||
@@ -625,8 +587,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
pre_run_error.map(|e| e.to_string()),
|
||||
email,
|
||||
visible_to_owner,
|
||||
root_job,
|
||||
tag
|
||||
root_job
|
||||
)
|
||||
.fetch_one(&mut tx)
|
||||
.await
|
||||
@@ -684,3 +645,127 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> serde_json::Value {
|
||||
let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown");
|
||||
serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler})
|
||||
}
|
||||
|
||||
pub async fn get_hub_script(
|
||||
client: &reqwest::Client,
|
||||
path: String,
|
||||
email: &str,
|
||||
) -> error::Result<HubScript> {
|
||||
get_full_hub_script_by_path(email, StripPath(path), client)
|
||||
.await
|
||||
.map(|e| e)
|
||||
}
|
||||
|
||||
#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
|
||||
pub struct QueuedJob {
|
||||
pub workspace_id: String,
|
||||
pub id: Uuid,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parent_job: Option<Uuid>,
|
||||
pub created_by: String,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub scheduled_for: chrono::DateTime<chrono::Utc>,
|
||||
pub running: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_hash: Option<ScriptHash>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub script_path: Option<String>,
|
||||
pub args: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logs: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_code: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_lock: Option<String>,
|
||||
pub canceled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_by: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub canceled_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub job_kind: JobKind,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub schedule_path: Option<String>,
|
||||
pub permissioned_as: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub flow_status: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub raw_flow: Option<serde_json::Value>,
|
||||
pub is_flow_step: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub language: Option<ScriptLang>,
|
||||
pub same_worker: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pre_run_error: Option<String>,
|
||||
pub email: String,
|
||||
pub visible_to_owner: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suspend: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub mem_peak: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub root_job: Option<Uuid>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub leaf_jobs: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn script_path(&self) -> &str {
|
||||
self.script_path
|
||||
.as_ref()
|
||||
.map(String::as_str)
|
||||
.unwrap_or("tmp/main")
|
||||
}
|
||||
}
|
||||
|
||||
impl QueuedJob {
|
||||
pub fn parse_raw_flow(&self) -> Option<FlowValue> {
|
||||
self.raw_flow
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
|
||||
}
|
||||
|
||||
pub fn parse_flow_status(&self) -> Option<FlowStatus> {
|
||||
self.flow_status
|
||||
.as_ref()
|
||||
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
|
||||
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
|
||||
#[serde(rename_all(serialize = "lowercase"))]
|
||||
pub enum JobKind {
|
||||
Script,
|
||||
#[allow(non_camel_case_types)]
|
||||
Script_Hub,
|
||||
Preview,
|
||||
Dependencies,
|
||||
Flow,
|
||||
FlowPreview,
|
||||
Identity,
|
||||
FlowDependencies,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum JobPayload {
|
||||
ScriptHub { path: String },
|
||||
ScriptHash { hash: ScriptHash, path: String },
|
||||
Code(RawCode),
|
||||
Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
|
||||
FlowDependencies { path: String },
|
||||
Flow(String),
|
||||
RawFlow { value: FlowValue, path: Option<String> },
|
||||
Identity,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Deserialize, Debug)]
|
||||
pub struct RawCode {
|
||||
pub content: String,
|
||||
pub path: Option<String>,
|
||||
pub language: ScriptLang,
|
||||
pub lock: Option<String>,
|
||||
}
|
||||
|
||||
@@ -6,17 +6,16 @@
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
use crate::push;
|
||||
use crate::QueueTransaction;
|
||||
use crate::{push, JobPayload};
|
||||
use sqlx::{query_scalar, Postgres, Transaction};
|
||||
use std::str::FromStr;
|
||||
use windmill_common::jobs::JobPayload;
|
||||
use windmill_common::{
|
||||
error::{self, Result},
|
||||
schedule::Schedule,
|
||||
users::username_to_permissioned_as,
|
||||
utils::{now_from_db, StripPath},
|
||||
};
|
||||
use crate::{QueueTransaction};
|
||||
|
||||
pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
mut tx: QueueTransaction<'c, R>,
|
||||
@@ -67,19 +66,18 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
}
|
||||
}
|
||||
|
||||
let (payload, tag) = if schedule.is_flow {
|
||||
(JobPayload::Flow(schedule.script_path), None)
|
||||
let payload = if schedule.is_flow {
|
||||
JobPayload::Flow(schedule.script_path)
|
||||
} else {
|
||||
let (hash, tag) = windmill_common::get_latest_hash_for_path(
|
||||
tx.transaction_mut(),
|
||||
&schedule.workspace_id,
|
||||
&schedule.script_path,
|
||||
)
|
||||
.await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash, path: schedule.script_path },
|
||||
tag,
|
||||
)
|
||||
JobPayload::ScriptHash {
|
||||
hash: windmill_common::get_latest_hash_for_path(
|
||||
tx.transaction_mut(),
|
||||
&schedule.workspace_id,
|
||||
&schedule.script_path,
|
||||
)
|
||||
.await?,
|
||||
path: schedule.script_path,
|
||||
}
|
||||
};
|
||||
|
||||
sqlx::query!(
|
||||
@@ -106,7 +104,6 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
|
||||
false,
|
||||
None,
|
||||
true,
|
||||
tag,
|
||||
)
|
||||
.await?;
|
||||
Ok(tx) // TODO: Bubble up pushed UUID from here
|
||||
|
||||
@@ -30,7 +30,7 @@ class WindmillFinder(MetaPathFinder):
|
||||
script_path = "/".join(splitted)
|
||||
import requests
|
||||
|
||||
url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py"
|
||||
url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}"
|
||||
|
||||
r = requests.get(
|
||||
url, headers={"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}
|
||||
|
||||
@@ -3,7 +3,7 @@ name: "python download pip"
|
||||
mode: ONCE
|
||||
hostname: "python"
|
||||
log_level: ERROR
|
||||
time_limit: 900
|
||||
time_limit: 300
|
||||
|
||||
rlimit_as: 2048
|
||||
rlimit_cpu: 1000
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{fs::File, io::AsyncReadExt};
|
||||
use windmill_common::error::{self, Error};
|
||||
use windmill_queue::CLOUD_HOSTED;
|
||||
|
||||
use crate::MAX_RESULT_SIZE;
|
||||
|
||||
pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
|
||||
let mut file = File::open(format!("{job_dir}/result.json")).await?;
|
||||
let mut content = "".to_string();
|
||||
file.read_to_string(&mut content).await?;
|
||||
if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
|
||||
return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB).
|
||||
If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
|
||||
}
|
||||
serde_json::from_str(&content)
|
||||
.map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn set_logs(logs: &str, id: &uuid::Uuid, db: &Pool<Postgres>) {
|
||||
if sqlx::query!(
|
||||
"UPDATE queue SET logs = $1 WHERE id = $2",
|
||||
logs.to_owned(),
|
||||
id
|
||||
)
|
||||
.execute(db)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::error!(%id, "error updating logs for id {id}")
|
||||
};
|
||||
}
|
||||
|
||||
pub fn capitalize(s: &str) -> String {
|
||||
let mut c = s.chars();
|
||||
match c.next() {
|
||||
None => String::new(),
|
||||
Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
|
||||
}
|
||||
}
|
||||
@@ -1,521 +0,0 @@
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::{ROOT_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_CACHE_RATE, TAR_PIP_TMP_CACHE_DIR, TMP_DIR};
|
||||
#[cfg(feature = "enterprise")]
|
||||
use itertools::Itertools;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use rand::Rng;
|
||||
#[cfg(feature = "enterprise")]
|
||||
use std::process::Stdio;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use tokio::{process::Command, sync::mpsc::Sender, time::Instant};
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use windmill_common::error;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
const TAR_CACHE_FILENAME: &str = "denogocache.tar";
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<()> {
|
||||
tracing::info!("Started building and pushing piptar {folder}");
|
||||
let start = Instant::now();
|
||||
let folder_name = folder.split("/").last().unwrap();
|
||||
let tar_path = format!("{TAR_PIP_TMP_CACHE_DIR}/{folder_name}.tar",);
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"tar",
|
||||
vec!["-c", "-f", &tar_path, "-C", &folder, "."],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to tar cache. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
let tar_metadata = tokio::fs::metadata(&tar_path).await;
|
||||
if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
|
||||
tracing::info!("Failed to tar cache: {folder}");
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to tar cache: {folder}"
|
||||
)));
|
||||
}
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&tar_path,
|
||||
&format!(":s3,env_auth=true:{bucket}/tar/pip/{folder_name}.tar"),
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--s3-no-check-bucket",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy piptar {folder} to bucket. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying piptar {folder} to bucket {bucket} as tar, took: {:?}s. Size of tar: {}",
|
||||
start.elapsed().as_secs(),
|
||||
tar_metadata.unwrap().len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> {
|
||||
use tokio::fs::metadata;
|
||||
let folder_name = folder.split("/").last().unwrap();
|
||||
|
||||
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
|
||||
|
||||
let start = Instant::now();
|
||||
let tar_path = format!("tar/pip/{folder_name}.tar");
|
||||
let target = format!("{ROOT_TMP_CACHE_DIR}/{tar_path}.single");
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&format!(":s3,env_auth=true:{bucket}/{tar_path}"),
|
||||
&target,
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!(
|
||||
"Failed to copy tar {folder_name} from bucket. Error: {:?}",
|
||||
e
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if metadata(&target).await.is_err() {
|
||||
tracing::info!(
|
||||
"piptar {folder_name} not found in bucket. Took {:?}ms",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"tar {folder_name} does not exist in bucket"
|
||||
)));
|
||||
}
|
||||
|
||||
extract_pip_tar(&target, &folder).await?;
|
||||
tokio::fs::remove_file(&target).await?;
|
||||
tracing::info!(
|
||||
"Finished pulling and extracting {folder_name}. Took {:?}ms",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn cache_global(bucket: &str, tx: Sender<()>) -> error::Result<()> {
|
||||
copy_cache_from_bucket(bucket, tx).await?;
|
||||
copy_cache_to_bucket(bucket).await?;
|
||||
|
||||
// this is to prevent excessive tar upload. 1/100*15min = each worker sync its tar once per day on average
|
||||
if rand::thread_rng().gen_range(0..*TAR_CACHE_RATE) == 0 {
|
||||
copy_cache_to_bucket_as_tar(bucket).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_from_bucket(bucket: &str, tx: Sender<()>) -> error::Result<()> {
|
||||
tracing::info!("Copying cache from bucket in the background {bucket}");
|
||||
let bucket = bucket.to_string();
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copy",
|
||||
&format!(":s3,env_auth=true:{bucket}"),
|
||||
&ROOT_TMP_CACHE_DIR,
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"+ tar/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy cache from bucket. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying cache from bucket {bucket}, took {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
|
||||
tx.send(()).await.expect("can send copy cache signal");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
|
||||
tracing::info!("Copying cache to bucket {bucket}");
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copy",
|
||||
&ROOT_TMP_CACHE_DIR,
|
||||
&format!(":s3,env_auth=true:{bucket}"),
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy cache to bucket. Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
tracing::info!(
|
||||
"Finished copying cache to bucket {bucket}, took: {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
|
||||
tracing::info!("Copying cache to bucket {bucket} as tar");
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"tar",
|
||||
vec![
|
||||
"-c",
|
||||
"-f",
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
"go",
|
||||
"deno",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to tar cache. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
let tar_metadata =
|
||||
tokio::fs::metadata(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await;
|
||||
if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
|
||||
tracing::info!("Failed to tar cache");
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
&format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
"--s3-no-check-bucket",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to copy tar to bucket. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
|
||||
{
|
||||
tracing::info!("Failed to remove tar cache. Error: {:?}", e);
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying cache to bucket {bucket} as tar, took: {:?}s. Size of new tar: {}",
|
||||
start.elapsed().as_secs(),
|
||||
tar_metadata.unwrap().len()
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
|
||||
use tokio::fs::metadata;
|
||||
|
||||
tracing::info!("Copying denogo cache from bucket {bucket} as tar");
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copyto",
|
||||
&format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed copying denogo tar from cache. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_CACHE_DIR,
|
||||
"tar",
|
||||
vec![
|
||||
"-xpvf",
|
||||
&format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed to untar denogo. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
let denogen = format!("{ROOT_TMP_CACHE_DIR}deno/gen/file");
|
||||
if metadata(&denogen).await.is_ok() {
|
||||
let _ = tokio::fs::remove_dir_all(denogen).await;
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
|
||||
{
|
||||
tracing::info!("Failed to remove denotar cache. Error: {:?}", e);
|
||||
return;
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying denogotar from bucket {bucket} as tar, took: {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_all_piptars_from_bucket(bucket: &str) {
|
||||
tracing::info!("Copying all piptars cache from bucket {bucket}");
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
if let Err(e) = execute_command(
|
||||
ROOT_CACHE_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"copy",
|
||||
&format!(":s3,env_auth=true:{bucket}/tar/pip/"),
|
||||
&TAR_PIP_TMP_CACHE_DIR,
|
||||
"-v",
|
||||
"--size-only",
|
||||
"--fast-list",
|
||||
],
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::info!("Failed transferring all piptars from cache. Error: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished transferring piptars from bucket {bucket} as tar, took: {:?}s",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
}
|
||||
|
||||
// async fn check_if_bucket_syncable(bucket: &str) -> bool {
|
||||
// match Command::new("rclone")
|
||||
// .arg("lsf")
|
||||
// .arg(format!(":s3,env_auth=true:{bucket}/NOSYNC"))
|
||||
|
||||
// .arg("-vv")
|
||||
// .arg("--fast-list")
|
||||
// .stdin(Stdio::null())
|
||||
// .stdout(Stdio::null())
|
||||
// .output()
|
||||
// .await;
|
||||
// return true;
|
||||
// }
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_tmp_cache_to_cache() -> error::Result<()> {
|
||||
let start: Instant = Instant::now();
|
||||
execute_command(
|
||||
TMP_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"sync",
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
ROOT_CACHE_DIR,
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
tracing::info!(
|
||||
"Finished copying local tmp cache to local cache. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
|
||||
if let Err(e) = untar_all_piptars().await {
|
||||
tracing::info!("Failed to untar piptars. Error: {:?}", e);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn untar_all_piptars() -> error::Result<()> {
|
||||
use tokio::fs::{self, metadata};
|
||||
|
||||
use crate::PIP_CACHE_DIR;
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
|
||||
let mut entries = fs::read_dir(TAR_PIP_TMP_CACHE_DIR).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
if let Err(e) = {
|
||||
let entry_path = entry.path();
|
||||
let path = entry_path.to_str().expect("Could not convert path to str");
|
||||
let folder = format!(
|
||||
"{PIP_CACHE_DIR}/{}",
|
||||
path.split('/')
|
||||
.last()
|
||||
.unwrap()
|
||||
.strip_suffix(".tar")
|
||||
.unwrap()
|
||||
);
|
||||
if metadata(&folder).await.is_ok() {
|
||||
continue;
|
||||
}
|
||||
extract_pip_tar(&path, &folder).await?;
|
||||
Ok(()) as error::Result<()>
|
||||
} {
|
||||
tracing::info!("Failed to extract pip tar. Error: {:?}", e);
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
"Finished untarring all piptars. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn extract_pip_tar(tar: &str, folder: &str) -> error::Result<()> {
|
||||
use tokio::fs;
|
||||
|
||||
let start: Instant = Instant::now();
|
||||
fs::create_dir_all(&folder).await?;
|
||||
if let Err(e) = execute_command(&folder, "tar", vec!["-xpvf", tar]).await {
|
||||
tracing::info!("Failed to untar piptar. Error: {:?}", e);
|
||||
fs::remove_dir_all(&folder).await?;
|
||||
return Err(e);
|
||||
}
|
||||
tracing::info!(
|
||||
"Finished extracting pip tar {folder}. Took {}ms",
|
||||
start.elapsed().as_millis(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {
|
||||
let start: Instant = Instant::now();
|
||||
execute_command(
|
||||
TMP_DIR,
|
||||
"rclone",
|
||||
vec![
|
||||
"sync",
|
||||
ROOT_CACHE_DIR,
|
||||
ROOT_TMP_CACHE_DIR,
|
||||
"--filter",
|
||||
"- deno/gen/file/**",
|
||||
"--filter",
|
||||
"+ deno/**",
|
||||
"--filter",
|
||||
"+ go/**",
|
||||
"--filter",
|
||||
"- *",
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
tracing::info!(
|
||||
"Finished copying local cache to local tmp cache. Took {}ms",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
pub async fn execute_command(dir: &str, command: &str, args: Vec<&str>) -> error::Result<()> {
|
||||
match Command::new(command)
|
||||
.current_dir(dir)
|
||||
.args(args.clone())
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.spawn()
|
||||
{
|
||||
Ok(mut h) => {
|
||||
if !h.wait().await.unwrap().success() {
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to apply {command} with args: {}",
|
||||
args.iter().join(" ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(error::Error::ExecutionErr(format!(
|
||||
"Failed to apply {command} with args: {}. Error: {e:?}",
|
||||
args.iter().join(" ")
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,373 +0,0 @@
|
||||
use std::process::Stdio;
|
||||
|
||||
use itertools::Itertools;
|
||||
use tokio::{
|
||||
fs::{DirBuilder, File},
|
||||
io::AsyncReadExt,
|
||||
process::Command,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
jobs::QueuedJob,
|
||||
utils::calculate_hash,
|
||||
};
|
||||
use windmill_parser_go::parse_go_imports;
|
||||
|
||||
use crate::{
|
||||
common::{capitalize, read_result, set_logs},
|
||||
create_args_and_out_file, get_reserved_variables, handle_child, write_file,
|
||||
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GO_CACHE_DIR, HOME_ENV,
|
||||
NETRC, NSJAIL_PATH, PATH_ENV,
|
||||
};
|
||||
|
||||
const GO_REQ_SPLITTER: &str = "//go.sum\n";
|
||||
const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref GO_PATH: String = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_go_job(
|
||||
logs: &mut String,
|
||||
job: &QueuedJob,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
inner_content: &str,
|
||||
job_dir: &str,
|
||||
requirements_o: Option<String>,
|
||||
shared_mount: &str,
|
||||
base_internal_url: &str,
|
||||
worker_name: &str,
|
||||
) -> Result<serde_json::Value, Error> {
|
||||
//go does not like executing modules at temp root
|
||||
let job_dir = &format!("{job_dir}/go");
|
||||
let (skip_go_mod, skip_tidy) = if let Some(requirements) = requirements_o {
|
||||
gen_go_mod(inner_content, job_dir, &requirements).await?
|
||||
} else {
|
||||
(false, false)
|
||||
};
|
||||
|
||||
logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
|
||||
install_go_dependencies(
|
||||
&job.id,
|
||||
inner_content,
|
||||
logs,
|
||||
job_dir,
|
||||
db,
|
||||
true,
|
||||
skip_go_mod,
|
||||
skip_tidy,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
|
||||
set_logs(logs, &job.id, db).await;
|
||||
let client = &client.get_authed().await;
|
||||
create_args_and_out_file(client, job, job_dir).await?;
|
||||
{
|
||||
let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
|
||||
drop(inner_content);
|
||||
|
||||
const WRAPPER_CONTENT: &str = r#"package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"fmt"
|
||||
"mymod/inner"
|
||||
)
|
||||
|
||||
func main() {{
|
||||
|
||||
dat, err := os.ReadFile("args.json")
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
|
||||
var req inner.Req
|
||||
|
||||
if err := json.Unmarshal(dat, &req); err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
|
||||
res, err := inner.Run(req)
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
res_json, err := json.Marshal(res)
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
_, err = f.WriteString(string(res_json))
|
||||
if err != nil {{
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}}
|
||||
}}"#;
|
||||
|
||||
write_file(job_dir, "main.go", WRAPPER_CONTENT).await?;
|
||||
|
||||
{
|
||||
let spread = &sig
|
||||
.args
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|x| format!("req.{}", capitalize(&x.name)))
|
||||
.join(", ");
|
||||
let req_body = &sig
|
||||
.args
|
||||
.into_iter()
|
||||
.map(|x| {
|
||||
format!(
|
||||
"{} {} `json:\"{}\"`",
|
||||
capitalize(&x.name),
|
||||
windmill_parser_go::otyp_to_string(x.otyp),
|
||||
x.name
|
||||
)
|
||||
})
|
||||
.join("\n");
|
||||
let runner_content: String = format!(
|
||||
r#"package inner
|
||||
type Req struct {{
|
||||
{req_body}
|
||||
}}
|
||||
|
||||
func Run(req Req) (interface{{}}, error){{
|
||||
return main({spread})
|
||||
}}
|
||||
|
||||
"#,
|
||||
);
|
||||
write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?;
|
||||
}
|
||||
}
|
||||
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
|
||||
reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
|
||||
|
||||
let child = if !*DISABLE_NSJAIL {
|
||||
let _ = write_file(
|
||||
job_dir,
|
||||
"run.config.proto",
|
||||
&NSJAIL_CONFIG_RUN_GO_CONTENT
|
||||
.replace("{JOB_DIR}", job_dir)
|
||||
.replace("{CACHE_DIR}", GO_CACHE_DIR)
|
||||
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
|
||||
.replace("{SHARED_MOUNT}", shared_mount),
|
||||
)
|
||||
.await?;
|
||||
let build_go = Command::new(GO_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.env("GOPATH", GO_CACHE_DIR)
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
.args(vec!["build", "main.go"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
build_go,
|
||||
false,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"go build",
|
||||
)
|
||||
.await?;
|
||||
|
||||
Command::new(NSJAIL_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(reserved_variables)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
} else {
|
||||
if let Some(ref netrc) = *NETRC {
|
||||
write_file(&HOME_ENV, ".netrc", netrc).await?;
|
||||
}
|
||||
Command::new(GO_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(reserved_variables)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.env("GOPATH", GO_CACHE_DIR)
|
||||
.env("GOPRIVATE", GOPRIVATE.as_ref().unwrap_or(&String::new()))
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
.args(vec!["run", "main.go"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
};
|
||||
handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"go run",
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
}
|
||||
|
||||
async fn gen_go_mod(
|
||||
inner_content: &str,
|
||||
job_dir: &str,
|
||||
requirements: &str,
|
||||
) -> error::Result<(bool, bool)> {
|
||||
gen_go_mymod(inner_content, job_dir).await?;
|
||||
|
||||
let md = requirements.split_once(GO_REQ_SPLITTER);
|
||||
if let Some((req, sum)) = md {
|
||||
write_file(job_dir, "go.mod", &req).await?;
|
||||
write_file(job_dir, "go.sum", &sum).await?;
|
||||
Ok((true, true))
|
||||
} else {
|
||||
write_file(job_dir, "go.mod", &requirements).await?;
|
||||
Ok((true, false))
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn install_go_dependencies(
|
||||
job_id: &Uuid,
|
||||
code: &str,
|
||||
logs: &mut String,
|
||||
job_dir: &str,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
non_dep_job: bool,
|
||||
skip_go_mod: bool,
|
||||
has_sum: bool,
|
||||
worker_name: &str,
|
||||
w_id: &str,
|
||||
) -> error::Result<String> {
|
||||
if !skip_go_mod {
|
||||
gen_go_mymod(code, job_dir).await?;
|
||||
let child = Command::new("go")
|
||||
.current_dir(job_dir)
|
||||
.args(vec!["mod", "init", "mymod"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
|
||||
handle_child(job_id, db, logs, child, false, worker_name, w_id, "go init").await?;
|
||||
}
|
||||
|
||||
let mut new_lockfile = false;
|
||||
|
||||
let hash = if !has_sum {
|
||||
calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
|
||||
} else {
|
||||
"".to_string()
|
||||
};
|
||||
|
||||
let mut skip_tidy = has_sum;
|
||||
|
||||
if !has_sum {
|
||||
if let Some(cached) = sqlx::query_scalar!(
|
||||
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
|
||||
hash
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
{
|
||||
logs.push_str(&format!("\nfound cached resolution"));
|
||||
gen_go_mod(code, job_dir, &cached).await?;
|
||||
skip_tidy = true;
|
||||
new_lockfile = false;
|
||||
} else {
|
||||
new_lockfile = true;
|
||||
}
|
||||
}
|
||||
|
||||
let mod_command = if skip_tidy { "download" } else { "tidy" };
|
||||
let child = Command::new(GO_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env("GOPATH", GO_CACHE_DIR)
|
||||
.args(vec!["mod", mod_command])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
&format!("go {mod_command}"),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
|
||||
if (!new_lockfile || has_sum) && non_dep_job {
|
||||
return Ok("".to_string());
|
||||
}
|
||||
|
||||
let mut req_content = "".to_string();
|
||||
|
||||
let mut file = File::open(format!("{job_dir}/go.mod")).await?;
|
||||
file.read_to_string(&mut req_content).await?;
|
||||
req_content.push_str(GO_REQ_SPLITTER);
|
||||
let sum_path = format!("{job_dir}/go.sum");
|
||||
if tokio::fs::metadata(&sum_path).await.is_ok() {
|
||||
let mut file = File::open(sum_path).await?;
|
||||
file.read_to_string(&mut req_content).await?;
|
||||
}
|
||||
|
||||
if non_dep_job {
|
||||
sqlx::query!(
|
||||
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
|
||||
hash,
|
||||
req_content
|
||||
).fetch_optional(db).await?;
|
||||
|
||||
return Ok(String::new());
|
||||
} else {
|
||||
Ok(req_content)
|
||||
}
|
||||
}
|
||||
|
||||
async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> {
|
||||
let code = if code.trim_start().starts_with("package") {
|
||||
code.to_string()
|
||||
} else {
|
||||
format!("package inner; {code}")
|
||||
};
|
||||
|
||||
let mymod_dir = format!("{job_dir}/inner");
|
||||
DirBuilder::new()
|
||||
.recursive(true)
|
||||
.create(&mymod_dir)
|
||||
.await
|
||||
.expect("could not create go's mymod dir");
|
||||
|
||||
write_file(&mymod_dir, "inner_main.go", &code).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -10,13 +10,11 @@ use sqlx::{Pool, Postgres};
|
||||
use tracing::instrument;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error::Error,
|
||||
flow_status::FlowStatusModule,
|
||||
jobs::{JobKind, QueuedJob},
|
||||
schedule::Schedule,
|
||||
METRICS_ENABLED,
|
||||
error::Error, flow_status::FlowStatusModule, schedule::Schedule, METRICS_ENABLED,
|
||||
};
|
||||
use windmill_queue::{
|
||||
delete_job, schedule::get_schedule_opt, JobKind, QueueTransaction, QueuedJob, CLOUD_HOSTED,
|
||||
};
|
||||
use windmill_queue::{delete_job, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED};
|
||||
|
||||
#[instrument(level = "trace", skip_all)]
|
||||
pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
@@ -126,10 +124,9 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
, email
|
||||
, visible_to_owner
|
||||
, mem_peak
|
||||
, tag
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
|
||||
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29, $30)
|
||||
$10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
|
||||
ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
|
||||
queued_job.workspace_id,
|
||||
queued_job.id,
|
||||
@@ -159,8 +156,7 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
|
||||
duration: Option<i64>,
|
||||
queued_job.email,
|
||||
queued_job.visible_to_owner,
|
||||
mem_peak,
|
||||
queued_job.tag,
|
||||
mem_peak
|
||||
)
|
||||
.execute(&mut tx)
|
||||
.await
|
||||
|
||||
@@ -143,6 +143,7 @@ fn add_closing_bracket(s: &str) -> String {
|
||||
s
|
||||
}
|
||||
|
||||
const SPLIT_PAT: &str = ";";
|
||||
async fn eval(
|
||||
context: &mut JsRuntime,
|
||||
expr: &str,
|
||||
@@ -150,6 +151,21 @@ async fn eval(
|
||||
by_id: Option<IdContext>,
|
||||
has_client: bool,
|
||||
) -> anyhow::Result<serde_json::Value> {
|
||||
let exprs = expr
|
||||
.trim()
|
||||
.split(SPLIT_PAT)
|
||||
.map(|x| x.trim())
|
||||
.filter(|x| !x.is_empty())
|
||||
.collect::<Vec<&str>>();
|
||||
let expr = if exprs.is_empty() {
|
||||
"return undefined;".to_string()
|
||||
} else {
|
||||
format!(
|
||||
"{};\n return {};",
|
||||
exprs.iter().take(exprs.len() - 1).join(";\n"),
|
||||
exprs.last().unwrap()
|
||||
)
|
||||
};
|
||||
let (api_code, by_id_code) = if has_client {
|
||||
let by_id_code = if let Some(by_id) = by_id {
|
||||
format!(
|
||||
@@ -216,18 +232,13 @@ async function resource(path) {{
|
||||
(String::new(), String::new())
|
||||
};
|
||||
|
||||
let f = if expr.contains("return ") {
|
||||
expr.to_string()
|
||||
} else {
|
||||
format!("return {expr}")
|
||||
};
|
||||
let code = format!(
|
||||
r#"
|
||||
{api_code}
|
||||
{}
|
||||
{by_id_code}
|
||||
(async () => {{
|
||||
{f};
|
||||
{expr}
|
||||
}})()
|
||||
"#,
|
||||
env.into_iter()
|
||||
@@ -240,7 +251,8 @@ async function resource(path) {{
|
||||
})
|
||||
.join(""),
|
||||
);
|
||||
let global = context.execute_script("<anon>", code.into())?;
|
||||
tracing::debug!("{}", code);
|
||||
let global = context.execute_script("<anon>", code)?;
|
||||
let global = context.resolve_value(global).await?;
|
||||
|
||||
let scope = &mut context.handle_scope();
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
mod common;
|
||||
mod global_cache;
|
||||
mod go_executor;
|
||||
mod jobs;
|
||||
mod js_eval;
|
||||
mod python_executor;
|
||||
mod worker;
|
||||
mod worker_flow;
|
||||
|
||||
|
||||
92
backend/windmill-worker/src/main2.rs
Normal file
92
backend/windmill-worker/src/main2.rs
Normal file
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Author: Ruben Fiszel
|
||||
* Copyright: Windmill Labs, Inc 2022
|
||||
* This file and its contents are licensed under the AGPLv3 License.
|
||||
* Please see the included NOTICE for copyright information and
|
||||
* LICENSE-AGPL for a copy of the license.
|
||||
*/
|
||||
|
||||
// use std::{net::SocketAddr, time::Duration};
|
||||
|
||||
// use anyhow::Context;
|
||||
// use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
|
||||
// use windmill_common::{
|
||||
// error::{self, Error},
|
||||
// utils::rd_string,
|
||||
// };
|
||||
|
||||
// #[tokio::main]
|
||||
// async fn main() -> anyhow::Result<()> {
|
||||
// // dotenv().ok();
|
||||
|
||||
// windmill_common::tracing_init::initialize_tracing();
|
||||
|
||||
// let db = async {
|
||||
// let database_url = std::env::var("DATABASE_URL")
|
||||
// .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
|
||||
|
||||
// let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
|
||||
// Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
|
||||
// Err(_) => 10,
|
||||
// };
|
||||
|
||||
// Ok::<Pool<Postgres>, error::Error>(
|
||||
// PgPoolOptions::new()
|
||||
// .max_connections(max_connections)
|
||||
// .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
|
||||
// .connect(&database_url)
|
||||
// .await
|
||||
// .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
|
||||
// )
|
||||
// }
|
||||
// .await?;
|
||||
|
||||
// let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
|
||||
// .ok()
|
||||
// .map(|s| {
|
||||
// s.parse::<bool>()
|
||||
// .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
|
||||
// .or_else(|_| s.parse::<SocketAddr>().map(Some))
|
||||
// })
|
||||
// .transpose()?
|
||||
// .flatten();
|
||||
|
||||
// let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
|
||||
// let shutdown_signal = windmill_common::shutdown_signal(tx);
|
||||
|
||||
// let workers_f = async {
|
||||
// let instance_name = rd_string(5);
|
||||
|
||||
// let ip = windmill_common::external_ip::get_ip()
|
||||
// .await
|
||||
// .unwrap_or_else(|e| {
|
||||
// tracing::warn!(error = e.to_string(), "failed to get external IP");
|
||||
// "unretrievable IP".to_string()
|
||||
// });
|
||||
// let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
|
||||
// windmill_worker::run_worker(
|
||||
// &db.clone(),
|
||||
// &instance_name,
|
||||
// worker_name,
|
||||
// 1,
|
||||
// 1,
|
||||
// &ip,
|
||||
// rx.resubscribe(),
|
||||
// )
|
||||
// .await;
|
||||
// Ok(()) as anyhow::Result<()>
|
||||
// };
|
||||
|
||||
// let metrics_f = async {
|
||||
// match metrics_addr {
|
||||
// Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
|
||||
// .await
|
||||
// .map_err(anyhow::Error::from),
|
||||
// None => Ok(()),
|
||||
// }
|
||||
// };
|
||||
|
||||
// futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
|
||||
|
||||
// Ok(())
|
||||
// }
|
||||
@@ -1,595 +0,0 @@
|
||||
use std::process::Stdio;
|
||||
|
||||
use itertools::Itertools;
|
||||
use regex::Regex;
|
||||
use sqlx::{Pool, Postgres};
|
||||
use tokio::{
|
||||
fs::{metadata, DirBuilder, File},
|
||||
io::AsyncReadExt,
|
||||
process::Command,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use windmill_common::{
|
||||
error::{self, Error},
|
||||
jobs::QueuedJob,
|
||||
utils::calculate_hash,
|
||||
};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref PYTHON_PATH: String =
|
||||
std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
|
||||
|
||||
static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
|
||||
static ref PIP_EXTRA_INDEX_URL: Option<String> = std::env::var("PIP_EXTRA_INDEX_URL").ok();
|
||||
static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
|
||||
static ref PIP_LOCAL_DEPENDENCIES: Option<Vec<String>> = {
|
||||
let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
|
||||
.ok()
|
||||
.map(|x| x.split(',').map(|x| x.to_string()).collect());
|
||||
if pip_local_dependencies == Some(vec!["".to_string()]) {
|
||||
None
|
||||
} else {
|
||||
pip_local_dependencies
|
||||
}
|
||||
};
|
||||
|
||||
static ref ADDITIONAL_PYTHON_PATHS: Option<Vec<String>> = std::env::var("ADDITIONAL_PYTHON_PATHS")
|
||||
.ok()
|
||||
.map(|x| x.split(':').map(|x| x.to_string()).collect());
|
||||
|
||||
static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
|
||||
|
||||
}
|
||||
|
||||
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
|
||||
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
|
||||
const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
use crate::global_cache::{build_tar_and_push, pull_from_tar};
|
||||
|
||||
use crate::{
|
||||
common::{read_result, set_logs},
|
||||
create_args_and_out_file, get_reserved_variables, handle_child, write_file,
|
||||
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV,
|
||||
PIP_CACHE_DIR, S3_CACHE_BUCKET,
|
||||
};
|
||||
|
||||
pub async fn create_dependencies_dir(job_dir: &str) {
|
||||
DirBuilder::new()
|
||||
.recursive(true)
|
||||
.create(&format!("{job_dir}/dependencies"))
|
||||
.await
|
||||
.expect("could not create dependencies dir");
|
||||
}
|
||||
|
||||
pub async fn pip_compile(
|
||||
job_id: &Uuid,
|
||||
requirements: &str,
|
||||
logs: &mut String,
|
||||
job_dir: &str,
|
||||
db: &Pool<Postgres>,
|
||||
worker_name: &str,
|
||||
w_id: &str,
|
||||
) -> error::Result<String> {
|
||||
logs.push_str(&format!("\nresolving dependencies..."));
|
||||
set_logs(logs, job_id, db).await;
|
||||
logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
|
||||
let req_hash = calculate_hash(&requirements);
|
||||
if let Some(cached) = sqlx::query_scalar!(
|
||||
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
|
||||
req_hash
|
||||
)
|
||||
.fetch_optional(db)
|
||||
.await?
|
||||
{
|
||||
logs.push_str(&format!("\nfound cached resolution"));
|
||||
return Ok(cached);
|
||||
}
|
||||
let file = "requirements.in";
|
||||
let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
|
||||
let deps = pip_local_dependencies.clone();
|
||||
requirements
|
||||
.lines()
|
||||
.filter(|s| !deps.contains(&s.to_string()))
|
||||
.join("\n")
|
||||
} else {
|
||||
requirements.to_string()
|
||||
};
|
||||
write_file(job_dir, file, &requirements).await?;
|
||||
|
||||
let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
|
||||
if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
|
||||
args.extend(["--extra-index-url", url]);
|
||||
}
|
||||
if let Some(url) = PIP_INDEX_URL.as_ref() {
|
||||
args.extend(["--index-url", url]);
|
||||
}
|
||||
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
|
||||
args.extend(["--trusted-host", host]);
|
||||
}
|
||||
let child = Command::new("pip-compile")
|
||||
.current_dir(job_dir)
|
||||
.args(args)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?;
|
||||
handle_child(
|
||||
job_id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
"pip-compile",
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
|
||||
let path_lock = format!("{job_dir}/requirements.txt");
|
||||
let mut file = File::open(path_lock).await?;
|
||||
let mut req_content = "".to_string();
|
||||
file.read_to_string(&mut req_content).await?;
|
||||
let lockfile = req_content
|
||||
.lines()
|
||||
.filter(|x| !x.trim_start().starts_with('#'))
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join("\n");
|
||||
sqlx::query!(
|
||||
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
|
||||
req_hash,
|
||||
lockfile
|
||||
).fetch_optional(db).await?;
|
||||
Ok(lockfile)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_python_job(
|
||||
requirements_o: Option<String>,
|
||||
job_dir: &str,
|
||||
worker_dir: &str,
|
||||
worker_name: &str,
|
||||
job: &QueuedJob,
|
||||
logs: &mut String,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
client: &AuthedClientBackgroundTask,
|
||||
inner_content: &String,
|
||||
shared_mount: &str,
|
||||
base_internal_url: &str,
|
||||
) -> windmill_common::error::Result<serde_json::Value> {
|
||||
create_dependencies_dir(job_dir).await;
|
||||
|
||||
let mut additional_python_paths: Vec<String> =
|
||||
ADDITIONAL_PYTHON_PATHS.to_owned().unwrap_or_else(|| vec![]);
|
||||
|
||||
let requirements = match requirements_o {
|
||||
Some(r) => r,
|
||||
None => {
|
||||
let requirements = windmill_parser_py::parse_python_imports(&inner_content)?.join("\n");
|
||||
if requirements.is_empty() {
|
||||
"".to_string()
|
||||
} else {
|
||||
pip_compile(
|
||||
&job.id,
|
||||
&requirements,
|
||||
logs,
|
||||
job_dir,
|
||||
db,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
Error::ExecutionErr(format!("pip compile failed: {}", e.to_string()))
|
||||
})?
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if requirements.len() > 0 {
|
||||
additional_python_paths = handle_python_reqs(
|
||||
requirements
|
||||
.split("\n")
|
||||
.filter(|x| !x.starts_with("--"))
|
||||
.collect(),
|
||||
&job.id,
|
||||
&job.workspace_id,
|
||||
logs,
|
||||
db,
|
||||
worker_name,
|
||||
job_dir,
|
||||
worker_dir,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
|
||||
|
||||
set_logs(logs, &job.id, db).await;
|
||||
|
||||
let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
|
||||
|
||||
let script_path_splitted = &job.script_path().split("/");
|
||||
let dirs_full = script_path_splitted
|
||||
.clone()
|
||||
.take(script_path_splitted.clone().count() - 1)
|
||||
.join("/")
|
||||
.replace("-", "_");
|
||||
let dirs = if dirs_full.len() > 0 {
|
||||
dirs_full
|
||||
} else {
|
||||
"tmp".to_string()
|
||||
};
|
||||
let last = script_path_splitted
|
||||
.clone()
|
||||
.last()
|
||||
.unwrap()
|
||||
.replace("-", "_")
|
||||
.replace(" ", "_")
|
||||
.to_lowercase();
|
||||
let module_dir = format!("{}/{}", job_dir, dirs);
|
||||
tokio::fs::create_dir_all(format!("{module_dir}/")).await?;
|
||||
let _ = write_file(&module_dir, &format!("{last}.py"), inner_content).await?;
|
||||
if relative_imports {
|
||||
let _ = write_file(&job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
|
||||
}
|
||||
|
||||
let sig = windmill_parser_py::parse_python_signature(inner_content)?;
|
||||
let transforms = sig
|
||||
.args
|
||||
.iter()
|
||||
.map(|x| match x.typ {
|
||||
windmill_parser::Typ::Bytes => {
|
||||
let name = &x.name;
|
||||
format!(
|
||||
"if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n \
|
||||
kwargs[\"{name}\"] = base64.b64decode(kwargs[\"{name}\"])\n",
|
||||
)
|
||||
}
|
||||
windmill_parser::Typ::Datetime => {
|
||||
let name = &x.name;
|
||||
format!(
|
||||
"if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n \
|
||||
kwargs[\"{name}\"] = datetime.strptime(kwargs[\"{name}\"], \
|
||||
'%Y-%m-%dT%H:%M')\n",
|
||||
)
|
||||
}
|
||||
_ => "".to_string(),
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join("");
|
||||
let client = client.get_authed().await;
|
||||
create_args_and_out_file(&client, job, job_dir).await?;
|
||||
|
||||
let import_loader = if relative_imports {
|
||||
"import loader"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let import_base64 = if sig
|
||||
.args
|
||||
.iter()
|
||||
.any(|x| x.typ == windmill_parser::Typ::Bytes)
|
||||
{
|
||||
"import base64"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let import_datetime = if sig
|
||||
.args
|
||||
.iter()
|
||||
.any(|x| x.typ == windmill_parser::Typ::Datetime)
|
||||
{
|
||||
"from datetime import datetime"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let spread = if sig.star_kwargs {
|
||||
"args = kwargs".to_string()
|
||||
} else {
|
||||
sig.args
|
||||
.into_iter()
|
||||
.map(|x| {
|
||||
let name = &x.name;
|
||||
if x.default.is_none() {
|
||||
format!("args[\"{name}\"] = kwargs.get(\"{name}\")")
|
||||
} else {
|
||||
format!(
|
||||
r#"args["{name}"] = kwargs.get("{name}")
|
||||
if args["{name}"] is None:
|
||||
del args["{name}"]"#
|
||||
)
|
||||
}
|
||||
})
|
||||
.join("\n")
|
||||
};
|
||||
|
||||
let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
|
||||
let wrapper_content: String = format!(
|
||||
r#"
|
||||
import json
|
||||
{import_loader}
|
||||
{import_base64}
|
||||
{import_datetime}
|
||||
import traceback
|
||||
import sys
|
||||
from {module_dir_dot} import {last} as inner_script
|
||||
|
||||
|
||||
with open("args.json") as f:
|
||||
kwargs = json.load(f, strict=False)
|
||||
args = {{}}
|
||||
{transforms}
|
||||
{spread}
|
||||
for k, v in list(args.items()):
|
||||
if v == '<function call>':
|
||||
del args[k]
|
||||
|
||||
def to_b_64(v: bytes):
|
||||
import base64
|
||||
b64 = base64.b64encode(v)
|
||||
return b64.decode('ascii')
|
||||
|
||||
try:
|
||||
res = inner_script.main(**args)
|
||||
typ = type(res)
|
||||
if typ.__name__ == 'DataFrame':
|
||||
if typ.__module__ == 'pandas.core.frame':
|
||||
res = res.values.tolist()
|
||||
elif typ.__module__ == 'polars.dataframe.frame':
|
||||
res = res.rows()
|
||||
elif typ.__name__ == 'bytes':
|
||||
res = to_b_64(res)
|
||||
elif typ.__name__ == 'dict':
|
||||
for k, v in res.items():
|
||||
if type(v).__name__ == 'bytes':
|
||||
res[k] = to_b_64(v)
|
||||
res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
|
||||
with open("result.json", 'w') as f:
|
||||
f.write(res_json)
|
||||
except Exception as e:
|
||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
tb = traceback.format_tb(exc_traceback)
|
||||
with open("result.json", 'w') as f:
|
||||
err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}, separators=(',', ':'), default=str).replace('\n', '')
|
||||
f.write(err_json)
|
||||
sys.exit(1)
|
||||
"#,
|
||||
);
|
||||
write_file(job_dir, "wrapper.py", &wrapper_content).await?;
|
||||
|
||||
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
|
||||
let additional_python_paths_folders = additional_python_paths.iter().join(":");
|
||||
if !*DISABLE_NSJAIL {
|
||||
let shared_deps = additional_python_paths
|
||||
.into_iter()
|
||||
.map(|pp| {
|
||||
format!(
|
||||
r#"
|
||||
mount {{
|
||||
src: "{pp}"
|
||||
dst: "{pp}"
|
||||
is_bind: true
|
||||
rw: false
|
||||
}}
|
||||
"#
|
||||
)
|
||||
})
|
||||
.join("\n");
|
||||
let _ = write_file(
|
||||
job_dir,
|
||||
"run.config.proto",
|
||||
&NSJAIL_CONFIG_RUN_PYTHON3_CONTENT
|
||||
.replace("{JOB_DIR}", job_dir)
|
||||
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
|
||||
.replace("{SHARED_MOUNT}", shared_mount)
|
||||
.replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
|
||||
.replace("{MAIN}", format!("{dirs}/{last}").as_str())
|
||||
.replace(
|
||||
"{ADDITIONAL_PYTHON_PATHS}",
|
||||
additional_python_paths_folders.as_str(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
worker_name = %worker_name,
|
||||
job_id = %job.id,
|
||||
workspace_id = %job.workspace_id,
|
||||
"started python code execution {}",
|
||||
job.id
|
||||
);
|
||||
let child = if !*DISABLE_NSJAIL {
|
||||
Command::new(NSJAIL_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
|
||||
.envs(reserved_variables)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.args(vec![
|
||||
"--config",
|
||||
"run.config.proto",
|
||||
"--",
|
||||
PYTHON_PATH.as_str(),
|
||||
"-u",
|
||||
"-m",
|
||||
"wrapper",
|
||||
])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
} else {
|
||||
Command::new(PYTHON_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(reserved_variables)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.args(vec!["-u", "-m", "wrapper"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
};
|
||||
|
||||
handle_child(
|
||||
&job.id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
!*DISABLE_NSJAIL,
|
||||
worker_name,
|
||||
&job.workspace_id,
|
||||
"python run",
|
||||
)
|
||||
.await?;
|
||||
read_result(job_dir).await
|
||||
}
|
||||
|
||||
pub async fn handle_python_reqs(
|
||||
requirements: Vec<&str>,
|
||||
job_id: &Uuid,
|
||||
w_id: &str,
|
||||
logs: &mut String,
|
||||
db: &sqlx::Pool<sqlx::Postgres>,
|
||||
worker_name: &str,
|
||||
job_dir: &str,
|
||||
worker_dir: &str,
|
||||
) -> error::Result<Vec<String>> {
|
||||
let mut req_paths: Vec<String> = vec![];
|
||||
let mut vars = vec![("PATH", PATH_ENV.as_str())];
|
||||
if !*DISABLE_NSJAIL {
|
||||
if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
|
||||
vars.push(("EXTRA_INDEX_URL", url));
|
||||
}
|
||||
if let Some(url) = PIP_INDEX_URL.as_ref() {
|
||||
vars.push(("INDEX_URL", url));
|
||||
}
|
||||
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
|
||||
vars.push(("TRUSTED_HOST", host));
|
||||
}
|
||||
let _ = write_file(
|
||||
job_dir,
|
||||
"download.config.proto",
|
||||
&NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
|
||||
.replace("{WORKER_DIR}", &worker_dir)
|
||||
.replace("{CACHE_DIR}", PIP_CACHE_DIR)
|
||||
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
|
||||
)
|
||||
.await?;
|
||||
};
|
||||
|
||||
for req in requirements {
|
||||
// todo: handle many reqs
|
||||
let venv_p = format!("{PIP_CACHE_DIR}/{req}");
|
||||
if metadata(&venv_p).await.is_ok() {
|
||||
req_paths.push(venv_p);
|
||||
continue;
|
||||
}
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
if let Some(ref bucket) = *S3_CACHE_BUCKET {
|
||||
sqlx::query_scalar!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id)
|
||||
.execute(db)
|
||||
.await?;
|
||||
if pull_from_tar(bucket, venv_p.clone()).await.is_ok() {
|
||||
req_paths.push(venv_p.clone());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
logs.push_str("\n--- PIP INSTALL ---\n");
|
||||
logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
|
||||
|
||||
tracing::info!(
|
||||
worker_name = %worker_name,
|
||||
job_id = %job_id,
|
||||
workspace_id = %w_id,
|
||||
"started setup python dependencies"
|
||||
);
|
||||
|
||||
let child = if !*DISABLE_NSJAIL {
|
||||
tracing::info!(
|
||||
worker_name = %worker_name,
|
||||
job_id = %job_id,
|
||||
workspace_id = %w_id,
|
||||
"starting nsjail"
|
||||
);
|
||||
let mut vars = vars.clone();
|
||||
let req = req.to_string();
|
||||
vars.push(("REQ", &req));
|
||||
vars.push(("TARGET", &venv_p));
|
||||
Command::new(NSJAIL_PATH.as_str())
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(vars)
|
||||
.args(vec!["--config", "download.config.proto"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
} else {
|
||||
let mut args = vec![
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
&req,
|
||||
"-I",
|
||||
"--no-deps",
|
||||
"--no-color",
|
||||
"--isolated",
|
||||
"--no-warn-conflicts",
|
||||
"--disable-pip-version-check",
|
||||
"-t",
|
||||
venv_p.as_str(),
|
||||
];
|
||||
if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
|
||||
args.extend(["--extra-index-url", url]);
|
||||
}
|
||||
if let Some(url) = PIP_INDEX_URL.as_ref() {
|
||||
args.extend(["--index-url", url]);
|
||||
}
|
||||
if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
|
||||
args.extend(["--trusted-host", &host]);
|
||||
}
|
||||
Command::new(PYTHON_PATH.as_str())
|
||||
.env_clear()
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.args(args)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()?
|
||||
};
|
||||
|
||||
let child = handle_child(
|
||||
&job_id,
|
||||
db,
|
||||
logs,
|
||||
child,
|
||||
false,
|
||||
worker_name,
|
||||
&w_id,
|
||||
&format!("pip install {req}"),
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
worker_name = %worker_name,
|
||||
job_id = %job_id,
|
||||
workspace_id = %w_id,
|
||||
is_ok = child.is_ok(),
|
||||
"finished setting up python dependencies {}",
|
||||
job_id
|
||||
);
|
||||
child?;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
if let Some(ref bucket) = *S3_CACHE_BUCKET {
|
||||
let venv_p = venv_p.clone();
|
||||
tokio::spawn(build_tar_and_push(bucket, venv_p));
|
||||
}
|
||||
req_paths.push(venv_p);
|
||||
}
|
||||
Ok(req_paths)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,9 +19,6 @@ use tokio::sync::mpsc::Sender;
|
||||
use tracing::instrument;
|
||||
use uuid::Uuid;
|
||||
use windmill_common::flow_status::{FlowStatusModuleWParent, Iterator, JobResult};
|
||||
use windmill_common::jobs::{
|
||||
script_hash_to_tag, script_path_to_payload, JobPayload, QueuedJob, RawCode,
|
||||
};
|
||||
use windmill_common::{
|
||||
error::{self, to_anyhow, Error},
|
||||
flow_status::{
|
||||
@@ -33,7 +30,9 @@ use windmill_common::{
|
||||
|
||||
type DB = sqlx::Pool<sqlx::Postgres>;
|
||||
|
||||
use windmill_queue::{canceled_job_to_result, get_queued_job, push, QueueTransaction};
|
||||
use windmill_queue::{
|
||||
canceled_job_to_result, get_queued_job, push, JobPayload, QueueTransaction, QueuedJob, RawCode,
|
||||
};
|
||||
|
||||
// #[instrument(level = "trace", skip_all)]
|
||||
pub async fn update_flow_status_after_job_completion<
|
||||
@@ -1316,7 +1315,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
|
||||
ContinuePayload::ForloopJobs { n, .. } => *n,
|
||||
};
|
||||
for i in (0..len).into_iter() {
|
||||
let payload_tag = match &job_payloads {
|
||||
let payload = match &job_payloads {
|
||||
ContinuePayload::SingleJob(payload) => payload.clone(),
|
||||
ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(),
|
||||
ContinuePayload::ForloopJobs { modules, .. } => {
|
||||
@@ -1325,16 +1324,13 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
|
||||
failure_module.id_append(&format!("{}/{}", status.step, i));
|
||||
fm = Some(failure_module);
|
||||
}
|
||||
JobPayloadWithTag {
|
||||
payload: JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules: (*modules).clone(),
|
||||
failure_module: fm.clone(),
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
|
||||
JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules: (*modules).clone(),
|
||||
failure_module: fm.clone(),
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
tag: None,
|
||||
path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -1374,11 +1370,10 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
|
||||
} else {
|
||||
flow_job.root_job.or_else(|| Some(flow_job.id))
|
||||
};
|
||||
|
||||
let (uuid, inner_tx) = push(
|
||||
tx,
|
||||
&flow_job.workspace_id,
|
||||
payload_tag.payload,
|
||||
payload,
|
||||
ok.unwrap_or_else(|| Map::new()),
|
||||
&flow_job.created_by,
|
||||
&flow_job.email,
|
||||
@@ -1391,7 +1386,6 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
|
||||
continue_on_same_worker,
|
||||
err,
|
||||
flow_job.visible_to_owner,
|
||||
payload_tag.tag,
|
||||
)
|
||||
.await?;
|
||||
tx = inner_tx;
|
||||
@@ -1610,15 +1604,10 @@ enum NextStatus {
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct JobPayloadWithTag {
|
||||
payload: JobPayload,
|
||||
tag: Option<String>,
|
||||
}
|
||||
enum ContinuePayload {
|
||||
SingleJob(JobPayloadWithTag),
|
||||
SingleJob(JobPayload),
|
||||
ForloopJobs { n: usize, modules: Vec<FlowModule> },
|
||||
BranchAllJobs(Vec<JobPayloadWithTag>),
|
||||
BranchAllJobs(Vec<JobPayload>),
|
||||
}
|
||||
|
||||
enum NextFlowTransform {
|
||||
@@ -1626,6 +1615,22 @@ enum NextFlowTransform {
|
||||
Continue(ContinuePayload, NextStatus),
|
||||
}
|
||||
|
||||
// a similar function exists on the backend
|
||||
// TODO: rewrite this to use an endpoint in the backend directly, instead of checking for hub itself, and then using the API
|
||||
async fn script_path_to_payload<'c>(
|
||||
script_path: &str,
|
||||
db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
|
||||
w_id: &String,
|
||||
) -> Result<JobPayload, Error> {
|
||||
let job_payload = if script_path.starts_with("hub/") {
|
||||
JobPayload::ScriptHub { path: script_path.to_owned() }
|
||||
} else {
|
||||
let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
|
||||
JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
|
||||
};
|
||||
Ok(job_payload)
|
||||
}
|
||||
|
||||
async fn compute_next_flow_transform<'c>(
|
||||
flow_job: &QueuedJob,
|
||||
flow: &FlowValue,
|
||||
@@ -1642,10 +1647,7 @@ async fn compute_next_flow_transform<'c>(
|
||||
FlowModuleValue::Identity => Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag {
|
||||
payload: JobPayload::Identity,
|
||||
tag: None,
|
||||
}),
|
||||
ContinuePayload::SingleJob(JobPayload::Identity),
|
||||
NextStatus::NextStep,
|
||||
),
|
||||
)),
|
||||
@@ -1654,46 +1656,41 @@ async fn compute_next_flow_transform<'c>(
|
||||
Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag: None }),
|
||||
ContinuePayload::SingleJob(payload),
|
||||
NextStatus::NextStep,
|
||||
),
|
||||
))
|
||||
}
|
||||
FlowModuleValue::Script { path: script_path, hash: script_hash, .. } => {
|
||||
let (payload, tag) = if script_hash.is_none() {
|
||||
let payload = if script_hash.is_none() {
|
||||
script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
|
||||
} else {
|
||||
let hash = script_hash.clone().unwrap();
|
||||
let tag = script_hash_to_tag(&hash, &mut tx, &flow_job.workspace_id).await?;
|
||||
(
|
||||
JobPayload::ScriptHash { hash, path: script_path.to_owned() },
|
||||
tag,
|
||||
)
|
||||
JobPayload::ScriptHash {
|
||||
hash: script_hash.clone().unwrap(),
|
||||
path: script_path.to_owned(),
|
||||
}
|
||||
};
|
||||
Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag }),
|
||||
ContinuePayload::SingleJob(payload),
|
||||
NextStatus::NextStep,
|
||||
),
|
||||
))
|
||||
}
|
||||
FlowModuleValue::RawScript { path, content, language, lock, tag, .. } => {
|
||||
FlowModuleValue::RawScript { path, content, language, lock, .. } => {
|
||||
let path = path
|
||||
.clone()
|
||||
.or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
|
||||
Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag {
|
||||
payload: JobPayload::Code(RawCode {
|
||||
path,
|
||||
content: content.clone(),
|
||||
language: language.clone(),
|
||||
lock: lock.clone(),
|
||||
}),
|
||||
tag: tag.clone(),
|
||||
}),
|
||||
ContinuePayload::SingleJob(JobPayload::Code(RawCode {
|
||||
path,
|
||||
content: content.clone(),
|
||||
language: language.clone(),
|
||||
lock: lock.clone(),
|
||||
})),
|
||||
NextStatus::NextStep,
|
||||
),
|
||||
))
|
||||
@@ -1791,20 +1788,13 @@ async fn compute_next_flow_transform<'c>(
|
||||
Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag {
|
||||
payload: JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules: (*modules).clone(),
|
||||
failure_module: fm,
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
path: Some(format!(
|
||||
"{}/loop-{}",
|
||||
flow_job.script_path(),
|
||||
ns.index
|
||||
)),
|
||||
ContinuePayload::SingleJob(JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules: (*modules).clone(),
|
||||
failure_module: fm,
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
tag: None,
|
||||
path: Some(format!("{}/loop-{}", flow_job.script_path(), ns.index)),
|
||||
}),
|
||||
NextStatus::NextLoopIteration(ns),
|
||||
),
|
||||
@@ -1877,20 +1867,17 @@ async fn compute_next_flow_transform<'c>(
|
||||
Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag {
|
||||
payload: JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules,
|
||||
failure_module: fm,
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
path: Some(format!(
|
||||
"{}/branchone-{}",
|
||||
flow_job.script_path(),
|
||||
status.step
|
||||
)),
|
||||
ContinuePayload::SingleJob(JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules,
|
||||
failure_module: fm,
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
tag: None,
|
||||
path: Some(format!(
|
||||
"{}/branchone-{}",
|
||||
flow_job.script_path(),
|
||||
status.step
|
||||
)),
|
||||
}),
|
||||
NextStatus::BranchChosen(branch),
|
||||
),
|
||||
@@ -1920,20 +1907,17 @@ async fn compute_next_flow_transform<'c>(
|
||||
.id_append(&format!("{}/{i}", status.step));
|
||||
fm = Some(failure_module);
|
||||
}
|
||||
JobPayloadWithTag {
|
||||
payload: JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules: b.modules.clone(),
|
||||
failure_module: fm.clone(),
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
path: Some(format!(
|
||||
"{}/branchall-{}",
|
||||
flow_job.script_path(),
|
||||
i
|
||||
)),
|
||||
JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules: b.modules.clone(),
|
||||
failure_module: fm.clone(),
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
tag: None,
|
||||
path: Some(format!(
|
||||
"{}/branchall-{}",
|
||||
flow_job.script_path(),
|
||||
i
|
||||
)),
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
@@ -1994,20 +1978,17 @@ async fn compute_next_flow_transform<'c>(
|
||||
Ok((
|
||||
tx,
|
||||
NextFlowTransform::Continue(
|
||||
ContinuePayload::SingleJob(JobPayloadWithTag {
|
||||
payload: JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules,
|
||||
failure_module: fm.clone(),
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
path: Some(format!(
|
||||
"{}/branchall-{}",
|
||||
flow_job.script_path(),
|
||||
branch_status.branch
|
||||
)),
|
||||
ContinuePayload::SingleJob(JobPayload::RawFlow {
|
||||
value: FlowValue {
|
||||
modules,
|
||||
failure_module: fm.clone(),
|
||||
same_worker: flow.same_worker,
|
||||
},
|
||||
tag: None,
|
||||
path: Some(format!(
|
||||
"{}/branchall-{}",
|
||||
flow_job.script_path(),
|
||||
branch_status.branch
|
||||
)),
|
||||
}),
|
||||
NextStatus::NextBranchStep(NextBranch { status: branch_status, flow_jobs }),
|
||||
),
|
||||
|
||||
@@ -48,9 +48,6 @@ wmill sync pull
|
||||
wmill sync push
|
||||
```
|
||||
|
||||
We recommend using the --yaml option to use yaml instead of json as the encoding
|
||||
format. Yaml will be made the default soon.
|
||||
|
||||
### Pushing individual files
|
||||
|
||||
You can push individual resources using
|
||||
|
||||
162
cli/apps.ts
162
cli/apps.ts
@@ -1,65 +1,120 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
|
||||
import { Any, decoverto, model, property } from "./decoverto.ts";
|
||||
import {
|
||||
AppService,
|
||||
AppWithLastVersion,
|
||||
colors,
|
||||
Command,
|
||||
ListableApp,
|
||||
microdiff,
|
||||
Policy,
|
||||
Table,
|
||||
} from "./deps.ts";
|
||||
import {
|
||||
Difference,
|
||||
GlobalOptions,
|
||||
isSuperset,
|
||||
parseFromFile,
|
||||
removeType,
|
||||
PushDiffs,
|
||||
Resource,
|
||||
setValueByPath,
|
||||
} from "./types.ts";
|
||||
|
||||
export interface AppFile {
|
||||
@model()
|
||||
export class AppFile implements Resource, PushDiffs {
|
||||
@property(Any)
|
||||
value: any;
|
||||
@property(() => String)
|
||||
summary: string;
|
||||
@property(Any)
|
||||
policy: Policy;
|
||||
}
|
||||
|
||||
export async function pushApp(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
app: AppFile | AppWithLastVersion | undefined,
|
||||
newApp: AppFile,
|
||||
raw: boolean
|
||||
): Promise<void> {
|
||||
remotePath = removeType(remotePath, "app");
|
||||
if (raw) {
|
||||
// deleting old app if it exists in raw mode
|
||||
constructor(value: string, summary: string, policy: Policy) {
|
||||
this.value = value;
|
||||
this.summary = summary;
|
||||
this.policy = policy;
|
||||
}
|
||||
async pushDiffs(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
diffs: Difference[]
|
||||
): Promise<void> {
|
||||
let app: AppWithLastVersion | undefined = undefined;
|
||||
try {
|
||||
app = await AppService.getAppByPath({ workspace, path: remotePath });
|
||||
} catch {
|
||||
//ignore
|
||||
} catch (e) {}
|
||||
|
||||
if (app) {
|
||||
console.log(
|
||||
colors.bold.yellow(
|
||||
`Applying ${diffs.length} diffs to existing app... ${remotePath}`
|
||||
)
|
||||
);
|
||||
const changeset: {
|
||||
summary?: string | undefined;
|
||||
value?: any;
|
||||
policy?: Policy | undefined;
|
||||
} = {};
|
||||
for (const diff of diffs) {
|
||||
if (
|
||||
diff.type !== "REMOVE" &&
|
||||
diff.path[0] !== "value" &&
|
||||
diff.path[0] !== "policy" &&
|
||||
(diff.path.length !== 1 ||
|
||||
!["summary"].includes(diff.path[0] as string))
|
||||
) {
|
||||
throw new Error("Invalid app diff with path " + diff.path);
|
||||
}
|
||||
if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
||||
setValueByPath(changeset, diff.path, diff.value);
|
||||
} else if (diff.type === "REMOVE") {
|
||||
setValueByPath(changeset, diff.path, null);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
(!changeset?.policy ||
|
||||
JSON.stringify(changeset?.policy) == JSON.stringify(app.policy)) &&
|
||||
(!changeset?.value ||
|
||||
JSON.stringify(changeset?.value) == JSON.stringify(app.value)) &&
|
||||
(!changeset?.summary || changeset.summary == app.summary)
|
||||
) {
|
||||
console.log(
|
||||
colors.yellow(`No changes to push for app ${remotePath}, skipping`)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const hasChanges = Object.values(changeset).some(
|
||||
(v) => v !== null && typeof v !== "undefined"
|
||||
);
|
||||
if (!hasChanges) {
|
||||
return;
|
||||
}
|
||||
|
||||
await AppService.updateApp({
|
||||
workspace,
|
||||
path: remotePath,
|
||||
requestBody: changeset,
|
||||
});
|
||||
} else {
|
||||
console.log(colors.yellow.bold("Creating new app..."));
|
||||
|
||||
await AppService.createApp({
|
||||
workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
policy: this.policy,
|
||||
summary: this.summary,
|
||||
value: this.value,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (app) {
|
||||
if (isSuperset(newApp, app)) {
|
||||
return;
|
||||
}
|
||||
await AppService.updateApp({
|
||||
async push(workspace: string, remotePath: string): Promise<void> {
|
||||
await this.pushDiffs(
|
||||
workspace,
|
||||
path: remotePath,
|
||||
requestBody: {
|
||||
...newApp,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
console.log(colors.yellow.bold("Creating new app..."));
|
||||
|
||||
await AppService.createApp({
|
||||
workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
...newApp,
|
||||
},
|
||||
});
|
||||
remotePath,
|
||||
microdiff({}, this, { cyclesFix: false })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,26 +154,21 @@ async function push(opts: GlobalOptions, filePath: string) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
let app: AppWithLastVersion | undefined = undefined;
|
||||
try {
|
||||
app = await AppService.getAppByPath({
|
||||
workspace: workspace.workspaceId,
|
||||
path: remotePath,
|
||||
});
|
||||
} catch {
|
||||
// app doesn't exist
|
||||
}
|
||||
|
||||
await pushApp(
|
||||
workspace.workspaceId,
|
||||
remotePath,
|
||||
app,
|
||||
parseFromFile(filePath),
|
||||
false
|
||||
);
|
||||
await pushApp(filePath, workspace.workspaceId, remotePath);
|
||||
console.log(colors.bold.underline.green("App pushed"));
|
||||
}
|
||||
|
||||
export async function pushApp(
|
||||
filePath: string,
|
||||
workspace: string,
|
||||
remotePath: string
|
||||
) {
|
||||
const data = decoverto
|
||||
.type(AppFile)
|
||||
.rawToInstance(await Deno.readTextFile(filePath));
|
||||
await data.push(workspace, remotePath);
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
.description("app related commands")
|
||||
.action(list as any)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { colors, GlobalUserInfo, log, setClient, UserService } from "./deps.ts";
|
||||
import { colors, GlobalUserInfo, setClient, UserService } from "./deps.ts";
|
||||
import { loginInteractive, tryGetLoginInfo } from "./login.ts";
|
||||
import { GlobalOptions } from "./types.ts";
|
||||
import {
|
||||
@@ -53,7 +53,7 @@ export async function resolveWorkspace(
|
||||
): Promise<Workspace> {
|
||||
const res = await tryResolveWorkspace(opts);
|
||||
if (res.isError) {
|
||||
log.info(colors.red.bold(res.error));
|
||||
console.log(colors.red.bold(res.error));
|
||||
return Deno.exit(-1);
|
||||
} else {
|
||||
return res.value;
|
||||
@@ -75,7 +75,7 @@ export async function requireLogin(
|
||||
try {
|
||||
return await UserService.globalWhoami();
|
||||
} catch {
|
||||
log.info(
|
||||
console.log(
|
||||
"! Could not reach API given existing credentials. Attempting to reauth..."
|
||||
);
|
||||
const newToken = await loginInteractive(workspace.remote);
|
||||
@@ -119,7 +119,7 @@ export async function tryResolveVersion(
|
||||
|
||||
export function validatePath(path: string): boolean {
|
||||
if (!(path.startsWith("g") || path.startsWith("u") || path.startsWith("f"))) {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.red(
|
||||
"Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
|
||||
)
|
||||
|
||||
8
cli/decoverto.ts
Normal file
8
cli/decoverto.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
// globally shared decoverto instance
|
||||
import { Decoverto } from "npm:decoverto";
|
||||
const decoverto = new Decoverto();
|
||||
|
||||
// TODO: Properly type FlowModule
|
||||
|
||||
export { Any, array, map, MapShape, model, property } from "npm:decoverto";
|
||||
export { decoverto };
|
||||
@@ -1,6 +1,6 @@
|
||||
// windmill
|
||||
export { setClient } from "https://deno.land/x/windmill@v1.95.1/mod.ts";
|
||||
export * from "https://deno.land/x/windmill@v1.95.1/windmill-api/index.ts";
|
||||
export { setClient } from "https://deno.land/x/windmill@v1.69.3/mod.ts";
|
||||
export * from "https://deno.land/x/windmill@v1.69.3/windmill-api/index.ts";
|
||||
|
||||
// cliffy
|
||||
export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
|
||||
@@ -37,4 +37,3 @@ export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index
|
||||
export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
|
||||
export { default as gitignore_parser } from "npm:gitignore-parser";
|
||||
export { default as JSZip } from "npm:jszip@3.7.1";
|
||||
export * as log from "https://deno.land/std@0.186.0/log/mod.ts";
|
||||
|
||||
176
cli/flow.ts
176
cli/flow.ts
@@ -1,94 +1,121 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { GlobalOptions, isSuperset } from "./types.ts";
|
||||
import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";
|
||||
|
||||
import {
|
||||
Difference,
|
||||
GlobalOptions,
|
||||
PushDiffs,
|
||||
Resource,
|
||||
setValueByPath,
|
||||
} from "./types.ts";
|
||||
import {
|
||||
colors,
|
||||
Command,
|
||||
Flow,
|
||||
FlowModule,
|
||||
FlowService,
|
||||
JobService,
|
||||
microdiff,
|
||||
OpenFlowWPath,
|
||||
Table,
|
||||
} from "./deps.ts";
|
||||
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
|
||||
import { resolve, track_job } from "./script.ts";
|
||||
import { Any, decoverto, model, property } from "./decoverto.ts";
|
||||
|
||||
export interface FlowFile {
|
||||
// this is effectively "OpenFlow" but a copy as it is accepted by the CLI
|
||||
@model()
|
||||
export class FlowFile implements Resource, PushDiffs {
|
||||
@property(() => String)
|
||||
summary: string;
|
||||
@property(() => String)
|
||||
description?: string;
|
||||
@property(Any)
|
||||
value: any;
|
||||
@property(Any)
|
||||
schema?: any;
|
||||
}
|
||||
|
||||
const alreadySynced: string[] = [];
|
||||
|
||||
export async function pushFlow(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
localFlowPath: string
|
||||
): Promise<void> {
|
||||
if (alreadySynced.includes(localFlowPath)) {
|
||||
return;
|
||||
}
|
||||
alreadySynced.push(localFlowPath);
|
||||
let flow: Flow | undefined = undefined;
|
||||
try {
|
||||
flow = await FlowService.getFlowByPath({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
});
|
||||
} catch {
|
||||
// flow doesn't exist
|
||||
constructor(value: any, summary?: string) {
|
||||
this.summary = summary ?? "";
|
||||
this.value = value;
|
||||
}
|
||||
async pushDiffs(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
diffs: Difference[]
|
||||
): Promise<void> {
|
||||
if (
|
||||
await FlowService.existsFlowByPath({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
})
|
||||
) {
|
||||
console.log(
|
||||
colors.bold.yellow(
|
||||
`Applying ${diffs.length} diffs to existing flow... ${remotePath}`
|
||||
)
|
||||
);
|
||||
|
||||
if (!localFlowPath.endsWith("/")) {
|
||||
localFlowPath += "/";
|
||||
}
|
||||
const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
|
||||
const localFlow = yamlParse(localFlowRaw) as FlowFile;
|
||||
|
||||
function replaceInlineScripts(modules: FlowModule[]) {
|
||||
modules.forEach((m) => {
|
||||
if (m.value.type == "rawscript") {
|
||||
const path = m.value.content.split(" ")[1];
|
||||
m.value.content = Deno.readTextFileSync(localFlowPath + path);
|
||||
} else if (m.value.type == "forloopflow") {
|
||||
replaceInlineScripts(m.value.modules);
|
||||
} else if (m.value.type == "branchall") {
|
||||
m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
|
||||
} else if (m.value.type == "branchone") {
|
||||
m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
|
||||
replaceInlineScripts(m.value.default);
|
||||
// TODO: Make these optional in backend (not path ofc)
|
||||
const changeset: OpenFlowWPath = {
|
||||
path: remotePath,
|
||||
summary: this.summary,
|
||||
value: this.value,
|
||||
description: this.description, // This is OpenAPIed as optional, but isn't
|
||||
schema: this.schema, // Same
|
||||
};
|
||||
const base_changeset = { ...changeset };
|
||||
for (const diff of diffs) {
|
||||
if (
|
||||
diff.type !== "REMOVE" &&
|
||||
diff.path[0] !== "value" &&
|
||||
(diff.path.length !== 1 ||
|
||||
!["summary", "description", "schema"].includes(
|
||||
diff.path[0] as string
|
||||
))
|
||||
) {
|
||||
throw new Error("Invalid flow diff with path " + diff.path);
|
||||
}
|
||||
if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
||||
setValueByPath(changeset, diff.path, diff.value);
|
||||
} else if (diff.type === "REMOVE") {
|
||||
setValueByPath(changeset, diff.path, null);
|
||||
}
|
||||
}
|
||||
const hasChanges = Object.values(changeset).some(
|
||||
(v) => v !== null && typeof v !== "undefined"
|
||||
);
|
||||
if (!hasChanges) {
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
replaceInlineScripts(localFlow.value.modules);
|
||||
const update = {
|
||||
...changeset,
|
||||
...base_changeset,
|
||||
};
|
||||
|
||||
if (flow) {
|
||||
if (isSuperset(localFlow, flow)) {
|
||||
console.log(colors.bold.green("Flow is up to date"));
|
||||
return;
|
||||
await FlowService.updateFlow({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
requestBody: update,
|
||||
});
|
||||
} else {
|
||||
console.log(colors.bold.yellow("Creating new flow..."));
|
||||
await FlowService.createFlow({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
summary: this.summary,
|
||||
value: this.value,
|
||||
schema: this.schema,
|
||||
description: this.description,
|
||||
},
|
||||
});
|
||||
}
|
||||
console.log(colors.bold.yellow(`Updating flow ${remotePath}...`));
|
||||
await FlowService.updateFlow({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
...localFlow,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
console.log(colors.bold.yellow("Creating new flow..."));
|
||||
await FlowService.createFlow({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
...localFlow,
|
||||
},
|
||||
});
|
||||
}
|
||||
async push(workspace: string, remotePath: string): Promise<void> {
|
||||
await this.pushDiffs(
|
||||
workspace,
|
||||
remotePath,
|
||||
microdiff({}, this, { cyclesFix: false })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,10 +128,21 @@ async function push(opts: Options, filePath: string, remotePath: string) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
await pushFlow(workspace.workspaceId, remotePath, filePath);
|
||||
await pushFlow(filePath, workspace.workspaceId, remotePath);
|
||||
console.log(colors.bold.underline.green("Flow pushed"));
|
||||
}
|
||||
|
||||
export async function pushFlow(
|
||||
filePath: string,
|
||||
workspace: string,
|
||||
remotePath: string
|
||||
) {
|
||||
const data = decoverto
|
||||
.type(FlowFile)
|
||||
.rawToInstance(await Deno.readTextFile(filePath));
|
||||
await data.push(workspace, remotePath);
|
||||
}
|
||||
|
||||
async function list(opts: GlobalOptions & { showArchived?: boolean }) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
218
cli/folder.ts
218
cli/folder.ts
@@ -1,85 +1,135 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { colors, Command, Folder, FolderService, log, Table } from "./deps.ts";
|
||||
import { colors, Command, FolderService, microdiff } from "./deps.ts";
|
||||
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
|
||||
import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
|
||||
import {
|
||||
Difference,
|
||||
GlobalOptions,
|
||||
PushDiffs,
|
||||
Resource,
|
||||
setValueByPath,
|
||||
} from "./types.ts";
|
||||
import {
|
||||
array,
|
||||
decoverto,
|
||||
map,
|
||||
MapShape,
|
||||
model,
|
||||
property,
|
||||
} from "./decoverto.ts";
|
||||
|
||||
export interface FolderFile {
|
||||
@model()
|
||||
export class FolderFile implements Resource, PushDiffs {
|
||||
@property(array(() => String))
|
||||
owners: Array<string> | undefined;
|
||||
extra_perms: Map<string, boolean> | undefined;
|
||||
display_name: string | undefined;
|
||||
}
|
||||
|
||||
async function list(opts: GlobalOptions) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
const folders = await FolderService.listFolders({
|
||||
workspace: workspace.workspaceId,
|
||||
});
|
||||
|
||||
new Table()
|
||||
.header(["Name", "Owners", "Extra Perms"])
|
||||
.padding(2)
|
||||
.border(true)
|
||||
.body(
|
||||
folders.map((x) => [
|
||||
x.name,
|
||||
x.owners?.join(",") ?? "-",
|
||||
JSON.stringify(x.extra_perms ?? {}),
|
||||
])
|
||||
@property(
|
||||
map(
|
||||
() => String,
|
||||
() => Boolean,
|
||||
{ shape: MapShape.Object }
|
||||
)
|
||||
.render();
|
||||
}
|
||||
)
|
||||
extra_perms: Map<string, boolean> | undefined;
|
||||
@property(() => String)
|
||||
display_name: string | undefined;
|
||||
|
||||
export async function pushFolder(
|
||||
workspace: string,
|
||||
name: string,
|
||||
folder: Folder | FolderFile | undefined,
|
||||
localFolder: FolderFile,
|
||||
raw: boolean
|
||||
): Promise<void> {
|
||||
if (name.startsWith("/")) {
|
||||
name = name.substring(1);
|
||||
}
|
||||
if (name.startsWith("f/")) {
|
||||
name = name.substring(2);
|
||||
}
|
||||
name = name.split("/")[0];
|
||||
log.debug(`Processing local folder ${name}`);
|
||||
async push(workspace: string, remotePath: string): Promise<void> {
|
||||
if (remotePath.startsWith("/")) {
|
||||
remotePath = remotePath.substring(1);
|
||||
}
|
||||
if (remotePath.startsWith("f/")) {
|
||||
remotePath = remotePath.substring(2);
|
||||
}
|
||||
|
||||
if (raw) {
|
||||
// deleting old app if it exists in raw mode
|
||||
await this.pushDiffs(
|
||||
workspace,
|
||||
remotePath,
|
||||
microdiff({}, this, { cyclesFix: false })
|
||||
);
|
||||
}
|
||||
|
||||
async pushDiffs(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
diffs: Difference[]
|
||||
): Promise<void> {
|
||||
if (remotePath.startsWith("/")) {
|
||||
remotePath = remotePath.substring(1);
|
||||
}
|
||||
if (remotePath.startsWith("f/")) {
|
||||
remotePath = remotePath.substring(2);
|
||||
}
|
||||
|
||||
// TODO: Support this in backend
|
||||
let exists: boolean;
|
||||
try {
|
||||
folder = await FolderService.getFolder({ workspace, name });
|
||||
log.debug(`Folder ${name} exists on remote`);
|
||||
exists = !!(await FolderService.getFolder({
|
||||
workspace,
|
||||
name: remotePath,
|
||||
}));
|
||||
} catch {
|
||||
log.debug(`Folder ${name} does not exist on remote`);
|
||||
//ignore
|
||||
exists = false;
|
||||
}
|
||||
}
|
||||
if (exists) {
|
||||
console.log(
|
||||
colors.bold.yellow(
|
||||
`Applying ${diffs.length} diffs to existing folder... ${remotePath}`
|
||||
)
|
||||
);
|
||||
|
||||
if (folder) {
|
||||
if (isSuperset(localFolder, folder)) {
|
||||
log.debug(`Folder ${name} is up to date`);
|
||||
return;
|
||||
const changeset: {
|
||||
owners?: string[] | undefined;
|
||||
extra_perms?: any;
|
||||
display_name?: string | undefined;
|
||||
} = {};
|
||||
for (const diff of diffs) {
|
||||
if (
|
||||
diff.type !== "REMOVE" &&
|
||||
(diff.path.length !== 1 ||
|
||||
!["owners", "extra_perms", "display_name"].includes(
|
||||
diff.path[0] as string
|
||||
))
|
||||
) {
|
||||
console.log(diff.path);
|
||||
throw new Error("Invalid folder diff with path " + diff.path);
|
||||
}
|
||||
if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
||||
setValueByPath(changeset, diff.path, diff.value);
|
||||
} else if (diff.type === "REMOVE") {
|
||||
setValueByPath(changeset, diff.path, null);
|
||||
}
|
||||
}
|
||||
|
||||
const hasChanges = Object.values(changeset).some(
|
||||
(v) => v !== null && typeof v !== "undefined"
|
||||
);
|
||||
if (!hasChanges) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await FolderService.updateFolder({
|
||||
workspace: workspace,
|
||||
name: remotePath,
|
||||
requestBody: {
|
||||
...changeset,
|
||||
extra_perms: changeset.extra_perms
|
||||
? Object.fromEntries(this.extra_perms?.entries() ?? [])
|
||||
: undefined,
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(colors.red.bold(e.body));
|
||||
throw e;
|
||||
}
|
||||
} else {
|
||||
console.log(colors.bold.yellow("Creating new folder: " + remotePath));
|
||||
await FolderService.createFolder({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
name: remotePath,
|
||||
extra_perms: Object.fromEntries(this.extra_perms?.entries() ?? []),
|
||||
owners: this.owners,
|
||||
},
|
||||
});
|
||||
}
|
||||
log.debug(`Folder ${name} is not up-to-date, updating...`);
|
||||
await FolderService.updateFolder({
|
||||
workspace: workspace,
|
||||
name: name,
|
||||
requestBody: {
|
||||
...localFolder,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
console.log(colors.bold.yellow("Creating new folder: " + name));
|
||||
await FolderService.createFolder({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
name: name,
|
||||
...localFolder,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,21 +146,25 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
|
||||
throw new Error("file path must refer to a file.");
|
||||
}
|
||||
|
||||
console.log(colors.bold.yellow("Pushing folder..."));
|
||||
console.log(colors.bold.yellow("Pushing resource..."));
|
||||
|
||||
await pushFolder(
|
||||
workspace.workspaceId,
|
||||
remotePath,
|
||||
undefined,
|
||||
parseFromFile(filePath),
|
||||
false
|
||||
);
|
||||
console.log(colors.bold.underline.green("Folder pushed"));
|
||||
await pushFolder(workspace.workspaceId, filePath, remotePath);
|
||||
console.log(colors.bold.underline.green("Resource pushed"));
|
||||
}
|
||||
|
||||
export async function pushFolder(
|
||||
workspace: string,
|
||||
filePath: string,
|
||||
remotePath: string
|
||||
) {
|
||||
const data = decoverto
|
||||
.type(FolderFile)
|
||||
.rawToInstance(await Deno.readTextFile(filePath));
|
||||
data.push(workspace, remotePath);
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
.description("folder related commands")
|
||||
.action(list as any)
|
||||
.description("resource related commands")
|
||||
.command(
|
||||
"push",
|
||||
"push a local folder spec. This overrides any remote versions."
|
||||
|
||||
53
cli/hub.ts
53
cli/hub.ts
@@ -1,15 +1,14 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { Command, ResourceService, log } from "./deps.ts";
|
||||
import { Command } from "./deps.ts";
|
||||
import { requireLogin, resolveWorkspace } from "./context.ts";
|
||||
import { pushResourceType } from "./resource-type.ts";
|
||||
import { ResourceTypeFile } from "./resource-type.ts";
|
||||
import { GlobalOptions } from "./types.ts";
|
||||
|
||||
async function pull(opts: GlobalOptions) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
|
||||
if (workspace.workspaceId !== "admins") {
|
||||
log.info(
|
||||
"Should only sync to admins workspace, but current is not admins."
|
||||
console.log(
|
||||
"Should only sync to admins workspace, but current is not admins.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -25,12 +24,15 @@ async function pull(opts: GlobalOptions) {
|
||||
created_by: string;
|
||||
created_at: Date;
|
||||
comments: never[];
|
||||
}[] = await fetch("https://hub.windmill.dev/resource_types/list", {
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"X-email": userInfo.email,
|
||||
}[] = await fetch(
|
||||
"https://hub.windmill.dev/resource_types/list",
|
||||
{
|
||||
headers: {
|
||||
"Accept": "application/json",
|
||||
"X-email": userInfo.email,
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
.then((r) => r.json())
|
||||
.then((list: { id: number; name: string }[]) =>
|
||||
list.map((x) =>
|
||||
@@ -38,9 +40,9 @@ async function pull(opts: GlobalOptions) {
|
||||
"https://hub.windmill.dev/resource_types/" + x.id + "/" + x.name,
|
||||
{
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -48,7 +50,7 @@ async function pull(opts: GlobalOptions) {
|
||||
.then((x) =>
|
||||
x.map((x) =>
|
||||
x.json().catch((e) => {
|
||||
log.info(e);
|
||||
console.log(e);
|
||||
return undefined;
|
||||
})
|
||||
)
|
||||
@@ -56,23 +58,14 @@ async function pull(opts: GlobalOptions) {
|
||||
.then((x) => Promise.all(x))
|
||||
.then((x) => x.filter((x) => x).map((x) => x.resource_type));
|
||||
|
||||
const resourceTypes = await ResourceService.listResourceType({
|
||||
workspace: workspace.workspaceId,
|
||||
});
|
||||
|
||||
for (const x of list) {
|
||||
if (resourceTypes.find((y) => y.name === x.name)) {
|
||||
log.info("skipping " + x.name);
|
||||
continue;
|
||||
}
|
||||
log.info("syncing " + x.name);
|
||||
await pushResourceType(
|
||||
workspace.workspaceId,
|
||||
x.name + ".resource-type.json",
|
||||
undefined,
|
||||
x,
|
||||
false
|
||||
);
|
||||
for (
|
||||
const x of list
|
||||
) {
|
||||
console.log("syncing " + x.name);
|
||||
const f = new ResourceTypeFile();
|
||||
f.description = x.description;
|
||||
f.schema = JSON.parse(x.schema);
|
||||
await f.push(workspace.workspaceId, x.name);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
26
cli/login.ts
26
cli/login.ts
@@ -1,11 +1,11 @@
|
||||
import { GlobalOptions } from "./types.ts";
|
||||
import { colors, getAvailablePort, log, Secret, Select } from "./deps.ts";
|
||||
import { open } from "https://deno.land/x/open/index.ts";
|
||||
import { colors, getAvailablePort, Secret, Select } from "./deps.ts";
|
||||
import { open } from 'https://deno.land/x/open/index.ts';
|
||||
|
||||
export async function loginInteractive(remote: string) {
|
||||
let token: string | undefined;
|
||||
if (
|
||||
(await Select.prompt({
|
||||
await Select.prompt({
|
||||
message: "How do you want to login",
|
||||
options: [
|
||||
{
|
||||
@@ -17,7 +17,7 @@ export async function loginInteractive(remote: string) {
|
||||
value: "t",
|
||||
},
|
||||
],
|
||||
})) === "b"
|
||||
}) === "b"
|
||||
) {
|
||||
token = await browserLogin(remote);
|
||||
} else {
|
||||
@@ -29,7 +29,7 @@ export async function loginInteractive(remote: string) {
|
||||
|
||||
// deno-lint-ignore require-await
|
||||
export async function tryGetLoginInfo(
|
||||
opts: GlobalOptions
|
||||
opts: GlobalOptions,
|
||||
): Promise<string | undefined> {
|
||||
if (opts.token) {
|
||||
return opts.token;
|
||||
@@ -39,22 +39,22 @@ export async function tryGetLoginInfo(
|
||||
}
|
||||
|
||||
export async function browserLogin(
|
||||
baseUrl: string
|
||||
baseUrl: string,
|
||||
): Promise<string | undefined> {
|
||||
const port = await getAvailablePort();
|
||||
if (port == undefined) {
|
||||
log.info(colors.red.underline("failed to aquire port"));
|
||||
console.log(colors.red.underline("failed to aquire port"));
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const server = Deno.listen({ transport: "tcp", port });
|
||||
const url = `${baseUrl}user/cli?port=${port}`;
|
||||
log.info(`Login by going to ${url}`);
|
||||
const url = `${baseUrl}user/cli?port=${port}`
|
||||
console.log(`Login by going to ${url}`);
|
||||
try {
|
||||
await open(url);
|
||||
log.info("Opened browser for you");
|
||||
await open(url)
|
||||
console.log("Opened browser for you");
|
||||
} catch {
|
||||
console.error(`Failed to open browser, please navigate to ${url}`);
|
||||
console.error(`Failed to open browser, please navigate to ${url}`)
|
||||
}
|
||||
const firstConnection = await server.accept();
|
||||
const httpFirstConnection = Deno.serveHttp(firstConnection);
|
||||
@@ -63,7 +63,7 @@ export async function browserLogin(
|
||||
const token = params.get("token");
|
||||
// const _workspace = params.get("workspace");
|
||||
await firstRequest?.respondWith(
|
||||
Response.redirect(baseUrl + "user/cli-success", 302)
|
||||
Response.redirect(baseUrl + "user/cli-success", 302),
|
||||
);
|
||||
|
||||
setTimeout(() => {
|
||||
|
||||
34
cli/main.ts
34
cli/main.ts
@@ -18,16 +18,8 @@ import folder from "./folder.ts";
|
||||
import sync from "./sync.ts";
|
||||
import { tryResolveVersion } from "./context.ts";
|
||||
import { GlobalOptions } from "./types.ts";
|
||||
import * as log from "https://deno.land/std@0.186.0/log/mod.ts";
|
||||
|
||||
addEventListener("error", (event) => {
|
||||
if (event.error) {
|
||||
console.error("Error details of: " + event.error.message);
|
||||
console.error(JSON.stringify(event.error, null, 4));
|
||||
}
|
||||
});
|
||||
|
||||
export const VERSION = "v1.100.2";
|
||||
export const VERSION = "v1.87.0";
|
||||
|
||||
let command: any = new Command()
|
||||
.name("wmill")
|
||||
@@ -36,11 +28,6 @@ let command: any = new Command()
|
||||
"--workspace <workspace:string>",
|
||||
"Specify the target workspace. This overrides the default workspace."
|
||||
)
|
||||
.globalOption("--debug", "Show debug logs")
|
||||
.globalOption(
|
||||
"--show-diffs",
|
||||
"Show diff informations when syncing (may show sensitive informations)"
|
||||
)
|
||||
.globalOption(
|
||||
"--token <token:string>",
|
||||
"Specify an API token. This will override any stored token."
|
||||
@@ -86,26 +73,7 @@ if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
|
||||
command = command.command("push", push).command("pull", pull);
|
||||
}
|
||||
|
||||
export let showDiffs = false;
|
||||
try {
|
||||
const LOG_LEVEL = Deno.args.includes("--debug") ? "DEBUG" : "INFO";
|
||||
// const NO_COLORS = Deno.args.includes("--no-colors");
|
||||
showDiffs = Deno.args.includes("--show-diffs");
|
||||
|
||||
log.setup({
|
||||
handlers: {
|
||||
console: new log.handlers.ConsoleHandler(LOG_LEVEL, {
|
||||
formatter: "{msg}",
|
||||
}),
|
||||
},
|
||||
loggers: {
|
||||
default: {
|
||||
level: LOG_LEVEL,
|
||||
handlers: ["console"],
|
||||
},
|
||||
},
|
||||
});
|
||||
log.debug("Debug logging enabled. CLI build against " + VERSION);
|
||||
await command.parse(Deno.args);
|
||||
} catch (e) {
|
||||
if (e.name === "ApiError") {
|
||||
|
||||
@@ -30,7 +30,7 @@ export async function downloadZip(
|
||||
throw new Error(await zipResponse.text());
|
||||
}
|
||||
const blob = await zipResponse.blob();
|
||||
return await JSZip.loadAsync(blob as any);
|
||||
return await JSZip.loadAsync(blob);
|
||||
}
|
||||
|
||||
async function stub(
|
||||
|
||||
11
cli/push.ts
11
cli/push.ts
@@ -2,11 +2,14 @@
|
||||
import { colors, Command } from "./deps.ts";
|
||||
import { GlobalOptions } from "./types.ts";
|
||||
|
||||
async function stub(_opts: GlobalOptions, _dir?: string) {
|
||||
log.info(
|
||||
async function stub(
|
||||
_opts: GlobalOptions,
|
||||
_dir?: string,
|
||||
) {
|
||||
console.log(
|
||||
colors.red.underline(
|
||||
'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.'
|
||||
)
|
||||
'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,65 +1,118 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import {
|
||||
Difference,
|
||||
GlobalOptions,
|
||||
isSuperset,
|
||||
parseFromFile,
|
||||
removeType,
|
||||
PushDiffs,
|
||||
Resource as ResourceI,
|
||||
setValueByPath,
|
||||
} from "./types.ts";
|
||||
import { requireLogin, resolveWorkspace } from "./context.ts";
|
||||
import {
|
||||
colors,
|
||||
Command,
|
||||
log,
|
||||
EditResourceType,
|
||||
microdiff,
|
||||
ResourceService,
|
||||
ResourceType,
|
||||
Table,
|
||||
} from "./deps.ts";
|
||||
import { Any, decoverto, model, property } from "./decoverto.ts";
|
||||
|
||||
export interface ResourceTypeFile {
|
||||
@model()
|
||||
export class ResourceTypeFile implements ResourceI, PushDiffs {
|
||||
@property(Any)
|
||||
schema?: any;
|
||||
@property(() => String)
|
||||
description?: string;
|
||||
|
||||
async push(workspace: string, remotePath: string): Promise<void> {
|
||||
await this.pushDiffs(
|
||||
workspace,
|
||||
remotePath,
|
||||
microdiff({}, this, { cyclesFix: false }),
|
||||
);
|
||||
}
|
||||
|
||||
async pushDiffs(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
diffs: Difference[],
|
||||
): Promise<void> {
|
||||
if (
|
||||
await ResourceService.existsResourceType({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
})
|
||||
) {
|
||||
if (
|
||||
(await ResourceService.listResourceType({ workspace })).findIndex((x) =>
|
||||
x.name === remotePath
|
||||
) === -1
|
||||
) {
|
||||
console.log(
|
||||
"Resource type " + remotePath +
|
||||
" is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
|
||||
);
|
||||
return;
|
||||
}
|
||||
console.log(
|
||||
colors.yellow.bold(
|
||||
`Applying ${diffs.length} diffs to existing resource type... ${remotePath}`,
|
||||
),
|
||||
);
|
||||
const changeset: EditResourceType = {};
|
||||
for (const diff of diffs) {
|
||||
if (
|
||||
diff.type !== "REMOVE" &&
|
||||
(
|
||||
diff.path.length !== 1 ||
|
||||
!["schema", "description"].includes(diff.path[0] as string)
|
||||
)
|
||||
) {
|
||||
throw new Error("Invalid resource type diff with path " + diff.path);
|
||||
}
|
||||
if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
||||
setValueByPath(changeset, diff.path, diff.value);
|
||||
} else if (diff.type === "REMOVE") {
|
||||
setValueByPath(changeset, diff.path, null);
|
||||
}
|
||||
}
|
||||
|
||||
const hasChanges = Object.values(changeset).some((v) =>
|
||||
v !== null && typeof v !== "undefined"
|
||||
);
|
||||
if (!hasChanges) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ResourceService.updateResourceType({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
requestBody: changeset,
|
||||
});
|
||||
} else {
|
||||
console.log(colors.yellow.bold("Creating new resource type..."));
|
||||
await ResourceService.createResourceType({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
name: remotePath,
|
||||
description: this.description,
|
||||
schema: this.schema,
|
||||
workspace_id: workspace,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function pushResourceType(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
resource: ResourceTypeFile | ResourceType | undefined,
|
||||
localResource: ResourceTypeFile,
|
||||
raw: boolean
|
||||
): Promise<void> {
|
||||
remotePath = removeType(remotePath, "resource-type");
|
||||
if (raw) {
|
||||
try {
|
||||
resource = await ResourceService.getResourceType({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
});
|
||||
} catch {
|
||||
// resource type doesn't exist
|
||||
}
|
||||
}
|
||||
if (resource) {
|
||||
if (isSuperset(localResource, resource)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ResourceService.updateResourceType({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
requestBody: {
|
||||
...localResource,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
log.info(colors.yellow.bold("Creating new resource type..."));
|
||||
await ResourceService.createResourceType({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
name: remotePath,
|
||||
...localResource,
|
||||
},
|
||||
});
|
||||
}
|
||||
filePath: string,
|
||||
name: string,
|
||||
) {
|
||||
const data: ResourceTypeFile = decoverto.type(ResourceTypeFile).rawToInstance(
|
||||
await Deno.readTextFile(filePath),
|
||||
);
|
||||
await data.push(workspace, name);
|
||||
}
|
||||
|
||||
type PushOptions = GlobalOptions;
|
||||
@@ -71,16 +124,10 @@ async function push(opts: PushOptions, filePath: string, name: string) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
log.info(colors.bold.yellow("Pushing resource..."));
|
||||
console.log(colors.bold.yellow("Pushing resource..."));
|
||||
|
||||
await pushResourceType(
|
||||
workspace.workspaceId,
|
||||
name,
|
||||
undefined,
|
||||
parseFromFile(filePath),
|
||||
true
|
||||
);
|
||||
log.info(colors.bold.underline.green("Resource pushed"));
|
||||
await pushResourceType(workspace.workspaceId, filePath, name);
|
||||
console.log(colors.bold.underline.green("Resource pushed"));
|
||||
}
|
||||
|
||||
async function list(opts: GlobalOptions) {
|
||||
@@ -103,7 +150,7 @@ const command = new Command()
|
||||
.action(list as any)
|
||||
.command(
|
||||
"push",
|
||||
"push a local resource spec. This overrides any remote versions."
|
||||
"push a local resource spec. This overrides any remote versions.",
|
||||
)
|
||||
.arguments("<file_path:string> <name:string>")
|
||||
.action(push as any);
|
||||
|
||||
171
cli/resource.ts
171
cli/resource.ts
@@ -1,73 +1,130 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import {
|
||||
Difference,
|
||||
GlobalOptions,
|
||||
isSuperset,
|
||||
parseFromFile,
|
||||
removeType,
|
||||
PushDiffs,
|
||||
Resource as Resource2,
|
||||
setValueByPath,
|
||||
} from "./types.ts";
|
||||
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
|
||||
import {
|
||||
colors,
|
||||
Command,
|
||||
log,
|
||||
EditResource,
|
||||
microdiff,
|
||||
Resource,
|
||||
ResourceService,
|
||||
Table,
|
||||
} from "./deps.ts";
|
||||
import { Any, decoverto, model, property } from "./decoverto.ts";
|
||||
|
||||
export interface ResourceFile {
|
||||
value: any;
|
||||
@model()
|
||||
export class ResourceFile implements Resource2, PushDiffs {
|
||||
@property(Any)
|
||||
value?: any;
|
||||
@property(() => String)
|
||||
description?: string;
|
||||
@property(() => String)
|
||||
resource_type: string;
|
||||
@property(() => Boolean)
|
||||
is_oauth?: boolean; // deprecated
|
||||
|
||||
constructor(resource_type: string) {
|
||||
this.resource_type = resource_type;
|
||||
}
|
||||
async pushDiffs(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
diffs: Difference[]
|
||||
): Promise<void> {
|
||||
if (
|
||||
await ResourceService.existsResource({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
})
|
||||
) {
|
||||
console.log(
|
||||
colors.yellow.bold(
|
||||
`Applying ${diffs.length} diffs to existing resource... ${remotePath}`
|
||||
)
|
||||
);
|
||||
|
||||
const changeset: EditResource = {
|
||||
path: remotePath, // TODO: Remove this in backend
|
||||
};
|
||||
for (const diff of diffs) {
|
||||
if (diff.path[0] === "is_oauth") {
|
||||
//is_oauth is not updatable
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
diff.type !== "REMOVE" &&
|
||||
diff.path[0] !== "value" &&
|
||||
(diff.path.length !== 1 || diff.path[0] !== "description") &&
|
||||
diff.path[0] !== "resource_type"
|
||||
) {
|
||||
console.log(
|
||||
colors.red("Invalid variable diff with path " + diff.path)
|
||||
);
|
||||
throw new Error("Invalid folder diff with path " + diff.path);
|
||||
}
|
||||
if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
||||
setValueByPath(changeset, diff.path, diff.value);
|
||||
} else if (diff.type === "REMOVE") {
|
||||
setValueByPath(changeset, diff.path, null);
|
||||
}
|
||||
}
|
||||
|
||||
const hasChanges = Object.values(changeset).some(
|
||||
(v) => v !== null && typeof v !== "undefined"
|
||||
);
|
||||
if (!hasChanges) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ResourceService.updateResource({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
requestBody: changeset,
|
||||
});
|
||||
} else {
|
||||
if (typeof this.is_oauth !== "undefined") {
|
||||
console.log(
|
||||
colors.yellow(
|
||||
"! is_oauth has been removed in newer versions. Ignoring."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
console.log(colors.yellow.bold("Creating new resource..."));
|
||||
await ResourceService.createResource({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
resource_type: this.resource_type,
|
||||
value: this.value,
|
||||
description: this.description,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
async push(workspace: string, remotePath: string): Promise<void> {
|
||||
await this.pushDiffs(
|
||||
workspace,
|
||||
remotePath,
|
||||
microdiff({}, this, { cyclesFix: false })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function pushResource(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
resource: ResourceFile | Resource | undefined,
|
||||
localResource: ResourceFile,
|
||||
raw: boolean
|
||||
): Promise<void> {
|
||||
remotePath = removeType(remotePath, "resource");
|
||||
if (raw) {
|
||||
try {
|
||||
resource = await ResourceService.getResource({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
});
|
||||
} catch {
|
||||
// flow doesn't exist
|
||||
}
|
||||
}
|
||||
if (resource) {
|
||||
if (isSuperset(localResource, resource)) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ResourceService.updateResource({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
requestBody: { ...localResource },
|
||||
});
|
||||
} else {
|
||||
if (localResource.is_oauth) {
|
||||
log.info(
|
||||
colors.yellow(
|
||||
"! is_oauth has been removed in newer versions. Ignoring."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
log.info(colors.yellow.bold("Creating new resource..."));
|
||||
await ResourceService.createResource({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
...localResource,
|
||||
},
|
||||
});
|
||||
}
|
||||
filePath: string,
|
||||
remotePath: string
|
||||
) {
|
||||
const data = decoverto
|
||||
.type(ResourceFile)
|
||||
.rawToInstance(await Deno.readTextFile(filePath));
|
||||
await data.push(workspace, remotePath);
|
||||
}
|
||||
|
||||
type PushOptions = GlobalOptions;
|
||||
@@ -84,16 +141,10 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
|
||||
throw new Error("file path must refer to a file.");
|
||||
}
|
||||
|
||||
log.info(colors.bold.yellow("Pushing resource..."));
|
||||
console.log(colors.bold.yellow("Pushing resource..."));
|
||||
|
||||
await pushResource(
|
||||
workspace.workspaceId,
|
||||
remotePath,
|
||||
undefined,
|
||||
parseFromFile(filePath),
|
||||
true
|
||||
);
|
||||
log.info(colors.bold.underline.green(`Resource ${remotePath} pushed`));
|
||||
await pushResource(workspace.workspaceId, filePath, remotePath);
|
||||
console.log(colors.bold.underline.green(`Resource ${remotePath} pushed`));
|
||||
}
|
||||
|
||||
async function list(opts: GlobalOptions) {
|
||||
|
||||
224
cli/script.ts
224
cli/script.ts
@@ -5,24 +5,53 @@ import {
|
||||
colors,
|
||||
Command,
|
||||
JobService,
|
||||
log,
|
||||
NewScript,
|
||||
readAll,
|
||||
Script,
|
||||
ScriptService,
|
||||
Table,
|
||||
} from "./deps.ts";
|
||||
import { Any, array, decoverto, model, property } from "./decoverto.ts";
|
||||
import { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts";
|
||||
import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";
|
||||
|
||||
export interface ScriptFile {
|
||||
@model()
|
||||
export class ScriptFile {
|
||||
@property(() => String)
|
||||
parent_hash?: string;
|
||||
@property(() => String)
|
||||
summary: string;
|
||||
@property(() => String)
|
||||
description: string;
|
||||
@property(Any)
|
||||
schema?: any;
|
||||
@property(() => Boolean)
|
||||
is_template?: boolean;
|
||||
@property(array(() => String))
|
||||
lock?: Array<string>;
|
||||
@property({
|
||||
toInstance: (data) => {
|
||||
if (data == null) return data;
|
||||
|
||||
if (
|
||||
data === "script" ||
|
||||
data === "failure" ||
|
||||
data === "trigger" ||
|
||||
data === "command" ||
|
||||
data === "approvial"
|
||||
) {
|
||||
return data;
|
||||
}
|
||||
|
||||
throw new Error("Invalid kind " + data);
|
||||
},
|
||||
toPlain: (data) => data,
|
||||
})
|
||||
@property(() => String)
|
||||
kind?: "script" | "failure" | "trigger" | "command" | "approval";
|
||||
|
||||
constructor(summary: string, description: string) {
|
||||
this.summary = summary;
|
||||
this.description = description;
|
||||
}
|
||||
}
|
||||
|
||||
type PushOptions = GlobalOptions;
|
||||
@@ -38,16 +67,19 @@ async function push(opts: PushOptions, filePath: string) {
|
||||
if (!fstat.isFile) {
|
||||
throw new Error("file path must refer to a file.");
|
||||
}
|
||||
|
||||
if (filePath.endsWith(".script.json") || filePath.endsWith(".script.yaml")) {
|
||||
throw Error(
|
||||
"Cannot push a script metadata file, point to the script content file instead (.py, .ts, .go|.sh)"
|
||||
);
|
||||
let contentPath: string;
|
||||
let metaPath: string | undefined;
|
||||
if (filePath.endsWith(".script.json")) {
|
||||
metaPath = filePath;
|
||||
contentPath = await findContentFile(filePath);
|
||||
} else {
|
||||
contentPath = filePath;
|
||||
metaPath = undefined;
|
||||
}
|
||||
|
||||
await requireLogin(opts);
|
||||
await handleFile(filePath, workspace.workspaceId, []);
|
||||
log.info(colors.bold.underline.green(`Script ${remotePath} pushed`));
|
||||
await pushScript(metaPath, contentPath, workspace.workspaceId, remotePath);
|
||||
console.log(colors.bold.underline.green(`Script ${remotePath} pushed`));
|
||||
}
|
||||
|
||||
export async function handleScriptMetadata(
|
||||
@@ -55,9 +87,14 @@ export async function handleScriptMetadata(
|
||||
workspace: string,
|
||||
alreadySynced: string[]
|
||||
): Promise<boolean> {
|
||||
if (path.endsWith(".script.json") || path.endsWith(".script.yaml")) {
|
||||
if (path.endsWith(".script.json")) {
|
||||
const contentPath = await findContentFile(path);
|
||||
return handleFile(contentPath, workspace, alreadySynced);
|
||||
return handleFile(
|
||||
contentPath,
|
||||
await Deno.readTextFile(contentPath),
|
||||
workspace,
|
||||
alreadySynced
|
||||
);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
@@ -65,21 +102,19 @@ export async function handleScriptMetadata(
|
||||
|
||||
export async function handleFile(
|
||||
path: string,
|
||||
content: string,
|
||||
workspace: string,
|
||||
alreadySynced: string[]
|
||||
): Promise<boolean> {
|
||||
if (
|
||||
!path.includes(".inline_script.") &&
|
||||
(path.endsWith(".ts") ||
|
||||
path.endsWith(".py") ||
|
||||
path.endsWith(".go") ||
|
||||
path.endsWith(".sh"))
|
||||
path.endsWith(".ts") ||
|
||||
path.endsWith(".py") ||
|
||||
path.endsWith(".go") ||
|
||||
path.endsWith(".sh")
|
||||
) {
|
||||
if (alreadySynced.includes(path)) {
|
||||
return true;
|
||||
}
|
||||
log.debug(`Processing local script ${path}`);
|
||||
|
||||
alreadySynced.push(path);
|
||||
const remotePath = path.substring(0, path.length - 3);
|
||||
const metaPath = remotePath + ".script.json";
|
||||
@@ -87,16 +122,8 @@ export async function handleFile(
|
||||
try {
|
||||
await Deno.stat(metaPath);
|
||||
typed = JSON.parse(await Deno.readTextFile(metaPath));
|
||||
} catch {
|
||||
const metaPath = remotePath + ".script.yaml";
|
||||
try {
|
||||
await Deno.stat(metaPath);
|
||||
typed = yamlParse(await Deno.readTextFile(metaPath));
|
||||
} catch {
|
||||
// no meta file
|
||||
}
|
||||
}
|
||||
|
||||
typed = decoverto.type(ScriptFile).plainToInstance(typed);
|
||||
} catch {}
|
||||
const language = inferContentTypeFromFilePath(path);
|
||||
|
||||
let remote = undefined;
|
||||
@@ -105,11 +132,7 @@ export async function handleFile(
|
||||
workspace,
|
||||
path: remotePath,
|
||||
});
|
||||
log.debug(`Script ${remotePath} exists on remote`);
|
||||
} catch {
|
||||
log.debug(`Script ${remotePath} does not exist on remote`);
|
||||
}
|
||||
const content = await Deno.readTextFile(path);
|
||||
} catch {}
|
||||
|
||||
if (remote) {
|
||||
if (content === remote.content) {
|
||||
@@ -123,7 +146,7 @@ export async function handleFile(
|
||||
remote?.lock == typed.lock?.join("\n") &&
|
||||
JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
|
||||
) {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.yellow(
|
||||
`No change to push for script ${remotePath}, skipping`
|
||||
)
|
||||
@@ -131,15 +154,13 @@ export async function handleFile(
|
||||
return true;
|
||||
}
|
||||
}
|
||||
log.info(
|
||||
colors.yellow.bold(`Creating script with a parent ${remotePath}`)
|
||||
);
|
||||
|
||||
await ScriptService.createScript({
|
||||
workspace,
|
||||
requestBody: {
|
||||
content,
|
||||
description: typed?.description ?? "",
|
||||
language: language as NewScript.language,
|
||||
language,
|
||||
path: remotePath,
|
||||
summary: typed?.summary ?? "",
|
||||
is_template: typed?.is_template,
|
||||
@@ -149,17 +170,18 @@ export async function handleFile(
|
||||
schema: typed?.schema,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
log.info(
|
||||
colors.yellow.bold(`Creating script without parent ${remotePath}`)
|
||||
|
||||
console.log(
|
||||
colors.yellow.bold(`Creating script with a parent ${remotePath}`)
|
||||
);
|
||||
} else {
|
||||
// no parent hash
|
||||
await ScriptService.createScript({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
content,
|
||||
description: typed?.description ?? "",
|
||||
language: language as NewScript.language,
|
||||
language,
|
||||
path: remotePath,
|
||||
summary: typed?.summary ?? "",
|
||||
is_template: typed?.is_template,
|
||||
@@ -169,6 +191,9 @@ export async function handleFile(
|
||||
schema: typed?.schema,
|
||||
},
|
||||
});
|
||||
console.log(
|
||||
colors.yellow.bold(`Creating script without parent ${remotePath}`)
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -176,19 +201,12 @@ export async function handleFile(
|
||||
}
|
||||
|
||||
export async function findContentFile(filePath: string) {
|
||||
const candidates = filePath.endsWith("script.json")
|
||||
? [
|
||||
filePath.replace(".script.json", ".ts"),
|
||||
filePath.replace(".script.json", ".py"),
|
||||
filePath.replace(".script.json", ".go"),
|
||||
filePath.replace(".script.json", ".sh"),
|
||||
]
|
||||
: [
|
||||
filePath.replace(".script.yaml", ".ts"),
|
||||
filePath.replace(".script.yaml", ".py"),
|
||||
filePath.replace(".script.yaml", ".go"),
|
||||
filePath.replace(".script.yaml", ".sh"),
|
||||
];
|
||||
const candidates = [
|
||||
filePath.replace(".script.json", ".ts"),
|
||||
filePath.replace(".script.json", ".py"),
|
||||
filePath.replace(".script.json", ".go"),
|
||||
filePath.replace(".script.json", ".sh"),
|
||||
];
|
||||
const validCandidates = (
|
||||
await Promise.all(
|
||||
candidates.map((x) => {
|
||||
@@ -205,7 +223,7 @@ export async function findContentFile(filePath: string) {
|
||||
.map((x) => x.path);
|
||||
if (validCandidates.length > 1) {
|
||||
throw new Error(
|
||||
"No content path given and more than one candidate found: " +
|
||||
"No content path given and more then one candidate found: " +
|
||||
validCandidates.join(", ")
|
||||
);
|
||||
}
|
||||
@@ -234,6 +252,52 @@ export function inferContentTypeFromFilePath(
|
||||
return language;
|
||||
}
|
||||
|
||||
export async function pushScript(
|
||||
filePath: string | undefined,
|
||||
contentPath: string,
|
||||
workspace: string,
|
||||
remotePath: string
|
||||
) {
|
||||
const data = filePath
|
||||
? decoverto
|
||||
.type(ScriptFile)
|
||||
.rawToInstance(await Deno.readTextFile(filePath))
|
||||
: undefined;
|
||||
const content = await Deno.readTextFile(contentPath);
|
||||
|
||||
const language = inferContentTypeFromFilePath(contentPath);
|
||||
let parent_hash = data?.parent_hash;
|
||||
if (!parent_hash) {
|
||||
try {
|
||||
parent_hash = (
|
||||
await ScriptService.getScriptByPath({
|
||||
workspace: workspace,
|
||||
path: remotePath,
|
||||
})
|
||||
).hash;
|
||||
} catch {
|
||||
/* no parent. New Script. */
|
||||
}
|
||||
}
|
||||
|
||||
console.log(colors.bold.yellow("Pushing script..."));
|
||||
await ScriptService.createScript({
|
||||
workspace: workspace,
|
||||
requestBody: {
|
||||
path: remotePath,
|
||||
summary: data?.summary ?? "",
|
||||
content: content,
|
||||
description: data?.description ?? "",
|
||||
language: language,
|
||||
is_template: data?.is_template,
|
||||
kind: data?.kind,
|
||||
lock: data?.lock,
|
||||
parent_hash: parent_hash,
|
||||
schema: data?.schema,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function list(opts: GlobalOptions & { showArchived?: boolean }) {
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
@@ -312,7 +376,7 @@ async function run(
|
||||
id,
|
||||
})
|
||||
).result ?? {};
|
||||
log.info(result);
|
||||
console.log(result);
|
||||
|
||||
break;
|
||||
} catch {
|
||||
@@ -325,16 +389,16 @@ export async function track_job(workspace: string, id: string) {
|
||||
try {
|
||||
const result = await JobService.getCompletedJob({ workspace, id });
|
||||
|
||||
log.info(result.logs);
|
||||
log.info("\n");
|
||||
log.info(colors.bold.underline.green("Job Completed"));
|
||||
log.info("\n");
|
||||
console.log(result.logs);
|
||||
console.log();
|
||||
console.log(colors.bold.underline.green("Job Completed"));
|
||||
console.log();
|
||||
return;
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
log.info(colors.yellow("Waiting for Job " + id + " to start..."));
|
||||
console.log(colors.yellow("Waiting for Job " + id + " to start..."));
|
||||
|
||||
let logOffset = 0;
|
||||
let running = false;
|
||||
@@ -355,7 +419,7 @@ export async function track_job(workspace: string, id: string) {
|
||||
} catch {
|
||||
retry++;
|
||||
if (retry > 3) {
|
||||
log.info("failed to get job updated. skipping log streaming.");
|
||||
console.log("failed to get job updated. skipping log streaming.");
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
@@ -363,7 +427,7 @@ export async function track_job(workspace: string, id: string) {
|
||||
|
||||
if (!running && updates.running === true) {
|
||||
running = true;
|
||||
log.info(colors.green("Job running. Streaming logs..."));
|
||||
console.log(colors.green("Job running. Streaming logs..."));
|
||||
}
|
||||
|
||||
if (updates.new_logs) {
|
||||
@@ -378,7 +442,9 @@ export async function track_job(workspace: string, id: string) {
|
||||
|
||||
if (running && updates.running === false) {
|
||||
running = false;
|
||||
log.info(colors.yellow("Job suspended. Waiting for it to continue..."));
|
||||
console.log(
|
||||
colors.yellow("Job suspended. Waiting for it to continue...")
|
||||
);
|
||||
}
|
||||
}
|
||||
await new Promise((resolve, _) => setTimeout(() => resolve(undefined), 1000));
|
||||
@@ -386,17 +452,17 @@ export async function track_job(workspace: string, id: string) {
|
||||
try {
|
||||
const final_job = await JobService.getCompletedJob({ workspace, id });
|
||||
if ((final_job.logs?.length ?? -1) > logOffset) {
|
||||
log.info(final_job.logs!.substring(logOffset));
|
||||
console.log(final_job.logs!.substring(logOffset));
|
||||
}
|
||||
log.info("\n");
|
||||
console.log("\n");
|
||||
if (final_job.success) {
|
||||
log.info(colors.bold.underline.green("Job Completed"));
|
||||
console.log(colors.bold.underline.green("Job Completed"));
|
||||
} else {
|
||||
log.info(colors.bold.underline.red("Job Completed"));
|
||||
console.log(colors.bold.underline.red("Job Completed"));
|
||||
}
|
||||
log.info("\n");
|
||||
console.log();
|
||||
} catch {
|
||||
log.info("Job appears to have completed, but no data can be retrieved");
|
||||
console.log("Job appears to have completed, but no data can be retrieved");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -407,10 +473,10 @@ async function show(opts: GlobalOptions, path: string) {
|
||||
workspace: workspace.workspaceId,
|
||||
path,
|
||||
});
|
||||
log.info(colors.underline(s.path));
|
||||
if (s.description) log.info(s.description);
|
||||
log.info("");
|
||||
log.info(s.content);
|
||||
console.log(colors.underline(s.path));
|
||||
if (s.description) console.log(s.description);
|
||||
console.log("");
|
||||
console.log(s.content);
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
@@ -419,9 +485,9 @@ const command = new Command()
|
||||
.action(list as any)
|
||||
.command(
|
||||
"push",
|
||||
"push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh)"
|
||||
"push a local script spec. This overrides any remote versions. Can use a script file (.ts, .js, .py, .sh) or a script spec file (.json). "
|
||||
)
|
||||
.arguments("<path:file>")
|
||||
.arguments("<file_path:file>")
|
||||
.action(push as any)
|
||||
.command("show", "show a scripts content")
|
||||
.arguments("<path:file>")
|
||||
|
||||
450
cli/sync.ts
450
cli/sync.ts
@@ -6,6 +6,7 @@ import {
|
||||
ensureDir,
|
||||
gitignore_parser,
|
||||
JSZip,
|
||||
microdiff,
|
||||
path,
|
||||
ScriptService,
|
||||
FolderService,
|
||||
@@ -13,34 +14,29 @@ import {
|
||||
VariableService,
|
||||
AppService,
|
||||
FlowService,
|
||||
OpenFlow,
|
||||
FlowModule,
|
||||
RawScript,
|
||||
log,
|
||||
} from "./deps.ts";
|
||||
import {
|
||||
Difference,
|
||||
getTypeStrFromPath,
|
||||
GlobalOptions,
|
||||
parseFromPath,
|
||||
pushObj,
|
||||
showConflict,
|
||||
showDiff,
|
||||
inferTypeFromPath,
|
||||
setValueByPath,
|
||||
} from "./types.ts";
|
||||
import { downloadZip } from "./pull.ts";
|
||||
|
||||
import { handleScriptMetadata } from "./script.ts";
|
||||
|
||||
import { FolderFile } from "./folder.ts";
|
||||
import { ResourceTypeFile } from "./resource-type.ts";
|
||||
import { handleScriptMetadata, ScriptFile } from "./script.ts";
|
||||
import { ResourceFile } from "./resource.ts";
|
||||
import { FlowFile } from "./flow.ts";
|
||||
import { VariableFile } from "./variable.ts";
|
||||
import { handleFile } from "./script.ts";
|
||||
import { equal } from "https://deno.land/x/equal@v1.5.0/mod.ts";
|
||||
import {
|
||||
stringify as yamlStringify,
|
||||
parse as yamlParse,
|
||||
} from "https://deno.land/std@0.184.0/yaml/mod.ts";
|
||||
import * as Diff from "npm:diff";
|
||||
|
||||
type DynFSElement = {
|
||||
isDirectory: boolean;
|
||||
path: string;
|
||||
// getContentBytes(): Promise<Uint8Array>;
|
||||
getContentBytes(): Promise<Uint8Array>;
|
||||
getContentText(): Promise<string>;
|
||||
getChildren(): AsyncIterable<DynFSElement>;
|
||||
};
|
||||
@@ -51,152 +47,35 @@ async function FSFSElement(p: string): Promise<DynFSElement> {
|
||||
isDirectory: isDir,
|
||||
path: localP.substring(p.length + 1),
|
||||
async *getChildren(): AsyncIterable<DynFSElement> {
|
||||
if (!isDir) return [];
|
||||
for await (const e of Deno.readDir(localP)) {
|
||||
yield _internal_element(path.join(localP, e.name), e.isDirectory);
|
||||
}
|
||||
},
|
||||
// async getContentBytes(): Promise<Uint8Array> {
|
||||
// return await Deno.readFile(localP);
|
||||
// },
|
||||
async getContentBytes(): Promise<Uint8Array> {
|
||||
return await Deno.readFile(localP);
|
||||
},
|
||||
async getContentText(): Promise<string> {
|
||||
const content = await Deno.readTextFile(localP);
|
||||
return content;
|
||||
return await Deno.readTextFile(localP);
|
||||
},
|
||||
};
|
||||
}
|
||||
return _internal_element(p, (await Deno.stat(p)).isDirectory);
|
||||
}
|
||||
|
||||
function prioritizeName(name: string): string {
|
||||
if (name == "id") return "aa";
|
||||
if (name == "type") return "ab";
|
||||
if (name == "summary") return "ad";
|
||||
if (name == "name") return "ae";
|
||||
if (name == "display_name") return "af";
|
||||
if (name == "description") return "ag";
|
||||
if (name == "value") return "ah";
|
||||
if (name == "content") return "ai";
|
||||
if (name == "modules") return "aj";
|
||||
if (name == "failure_module") return "ak";
|
||||
if (name == "input_transforms") return "al";
|
||||
if (name == "lock") return "az";
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
export const yamlOptions = {
|
||||
sortKeys: (a: any, b: any) => {
|
||||
return prioritizeName(a).localeCompare(prioritizeName(b));
|
||||
},
|
||||
noCompatMode: true,
|
||||
noRefs: true,
|
||||
};
|
||||
|
||||
function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
|
||||
function ZipFSElement(zip: JSZip): DynFSElement {
|
||||
function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement {
|
||||
const isFlow = p.endsWith("flow.json");
|
||||
function transformPath() {
|
||||
if (isFlow) {
|
||||
return p.replace("flow.json", "flow");
|
||||
} else {
|
||||
return useYaml && p.endsWith(".json")
|
||||
? p.replaceAll(".json", ".yaml")
|
||||
: p;
|
||||
}
|
||||
}
|
||||
|
||||
interface InlineScript {
|
||||
path: string;
|
||||
content: string;
|
||||
}
|
||||
let counter = 0;
|
||||
const seen_names = new Set<string>();
|
||||
function assignPath(
|
||||
summary: string | undefined,
|
||||
language: RawScript.language
|
||||
): string {
|
||||
let name;
|
||||
if (summary && summary != "" && !seen_names.has(summary)) {
|
||||
name = summary.toLowerCase().replaceAll(" ", "_");
|
||||
seen_names.add(name);
|
||||
} else {
|
||||
name = `inline_script_${counter}`;
|
||||
while (seen_names.has(name)) {
|
||||
counter++;
|
||||
name = `inline_script_${counter}`;
|
||||
}
|
||||
seen_names.add(name);
|
||||
}
|
||||
let ext;
|
||||
if (language == "python3") ext = "py";
|
||||
else if (language == "deno") ext = "ts";
|
||||
else if (language == "go") ext = "go";
|
||||
else if (language == "bash") ext = "sh";
|
||||
return `${name}.inline_script.${ext}`;
|
||||
}
|
||||
function extractInlineScripts(modules: FlowModule[]): InlineScript[] {
|
||||
return modules.flatMap((m) => {
|
||||
if (m.value.type == "rawscript") {
|
||||
const path = assignPath(m.summary, m.value.language);
|
||||
const content = m.value.content;
|
||||
m.value.content = "!inline " + path;
|
||||
return [{ path: path, content: content }];
|
||||
} else if (m.value.type == "forloopflow") {
|
||||
return extractInlineScripts(m.value.modules);
|
||||
} else if (m.value.type == "branchall") {
|
||||
return m.value.branches.flatMap((b) =>
|
||||
extractInlineScripts(b.modules)
|
||||
);
|
||||
} else if (m.value.type == "branchone") {
|
||||
return [
|
||||
...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)),
|
||||
...extractInlineScripts(m.value.default),
|
||||
];
|
||||
} else {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
const flowPath = transformPath();
|
||||
return {
|
||||
isDirectory: isFlow,
|
||||
path: flowPath,
|
||||
isDirectory: false,
|
||||
path: p,
|
||||
// deno-lint-ignore require-yield
|
||||
async *getChildren(): AsyncIterable<DynFSElement> {
|
||||
if (isFlow) {
|
||||
const flow: OpenFlow = JSON.parse(await f.async("text"));
|
||||
const inlineScripts = extractInlineScripts(flow.value.modules);
|
||||
for (const s of inlineScripts) {
|
||||
yield {
|
||||
isDirectory: false,
|
||||
path: path.join(flowPath, s.path),
|
||||
async *getChildren() {},
|
||||
// deno-lint-ignore require-await
|
||||
async getContentText() {
|
||||
return s.content;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
yield {
|
||||
isDirectory: false,
|
||||
path: path.join(flowPath, "flow.yaml"),
|
||||
async *getChildren() {},
|
||||
// deno-lint-ignore require-await
|
||||
async getContentText() {
|
||||
return yamlStringify(flow, yamlOptions);
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error("Cannot get children of file");
|
||||
},
|
||||
async getContentBytes(): Promise<Uint8Array> {
|
||||
return await f.async("uint8array");
|
||||
},
|
||||
// async getContentBytes(): Promise<Uint8Array> {
|
||||
// return await f.async("uint8array");
|
||||
// },
|
||||
async getContentText(): Promise<string> {
|
||||
const content = await f.async("text");
|
||||
return useYaml && p.endsWith(".json")
|
||||
? yamlStringify(JSON.parse(content), yamlOptions)
|
||||
: content;
|
||||
return await f.async("text");
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -216,11 +95,9 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
|
||||
}
|
||||
}
|
||||
},
|
||||
// // deno-lint-ignore require-await
|
||||
// async getContentBytes(): Promise<Uint8Array> {
|
||||
// throw new Error("Cannot get content of folder");
|
||||
// },
|
||||
// deno-lint-ignore require-await
|
||||
async getContentBytes(): Promise<Uint8Array> {
|
||||
throw new Error("Cannot get content of folder");
|
||||
},
|
||||
async getContentText(): Promise<string> {
|
||||
throw new Error("Cannot get content of folder");
|
||||
},
|
||||
@@ -236,7 +113,7 @@ async function* readDirRecursiveWithIgnore(
|
||||
path: string;
|
||||
ignored: boolean;
|
||||
isDirectory: boolean;
|
||||
// getContentBytes(): Promise<Uint8Array>;
|
||||
getContentBytes(): Promise<Uint8Array>;
|
||||
getContentText(): Promise<string>;
|
||||
}> {
|
||||
const stack: {
|
||||
@@ -244,7 +121,7 @@ async function* readDirRecursiveWithIgnore(
|
||||
isDirectory: boolean;
|
||||
ignored: boolean;
|
||||
c(): AsyncIterable<DynFSElement>;
|
||||
// getContentBytes(): Promise<Uint8Array>;
|
||||
getContentBytes(): Promise<Uint8Array>;
|
||||
getContentText(): Promise<string>;
|
||||
}[] = [
|
||||
{
|
||||
@@ -252,9 +129,9 @@ async function* readDirRecursiveWithIgnore(
|
||||
ignored: ignore(root.path, root.isDirectory),
|
||||
isDirectory: root.isDirectory,
|
||||
c: root.getChildren,
|
||||
// getContentBytes(): Promise<Uint8Array> {
|
||||
// throw undefined;
|
||||
// },
|
||||
getContentBytes(): Promise<Uint8Array> {
|
||||
throw undefined;
|
||||
},
|
||||
getContentText(): Promise<string> {
|
||||
throw undefined;
|
||||
},
|
||||
@@ -264,12 +141,13 @@ async function* readDirRecursiveWithIgnore(
|
||||
while (stack.length > 0) {
|
||||
const e = stack.pop()!;
|
||||
yield e;
|
||||
if (!e.isDirectory) continue;
|
||||
for await (const e2 of e.c()) {
|
||||
stack.push({
|
||||
path: e2.path,
|
||||
ignored: e.ignored || ignore(e2.path, e2.isDirectory),
|
||||
isDirectory: e2.isDirectory,
|
||||
// getContentBytes: e2.getContentBytes,
|
||||
getContentBytes: e2.getContentBytes,
|
||||
getContentText: e2.getContentText,
|
||||
c: e2.getChildren,
|
||||
});
|
||||
@@ -285,38 +163,27 @@ type Change = Added | Deleted | Edit;
|
||||
|
||||
async function elementsToMap(
|
||||
els: DynFSElement,
|
||||
ignore: (path: string, isDirectory: boolean) => boolean,
|
||||
json: boolean
|
||||
ignore: (path: string, isDirectory: boolean) => boolean
|
||||
): Promise<{ [key: string]: string }> {
|
||||
const map: { [key: string]: string } = {};
|
||||
for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
|
||||
if (entry.isDirectory || entry.ignored) continue;
|
||||
if (json && entry.path.endsWith(".yaml")) continue;
|
||||
if (!json && entry.path.endsWith(".json")) continue;
|
||||
if (
|
||||
!["json", "yaml", "go", "sh", "ts", "py"].includes(
|
||||
entry.path.split(".").pop() ?? ""
|
||||
)
|
||||
)
|
||||
continue;
|
||||
const content = await entry.getContentText();
|
||||
map[entry.path] = content;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async function compareDynFSElement(
|
||||
els1: DynFSElement,
|
||||
els2: DynFSElement | undefined,
|
||||
ignore: (path: string, isDirectory: boolean) => boolean,
|
||||
json: boolean
|
||||
ignore: (path: string, isDirectory: boolean) => boolean
|
||||
): Promise<Change[]> {
|
||||
const [m1, m2] = els2
|
||||
? await Promise.all([
|
||||
elementsToMap(els1, ignore, json),
|
||||
elementsToMap(els2, ignore, json),
|
||||
elementsToMap(els1, ignore),
|
||||
elementsToMap(els2, ignore),
|
||||
])
|
||||
: [await elementsToMap(els1, ignore, json), {}];
|
||||
: [await elementsToMap(els1, ignore), {}];
|
||||
|
||||
const changes: Change[] = [];
|
||||
|
||||
@@ -325,9 +192,13 @@ async function compareDynFSElement(
|
||||
changes.push({ name: "added", path: k, content: v });
|
||||
} else if (
|
||||
m2[k] != v &&
|
||||
(!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k]))) &&
|
||||
(!k.endsWith(".yaml") || !equal(yamlParse(v), yamlParse(m2[k])))
|
||||
(!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k])))
|
||||
) {
|
||||
// await Deno.writeTextFile("/tmp/k", m2[k])
|
||||
// await Deno.writeTextFile("/tmp/v", v)
|
||||
// console.log(k)
|
||||
// if (k.includes("flow"))
|
||||
// Deno.exit(1)
|
||||
changes.push({ name: "edited", path: k, after: v, before: m2[k] });
|
||||
}
|
||||
}
|
||||
@@ -377,7 +248,7 @@ async function ignoreF() {
|
||||
(isNotWmillFile(p, isDirectory) || ignore.denies(p))
|
||||
);
|
||||
};
|
||||
} catch {
|
||||
} catch (e) {
|
||||
return (p: string, isDirectory: boolean) =>
|
||||
!isWhitelisted(p) && isNotWmillFile(p, isDirectory);
|
||||
}
|
||||
@@ -389,7 +260,6 @@ async function pull(
|
||||
yes: boolean;
|
||||
failConflicts: boolean;
|
||||
plainSecrets?: boolean;
|
||||
json?: boolean;
|
||||
}
|
||||
) {
|
||||
if (!opts.raw) {
|
||||
@@ -399,28 +269,20 @@ async function pull(
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
log.info(
|
||||
console.log(
|
||||
colors.gray(
|
||||
"Computing the files to update locally to match remote (taking .wmillignore into account)"
|
||||
)
|
||||
);
|
||||
const remote = ZipFSElement(
|
||||
(await downloadZip(workspace, opts.plainSecrets))!,
|
||||
!opts.json
|
||||
(await downloadZip(workspace, opts.plainSecrets))!
|
||||
);
|
||||
const local = opts.raw
|
||||
? undefined
|
||||
: await FSFSElement(path.join(Deno.cwd(), ".wmill"));
|
||||
const changes = await compareDynFSElement(
|
||||
remote,
|
||||
local,
|
||||
await ignoreF(),
|
||||
opts.json ?? false
|
||||
);
|
||||
: await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill"));
|
||||
const changes = await compareDynFSElement(remote, local, await ignoreF());
|
||||
|
||||
log.info(
|
||||
`remote (${workspace.name}) -> local: ${changes.length} changes to apply`
|
||||
);
|
||||
console.log(`remote -> local: ${changes.length} changes to apply`);
|
||||
if (changes.length > 0) {
|
||||
prettyChanges(changes);
|
||||
if (
|
||||
@@ -435,7 +297,7 @@ async function pull(
|
||||
}
|
||||
|
||||
const conflicts = [];
|
||||
log.info(colors.gray(`Applying changes to files ...`));
|
||||
console.log(colors.gray(`Applying changes to files ...`));
|
||||
for await (const change of changes) {
|
||||
const target = path.join(Deno.cwd(), change.path);
|
||||
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
|
||||
@@ -443,7 +305,7 @@ async function pull(
|
||||
try {
|
||||
const currentLocal = await Deno.readTextFile(target);
|
||||
if (currentLocal !== change.before && currentLocal !== change.after) {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.red(
|
||||
`Conflict detected on ${change.path}\nBoth local and remote have been modified.`
|
||||
)
|
||||
@@ -456,7 +318,7 @@ async function pull(
|
||||
});
|
||||
continue;
|
||||
} else if (opts.yes) {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.red(
|
||||
`Override local version with remote since --yes was passed and no --fail-conflicts.`
|
||||
)
|
||||
@@ -472,13 +334,13 @@ async function pull(
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (!change.path.endsWith(".json") && !change.path.endsWith(".yaml")) {
|
||||
log.info(`Editing script content of ${change.path}`);
|
||||
} catch {}
|
||||
if (!change.path.endsWith(".json")) {
|
||||
console.log(`Editing script content of ${change.path}`);
|
||||
} else {
|
||||
log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`);
|
||||
console.log(
|
||||
`Editing ${getTypeStrFromPath(change.path)} ${change.path}`
|
||||
);
|
||||
}
|
||||
await Deno.writeTextFile(target, change.after);
|
||||
|
||||
@@ -490,7 +352,9 @@ async function pull(
|
||||
await ensureDir(path.dirname(target));
|
||||
if (!opts.raw) {
|
||||
await ensureDir(path.dirname(stateTarget));
|
||||
log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`);
|
||||
console.log(
|
||||
`Adding ${getTypeStrFromPath(change.path)} ${change.path}`
|
||||
);
|
||||
}
|
||||
await Deno.writeTextFile(target, change.content);
|
||||
if (!opts.raw) {
|
||||
@@ -498,14 +362,14 @@ async function pull(
|
||||
}
|
||||
} else if (change.name === "deleted") {
|
||||
try {
|
||||
log.info(
|
||||
console.log(
|
||||
`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
|
||||
);
|
||||
await Deno.remove(target);
|
||||
if (!opts.raw) {
|
||||
await Deno.remove(stateTarget);
|
||||
}
|
||||
} catch {
|
||||
} catch (e) {
|
||||
if (!opts.raw) {
|
||||
await Deno.remove(stateTarget);
|
||||
}
|
||||
@@ -515,11 +379,11 @@ async function pull(
|
||||
if (opts.failConflicts) {
|
||||
if (conflicts.length > 0) {
|
||||
console.error(colors.red(`Conflicts were found`));
|
||||
log.info("Conflicts:");
|
||||
console.log("Conflicts:");
|
||||
for (const conflict of conflicts) {
|
||||
showConflict(conflict.path, conflict.local, conflict.change.after);
|
||||
}
|
||||
log.info(
|
||||
console.log(
|
||||
colors.red(`Please resolve theses conflicts manually by either:
|
||||
- reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
|
||||
- pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
|
||||
@@ -528,29 +392,49 @@ async function pull(
|
||||
Deno.exit(1);
|
||||
}
|
||||
}
|
||||
log.info(
|
||||
console.log(
|
||||
colors.green.underline(
|
||||
`Done! All ${changes.length} changes applied locally.`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function showConflict(path: string, local: string, remote: string) {
|
||||
console.log(colors.yellow(`- ${path}`));
|
||||
|
||||
let finalString = "";
|
||||
for (const part of Diff.diffLines(local, remote)) {
|
||||
if (part.removed) {
|
||||
// print red if removed without newline
|
||||
finalString += `\x1b[31m${part.value}\x1b[0m`;
|
||||
} else if (part.added) {
|
||||
// print green if added
|
||||
finalString += `\x1b[32m${part.value}\x1b[0m`;
|
||||
} else {
|
||||
// print white if unchanged
|
||||
finalString += `\x1b[37m${part.value}\x1b[0m`;
|
||||
}
|
||||
}
|
||||
console.log(finalString);
|
||||
console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
function prettyChanges(changes: Change[]) {
|
||||
for (const change of changes) {
|
||||
if (change.name === "added") {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.green(`+ ${getTypeStrFromPath(change.path)} ` + change.path)
|
||||
);
|
||||
} else if (change.name === "deleted") {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.red(`- ${getTypeStrFromPath(change.path)} ` + change.path)
|
||||
);
|
||||
} else if (change.name === "edited") {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path)
|
||||
);
|
||||
showDiff(change.before, change.after);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -569,10 +453,10 @@ function prettyChanges(changes: Change[]) {
|
||||
// }
|
||||
// }
|
||||
// if (diff.type === "REMOVE" || diff.type === "CHANGE") {
|
||||
// log.info(colors.red("- " + pathString + " = " + diff.oldValue));
|
||||
// console.log(colors.red("- " + pathString + " = " + diff.oldValue));
|
||||
// }
|
||||
// if (diff.type === "CREATE" || diff.type === "CHANGE") {
|
||||
// log.info(colors.green("+ " + pathString + " = " + diff.value));
|
||||
// console.log(colors.green("+ " + pathString + " = " + diff.value));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -588,45 +472,34 @@ async function push(
|
||||
skipPull: boolean;
|
||||
failConflicts: boolean;
|
||||
plainSecrets?: boolean;
|
||||
json?: boolean;
|
||||
}
|
||||
) {
|
||||
if (!opts.raw) {
|
||||
if (!opts.skipPull) {
|
||||
log.info(
|
||||
console.log(
|
||||
colors.gray("You need to be up-to-date before pushing, pulling first.")
|
||||
);
|
||||
await pull(opts);
|
||||
log.info(colors.green("Pull done, now pushing."));
|
||||
log.info("\n");
|
||||
console.log(colors.green("Pull done, now pushing."));
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
log.info(
|
||||
console.log(
|
||||
colors.gray(
|
||||
"Computing the files to update on the remote to match local (taking .wmillignore into account)"
|
||||
)
|
||||
);
|
||||
const remote = opts.raw
|
||||
? undefined
|
||||
: ZipFSElement(
|
||||
(await downloadZip(workspace, opts.plainSecrets))!,
|
||||
!opts.json
|
||||
);
|
||||
: ZipFSElement((await downloadZip(workspace, opts.plainSecrets))!);
|
||||
const local = await FSFSElement(path.join(Deno.cwd(), ""));
|
||||
const changes = await compareDynFSElement(
|
||||
local,
|
||||
remote,
|
||||
await ignoreF(),
|
||||
opts.json ?? false
|
||||
);
|
||||
const changes = await compareDynFSElement(local, remote, await ignoreF());
|
||||
|
||||
log.info(
|
||||
`remote (${workspace.name}) <- local: ${changes.length} changes to apply`
|
||||
);
|
||||
console.log(`remote <- local: ${changes.length} changes to apply`);
|
||||
if (changes.length > 0) {
|
||||
prettyChanges(changes);
|
||||
if (
|
||||
@@ -638,7 +511,7 @@ async function push(
|
||||
) {
|
||||
return;
|
||||
}
|
||||
log.info(colors.gray(`Applying changes to files ...`));
|
||||
console.log(colors.gray(`Applying changes to files ...`));
|
||||
const alreadySynced: string[] = [];
|
||||
for await (const change of changes) {
|
||||
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
|
||||
@@ -662,7 +535,12 @@ async function push(
|
||||
}
|
||||
continue;
|
||||
} else if (
|
||||
await handleFile(change.path, workspace.workspaceId, alreadySynced)
|
||||
await handleFile(
|
||||
change.path,
|
||||
change.after,
|
||||
workspace.workspaceId,
|
||||
alreadySynced
|
||||
)
|
||||
) {
|
||||
if (!opts.raw && stateExists) {
|
||||
await Deno.writeTextFile(stateTarget, change.after);
|
||||
@@ -671,56 +549,65 @@ async function push(
|
||||
}
|
||||
if (!opts.raw) {
|
||||
await ensureDir(path.dirname(stateTarget));
|
||||
log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`);
|
||||
console.log(
|
||||
`Editing ${getTypeStrFromPath(change.path)} ${change.path}`
|
||||
);
|
||||
}
|
||||
const oldObj = parseFromPath(change.path, change.before);
|
||||
const newObj = parseFromPath(change.path, change.after);
|
||||
const obj = inferTypeFromPath(change.path, JSON.parse(change.after));
|
||||
|
||||
pushObj(
|
||||
workspace.workspaceId,
|
||||
change.path,
|
||||
oldObj,
|
||||
newObj,
|
||||
opts.plainSecrets ?? false,
|
||||
opts.raw
|
||||
const diff = microdiff(
|
||||
inferTypeFromPath(change.path, JSON.parse(change.before)),
|
||||
obj,
|
||||
{ cyclesFix: false }
|
||||
);
|
||||
await applyDiff(
|
||||
workspace.workspaceId,
|
||||
change.path.split(".")[0],
|
||||
obj,
|
||||
diff,
|
||||
opts.plainSecrets
|
||||
);
|
||||
|
||||
if (!opts.raw && stateExists) {
|
||||
await Deno.writeTextFile(stateTarget, change.after);
|
||||
}
|
||||
} else if (change.name === "added") {
|
||||
if (
|
||||
change.path.endsWith(".script.json") ||
|
||||
change.path.endsWith(".script.yaml")
|
||||
) {
|
||||
if (change.path.endsWith(".script.json")) {
|
||||
continue;
|
||||
} else if (
|
||||
await handleFile(change.path, workspace.workspaceId, alreadySynced)
|
||||
await handleFile(
|
||||
change.path,
|
||||
change.content,
|
||||
workspace.workspaceId,
|
||||
alreadySynced
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (!opts.raw && stateExists) {
|
||||
await ensureDir(path.dirname(stateTarget));
|
||||
log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`);
|
||||
console.log(
|
||||
`Adding ${getTypeStrFromPath(change.path)} ${change.path}`
|
||||
);
|
||||
}
|
||||
const obj = parseFromPath(change.path, change.content);
|
||||
pushObj(
|
||||
const obj = inferTypeFromPath(change.path, JSON.parse(change.content));
|
||||
const diff = microdiff({}, obj, { cyclesFix: false });
|
||||
await applyDiff(
|
||||
workspace.workspaceId,
|
||||
change.path,
|
||||
undefined,
|
||||
change.path.split(".")[0],
|
||||
obj,
|
||||
opts.plainSecrets ?? false,
|
||||
opts.raw
|
||||
diff,
|
||||
opts.plainSecrets
|
||||
);
|
||||
|
||||
if (!opts.raw && stateExists) {
|
||||
await Deno.writeTextFile(stateTarget, change.content);
|
||||
}
|
||||
} else if (change.name === "deleted") {
|
||||
if (!change.path.includes(".json") && !change.path.includes(".yaml")) {
|
||||
if (!change.path.includes(".json")) {
|
||||
continue;
|
||||
}
|
||||
log.info(`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`);
|
||||
console.log(
|
||||
`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
|
||||
);
|
||||
const typ = getTypeStrFromPath(change.path);
|
||||
const workspaceId = workspace.workspaceId;
|
||||
switch (typ) {
|
||||
@@ -776,17 +663,52 @@ async function push(
|
||||
}
|
||||
try {
|
||||
await Deno.remove(stateTarget);
|
||||
} catch {
|
||||
// state target may not exist already
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
log.info(
|
||||
console.log(
|
||||
colors.green.underline(
|
||||
`Done! All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}.`
|
||||
`Done! All ${changes.length} changes pushed to the remote workspace.`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
async function applyDiff(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
file:
|
||||
| ScriptFile
|
||||
| VariableFile
|
||||
| FlowFile
|
||||
| ResourceFile
|
||||
| ResourceTypeFile
|
||||
| FolderFile,
|
||||
diffs: Difference[],
|
||||
plainSecrets?: boolean
|
||||
) {
|
||||
if (file instanceof ScriptFile) {
|
||||
throw new Error(
|
||||
"This code path should be unreachable - we should never generate diffs for scripts"
|
||||
);
|
||||
} else if (file instanceof FolderFile) {
|
||||
const parts = remotePath.split("/");
|
||||
if (parts[0] === "f") {
|
||||
remotePath = parts[1];
|
||||
} else {
|
||||
remotePath = parts[0];
|
||||
}
|
||||
}
|
||||
if (diffs.length === 0) {
|
||||
console.log("No diffs to apply to " + remotePath);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await file.pushDiffs(workspace, remotePath, diffs, plainSecrets);
|
||||
} catch (e) {
|
||||
console.error("Failing to apply diffs to " + remotePath);
|
||||
console.error(JSON.stringify(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
@@ -801,8 +723,6 @@ const command = new Command()
|
||||
.option("--yes", "Pull without needing confirmation")
|
||||
.option("--raw", "Pull without using state, just overwrite.")
|
||||
.option("--plain-secrets", "Pull secrets as plain text")
|
||||
.option("--json", "Use JSON instead of YAML")
|
||||
// deno-lint-ignore no-explicit-any
|
||||
.action(pull as any)
|
||||
.command("push")
|
||||
.description(
|
||||
@@ -816,8 +736,6 @@ const command = new Command()
|
||||
.option("--yes", "Push without needing confirmation")
|
||||
.option("--raw", "Push without using state, just overwrite.")
|
||||
.option("--plain-secrets", "Push secrets as plain text")
|
||||
.option("--json", "Use JSON instead of YAML")
|
||||
// deno-lint-ignore no-explicit-any
|
||||
.action(push as any);
|
||||
|
||||
export default command;
|
||||
|
||||
197
cli/types.ts
197
cli/types.ts
@@ -1,20 +1,31 @@
|
||||
// deno-lint-ignore-file no-explicit-any
|
||||
import { decoverto } from "./decoverto.ts";
|
||||
import { FlowFile } from "./flow.ts";
|
||||
import { ResourceTypeFile } from "./resource-type.ts";
|
||||
import { ResourceFile } from "./resource.ts";
|
||||
import { ScriptFile } from "./script.ts";
|
||||
import { VariableFile } from "./variable.ts";
|
||||
import { path } from "./deps.ts";
|
||||
import { FolderFile } from "./folder.ts";
|
||||
import { AppFile } from "./apps.ts";
|
||||
|
||||
import { colors, log, path } from "./deps.ts";
|
||||
import { pushApp } from "./apps.ts";
|
||||
import {
|
||||
parse as yamlParse,
|
||||
stringify as yamlStringify,
|
||||
} from "https://deno.land/std@0.184.0/yaml/mod.ts";
|
||||
import { equal } from "https://deno.land/x/equal@v1.5.0/equal.ts";
|
||||
import { pushFolder } from "./folder.ts";
|
||||
import { pushFlow } from "./flow.ts";
|
||||
import { pushResource } from "./resource.ts";
|
||||
import { pushResourceType } from "./resource-type.ts";
|
||||
import { pushVariable } from "./variable.ts";
|
||||
import * as Diff from "npm:diff";
|
||||
import { yamlOptions } from "./sync.ts";
|
||||
import { showDiffs } from "./main.ts";
|
||||
// TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
|
||||
// Then combine those with diffing, which then gives the new push impl
|
||||
export interface Resource {
|
||||
push(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
plainSecrets?: boolean
|
||||
): Promise<void>;
|
||||
}
|
||||
|
||||
export interface PushDiffs {
|
||||
pushDiffs(
|
||||
workspace: string,
|
||||
remotePath: string,
|
||||
diffs: Difference[],
|
||||
plainSecrets?: boolean
|
||||
): Promise<void>;
|
||||
}
|
||||
|
||||
export interface DifferenceCreate {
|
||||
type: "CREATE";
|
||||
@@ -37,109 +48,77 @@ export interface DifferenceChange {
|
||||
|
||||
export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;
|
||||
|
||||
export function setValueByPath(
|
||||
obj: any,
|
||||
path: (string | number)[],
|
||||
value: any
|
||||
) {
|
||||
let i;
|
||||
let lastObj = undefined;
|
||||
for (i = 0; i < path.length - 1; i++) {
|
||||
if (!obj) {
|
||||
let oldNewObj;
|
||||
if (typeof path[i] === "number") {
|
||||
oldNewObj = [];
|
||||
} else {
|
||||
oldNewObj = {};
|
||||
}
|
||||
lastObj[path[i - 1]] = oldNewObj;
|
||||
obj = oldNewObj;
|
||||
}
|
||||
lastObj = obj;
|
||||
obj = obj[path[i]];
|
||||
}
|
||||
if (!obj) {
|
||||
let oldNewObj;
|
||||
if (typeof path[i] === "number") {
|
||||
oldNewObj = [];
|
||||
} else {
|
||||
oldNewObj = {};
|
||||
}
|
||||
lastObj[path[i - 1]] = oldNewObj;
|
||||
obj = oldNewObj;
|
||||
}
|
||||
obj[path[i]] = value;
|
||||
}
|
||||
|
||||
export type GlobalOptions = {
|
||||
workspace: string | undefined;
|
||||
token: string | undefined;
|
||||
};
|
||||
|
||||
export function isSuperset(
|
||||
subset: Record<string, any>,
|
||||
superset: Record<string, any>
|
||||
): boolean {
|
||||
return Object.keys(subset).every((key) => {
|
||||
const eq = equal(subset[key], superset[key]);
|
||||
if (!eq && showDiffs) {
|
||||
const sub = subset[key];
|
||||
const supers = superset[key];
|
||||
if (!supers) {
|
||||
log.info(`Key ${key} not found in remote`);
|
||||
} else {
|
||||
log.info(`Found diff for ${key}:`);
|
||||
showDiff(
|
||||
yamlStringify(sub, yamlOptions),
|
||||
yamlStringify(supers, yamlOptions)
|
||||
);
|
||||
}
|
||||
}
|
||||
return eq;
|
||||
});
|
||||
}
|
||||
|
||||
export function showDiff(local: string, remote: string) {
|
||||
let finalString = "";
|
||||
for (const part of Diff.diffLines(local, remote)) {
|
||||
if (part.removed) {
|
||||
// print red if removed without newline
|
||||
finalString += `\x1b[31m${part.value}\x1b[0m`;
|
||||
} else if (part.added) {
|
||||
// print green if added
|
||||
finalString += `\x1b[32m${part.value}\x1b[0m`;
|
||||
} else {
|
||||
let lines = part.value.split("\n");
|
||||
|
||||
if (lines.length > 4) {
|
||||
lines = lines.slice(0, 2);
|
||||
lines.push("...");
|
||||
lines = lines.concat(part.value.split("\n").slice(-2));
|
||||
}
|
||||
// print white if unchanged
|
||||
finalString += `\x1b[37m${lines.join("\n")}\x1b[0m`;
|
||||
}
|
||||
}
|
||||
log.info(finalString);
|
||||
}
|
||||
|
||||
export function showConflict(path: string, local: string, remote: string) {
|
||||
log.info(colors.yellow(`- ${path}`));
|
||||
showDiff(local, remote);
|
||||
log.info("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
|
||||
log.info("\n");
|
||||
}
|
||||
|
||||
export function pushObj(
|
||||
workspace: string,
|
||||
export function inferTypeFromPath(
|
||||
p: string,
|
||||
befObj: any,
|
||||
newObj: any,
|
||||
plainSecrets: boolean,
|
||||
checkForCreate: boolean
|
||||
) {
|
||||
obj: any
|
||||
):
|
||||
| ScriptFile
|
||||
| VariableFile
|
||||
| FlowFile
|
||||
| ResourceFile
|
||||
| ResourceTypeFile
|
||||
| FolderFile
|
||||
| AppFile {
|
||||
const typeEnding = getTypeStrFromPath(p);
|
||||
|
||||
if (typeEnding === "app") {
|
||||
pushApp(workspace, p, befObj, newObj, checkForCreate);
|
||||
} else if (typeEnding === "folder") {
|
||||
pushFolder(workspace, p, befObj, newObj, checkForCreate);
|
||||
if (typeEnding === "folder") {
|
||||
return decoverto.type(FolderFile).plainToInstance(obj);
|
||||
} else if (typeEnding === "script") {
|
||||
return decoverto.type(ScriptFile).plainToInstance(obj);
|
||||
} else if (typeEnding === "variable") {
|
||||
pushVariable(workspace, p, befObj, newObj, plainSecrets, checkForCreate);
|
||||
return decoverto.type(VariableFile).plainToInstance(obj);
|
||||
} else if (typeEnding === "flow") {
|
||||
const flowName = p.split(".flow/")[0];
|
||||
pushFlow(workspace, flowName, flowName + ".flow");
|
||||
return decoverto.type(FlowFile).plainToInstance(obj);
|
||||
} else if (typeEnding === "resource") {
|
||||
pushResource(workspace, p, befObj, newObj, checkForCreate);
|
||||
return decoverto.type(ResourceFile).plainToInstance(obj);
|
||||
} else if (typeEnding === "resource-type") {
|
||||
pushResourceType(workspace, p, befObj, newObj, checkForCreate);
|
||||
return decoverto.type(ResourceTypeFile).plainToInstance(obj);
|
||||
} else if (typeEnding === "app") {
|
||||
return decoverto.type(AppFile).plainToInstance(obj);
|
||||
} else {
|
||||
throw new Error("infer type unreachable");
|
||||
}
|
||||
}
|
||||
|
||||
export function parseFromPath(p: string, content: string): any {
|
||||
return p.endsWith(".yaml")
|
||||
? yamlParse(content)
|
||||
: p.endsWith(".json")
|
||||
? JSON.parse(content)
|
||||
: content;
|
||||
}
|
||||
export function parseFromFile(p: string): any {
|
||||
if (p.endsWith(".json")) {
|
||||
return JSON.parse(Deno.readTextFileSync(p));
|
||||
} else if (p.endsWith(".yaml") || p.endsWith(".yml")) {
|
||||
return yamlParse(Deno.readTextFileSync(p));
|
||||
} else {
|
||||
throw new Error("Could not read file " + p);
|
||||
}
|
||||
}
|
||||
export function getTypeStrFromPath(
|
||||
p: string
|
||||
):
|
||||
@@ -150,9 +129,6 @@ export function getTypeStrFromPath(
|
||||
| "resource-type"
|
||||
| "folder"
|
||||
| "app" {
|
||||
if (p.includes(".flow/")) {
|
||||
return "flow";
|
||||
}
|
||||
const parsed = path.parse(p);
|
||||
if (
|
||||
parsed.ext == ".go" ||
|
||||
@@ -171,6 +147,7 @@ export function getTypeStrFromPath(
|
||||
if (
|
||||
typeEnding === "script" ||
|
||||
typeEnding === "variable" ||
|
||||
typeEnding === "flow" ||
|
||||
typeEnding === "resource" ||
|
||||
typeEnding === "resource-type" ||
|
||||
typeEnding === "app"
|
||||
@@ -180,13 +157,3 @@ export function getTypeStrFromPath(
|
||||
throw new Error("Could not infer type of path " + JSON.stringify(parsed));
|
||||
}
|
||||
}
|
||||
|
||||
export function removeType(str: string, type: string) {
|
||||
if (
|
||||
!str.endsWith("." + type + ".yaml") &&
|
||||
!str.endsWith("." + type + ".json")
|
||||
) {
|
||||
throw new Error(str + " does not end with ." + type + ".(yaml|json)");
|
||||
}
|
||||
return str.slice(0, str.length - type.length - 6);
|
||||
}
|
||||
|
||||
36
cli/user.ts
36
cli/user.ts
@@ -5,7 +5,6 @@ import {
|
||||
colors,
|
||||
Command,
|
||||
GlobalUserInfo,
|
||||
log,
|
||||
passwordGenerator,
|
||||
Table,
|
||||
UserService,
|
||||
@@ -37,7 +36,7 @@ async function list(opts: GlobalOptions) {
|
||||
x.company ?? "-",
|
||||
x.verified ? "true" : "false",
|
||||
x.super_admin ? "true" : "false",
|
||||
])
|
||||
]),
|
||||
)
|
||||
.render();
|
||||
}
|
||||
@@ -49,7 +48,7 @@ async function add(
|
||||
name?: string;
|
||||
},
|
||||
email: string,
|
||||
password?: string
|
||||
password?: string,
|
||||
) {
|
||||
await requireLogin(opts);
|
||||
const password_final = password ?? passwordGenerator("*", 15);
|
||||
@@ -63,34 +62,33 @@ async function add(
|
||||
},
|
||||
});
|
||||
if (!password) {
|
||||
log.info(colors.red("New Password: " + password_final));
|
||||
console.log(colors.red("New Password: " + password_final));
|
||||
}
|
||||
log.info(colors.underline.green("New User Created."));
|
||||
console.log(colors.underline.green("New User Created."));
|
||||
}
|
||||
async function remove(opts: GlobalOptions, email: string) {
|
||||
await requireLogin(opts);
|
||||
|
||||
await UserService.globalUserDelete({ email });
|
||||
log.info(colors.green("Deleted User " + email));
|
||||
console.log(colors.green("Deleted User " + email));
|
||||
}
|
||||
|
||||
async function createToken(
|
||||
opts: GlobalOptions & { email: string; password: string }
|
||||
opts: GlobalOptions & { email: string; password: string },
|
||||
) {
|
||||
if (opts.email && opts.password) {
|
||||
log.info(
|
||||
"Token: " +
|
||||
(await UserService.login({
|
||||
requestBody: {
|
||||
email: opts.email,
|
||||
password: opts.password,
|
||||
},
|
||||
}))
|
||||
console.log(
|
||||
"Token: " + await UserService.login({
|
||||
requestBody: {
|
||||
email: opts.email,
|
||||
password: opts.password,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await requireLogin(opts);
|
||||
log.info("Token: " + (await UserService.createToken({ requestBody: {} })));
|
||||
console.log("Token: " + await UserService.createToken({ requestBody: {} }));
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
@@ -101,7 +99,7 @@ const command = new Command()
|
||||
.option("--superadmin", "Specify to make the new user superadmin.")
|
||||
.option(
|
||||
"--company <company:string>",
|
||||
"Specify to set the company of the new user."
|
||||
"Specify to set the company of the new user.",
|
||||
)
|
||||
.option("--name <name:string>", "Specify to set the name of the new user.")
|
||||
.action(add as any)
|
||||
@@ -114,14 +112,14 @@ const command = new Command()
|
||||
"Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.",
|
||||
{
|
||||
depends: ["password"],
|
||||
}
|
||||
},
|
||||
)
|
||||
.option(
|
||||
"--password <password:string>",
|
||||
"Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.",
|
||||
{
|
||||
depends: ["email"],
|
||||
}
|
||||
},
|
||||
)
|
||||
.action(createToken as any);
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user