diff --git a/.github/workflows/backend-check.yml b/.github/workflows/backend-check.yml index 11f8af5c29..db3cdfcc33 100644 --- a/.github/workflows/backend-check.yml +++ b/.github/workflows/backend-check.yml @@ -17,9 +17,12 @@ jobs: with: fetch-depth: 0 + - name: Install mold and clang + run: sudo apt-get update && sudo apt-get install -y mold clang + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: cargo check working-directory: ./backend @@ -33,14 +36,14 @@ jobs: with: fetch-depth: 0 - - name: install xmlsec1 + - name: install xmlsec1 and gssapi run: | sudo apt-get update - sudo apt-get install -y libxml2-dev libxmlsec1-dev + sudo apt-get install -y libxml2-dev libxmlsec1-dev libkrb5-dev libsasl2-dev libcurl4-openssl-dev mold clang - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: cargo check working-directory: ./backend @@ -72,9 +75,12 @@ jobs: run: | ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + - name: Install mold and clang + run: sudo apt-get update && sudo apt-get install -y mold clang + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: cargo check working-directory: ./backend @@ -100,10 +106,10 @@ jobs: token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} fetch-depth: 0 - - name: install xmlsec1 + - name: install xmlsec1 and gssapi run: | sudo apt-get update - sudo apt-get install -y libxml2-dev libxmlsec1-dev + sudo apt-get install -y libxml2-dev libxmlsec1-dev libkrb5-dev libsasl2-dev libcurl4-openssl-dev mold clang - name: Substitute EE code (EE logic is behind feature flag) run: | diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 1f51ee2fa4..f5d0df5a44 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -19,7 +19,7 @@ defaults: jobs: cargo_test: - runs-on: ubicloud-standard-8 + runs-on: ubicloud-standard-16 services: postgres: image: postgres @@ -28,9 +28,20 @@ jobs: env: POSTGRES_DB: windmill POSTGRES_PASSWORD: changeme + POSTGRES_INITDB_ARGS: "-c max_connections=500" options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 + --health-retries 5 --shm-size=256mb + mysql: + image: mysql:8.0 + ports: + - 3306:3306 + env: + MYSQL_ROOT_PASSWORD: changeme + MYSQL_DATABASE: windmill_test + options: >- + --health-cmd "mysqladmin ping -h localhost" --health-interval 10s + --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v4 @@ -47,13 +58,25 @@ jobs: bun-version: 1.3.8 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: "20" - uses: astral-sh/setup-uv@v6.2.1 with: version: "0.9.24" + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.3" + tools: composer + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: false + - name: Install PowerShell, mold and clang + run: | + sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev + working-directory: / - uses: actions-rust-lang/setup-rust-toolchain@v1 with: - cache-workspaces: backend + cache: false toolchain: 1.90.0 - name: Read EE repo commit hash run: | @@ -183,19 +206,20 @@ jobs: restore-keys: | ${{ runner.os }}-duckdb-ffi- - name: cargo test - timeout-minutes: 16 + timeout-minutes: 30 env: SQLX_OFFLINE: true DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill DISABLE_EMBEDDING: true - RUST_LOG: info + RUST_LOG: "off" RUST_LOG_STYLE: never CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_BUILD_JOBS: 12 WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 TEST_NPM_REGISTRY: "http://localhost:4873/:_authToken=${{ env.NPM_TOKEN }}" run: | - deno --version && bun -v && node --version && go version && python3 --version + deno --version && bun -v && node --version && go version && python3 --version && php --version && ruby --version && pwsh --version && dotnet --version cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd .. - DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test --all -- --nocapture + DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp --all -- --nocapture --test-threads=10 diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index f467cd432f..c6bcea9f9c 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,native_trigger,static_frontend,all_languages,deno_core,mcp,private + features=ee_rhel secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build-publish-rh8-image.yml b/.github/workflows/build-publish-rh8-image.yml index fc10b6bcac..fc35b4c327 100644 --- a/.github/workflows/build-publish-rh8-image.yml +++ b/.github/workflows/build-publish-rh8-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,native_trigger,static_frontend,all_languages,deno_core,mcp,private + features=ee_rhel secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,native_trigger,static_frontend,all_languages,deno_core,mcp,private + features=ee_rhel secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 985e4d94fe..8647307e1b 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -11,7 +11,7 @@ env: jobs: cargo_build_windows: - runs-on: windows-latest + runs-on: blacksmith-16vcpu-windows-2025 steps: - uses: actions/checkout@v4 @@ -30,34 +30,41 @@ jobs: token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} fetch-depth: 0 - - name: Setup Rust - uses: actions-rs/toolchain@v1 + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: + cache-workspaces: backend toolchain: 1.90.0 - override: true - name: Substitute EE code shell: bash run: | ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + - name: Cargo check (fail fast on warnings) + timeout-minutes: 60 + env: + RUSTFLAGS: "-D warnings" + run: | + mkdir frontend/build && cd backend + New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + cargo check --features=ee_windows + - name: Cargo build dynamic libraries windows - timeout-minutes: 90 + timeout-minutes: 180 run: | cd backend/windmill-duckdb-ffi-internal cargo build --release -p windmill_duckdb_ffi_internal - name: Cargo build binary windows - timeout-minutes: 90 + timeout-minutes: 180 run: | vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" - mkdir frontend/build && cd backend - New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,nats,sqs_trigger,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,native_trigger,static_frontend,all_languages_windows,mcp,bedrock,private + cd backend + cargo build --release --features=ee_windows - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index a95e26d66b..8be9f6d187 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -75,10 +75,10 @@ jobs: npm install npm run generate-backend-client - - name: install xmlsec1 + - name: install xmlsec1 and gssapi run: | sudo apt-get update - sudo apt-get install -y libxml2-dev libxmlsec1-dev + sudo apt-get install -y libxml2-dev libxmlsec1-dev libkrb5-dev libsasl2-dev libcurl4-openssl-dev - uses: actions-rust-lang/setup-rust-toolchain@v1 with: diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 1042833c16..4a7fc2a874 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,license,http_trigger,zip,oauth2,postgres_trigger,mqtt_trigger,websocket,smtp,native_trigger,static_frontend,all_languages,deno_core,mcp + features=ce_rpi tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 71948bac1f..09d3728fba 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -92,7 +92,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,native_trigger,static_frontend,agent_worker_server,all_languages,deno_core,mcp,bedrock,private + features=ce tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -154,7 +154,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,native_trigger,static_frontend,all_languages,private,deno_core,mcp,bedrock + features=ee tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index 7e8a5e2acf..c098f856e2 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -68,11 +68,11 @@ jobs: with: workspaces: "./backend -> target" - - name: Install xmlsec build-time deps + - name: Install xmlsec and gssapi build-time deps run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - pkg-config libxml2-dev libssl-dev \ + pkg-config libxml2-dev libssl-dev libkrb5-dev \ xmlsec1 libxmlsec1-dev libxmlsec1-openssl - name: Run update-sqlx script diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index aac8b178e9..a20997069e 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -13,7 +13,7 @@ env: jobs: cargo_build_windows: - runs-on: windows-latest + runs-on: blacksmith-16vcpu-windows-2025 steps: - uses: actions/checkout@v4 @@ -32,11 +32,10 @@ jobs: token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} fetch-depth: 0 - - name: Setup Rust - uses: actions-rs/toolchain@v1 + - uses: actions-rust-lang/setup-rust-toolchain@v1 with: + cache-workspaces: backend toolchain: 1.90.0 - override: true - name: Substitute EE code shell: bash @@ -44,13 +43,13 @@ jobs: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Cargo build dynamic libraries windows - timeout-minutes: 90 + timeout-minutes: 180 run: | cd backend/windmill-duckdb-ffi-internal cargo build --release -p windmill_duckdb_ffi_internal - name: Cargo build windows - timeout-minutes: 90 + timeout-minutes: 180 run: | vcpkg.exe install openssl-windows:x64-windows vcpkg.exe install openssl:x64-windows-static @@ -59,7 +58,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,mqtt_trigger,gcp_trigger,websocket,smtp,native_trigger,static_frontend,all_languages_windows,mcp,bedrock,private + cargo build --release --features=ee_windows - name: Rename binary with corresponding architecture run: | Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" diff --git a/.github/workflows/spawn-ephemeral-backend.yml b/.github/workflows/spawn-ephemeral-backend.yml index bfc57639de..9070f5af0d 100644 --- a/.github/workflows/spawn-ephemeral-backend.yml +++ b/.github/workflows/spawn-ephemeral-backend.yml @@ -3,6 +3,8 @@ name: Spawn Ephemeral Backend on: issue_comment: types: [created] + pull_request_review_comment: + types: [created] workflow_dispatch: inputs: pr_number: @@ -11,11 +13,42 @@ on: type: number jobs: + determine-commenter: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '/spawnbackend')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/spawnbackend')) + runs-on: ubicloud-standard-2 + outputs: + commenter: ${{ steps.determine-commenter.outputs.commenter }} + steps: + - name: Determine commenter + id: determine-commenter + run: | + # Work out who wrote the comment / review + if [[ "${{ github.event_name }}" == "issue_comment" || \ + "${{ github.event_name }}" == "pull_request_review_comment" ]]; then + COMMENTER="${{ github.event.comment.user.login }}" + elif [[ "${{ github.event_name }}" == "pull_request_review" ]]; then + COMMENTER="${{ github.event.review.user.login }}" + else + COMMENTER="${{ github.event.issue.user.login }}" + fi + echo "commenter=$COMMENTER" >> $GITHUB_OUTPUT + + check-membership: + needs: determine-commenter + uses: ./.github/workflows/check-org-membership.yml + with: + commenter: ${{ needs.determine-commenter.outputs.commenter }} + secrets: + access_token: ${{ secrets.ORG_ACCESS_TOKEN }} + spawn-backend: + needs: [determine-commenter, check-membership] # Only run on PR comments that contain /spawn-backend, or manual dispatch if: | github.event_name == 'workflow_dispatch' || - (github.event.issue.pull_request && contains(github.event.comment.body, '/spawn-backend')) + (github.event.issue.pull_request && needs.check-membership.outputs.is_member == 'true') runs-on: ubuntu-latest permissions: pull-requests: write @@ -36,40 +69,80 @@ jobs: repo: context.repo.repo, pull_number: prNumber }); + + // Get branch name and format it for Cloudflare Pages + // Replace '/' with '-' for the URL + const branchName = pr.data.head.ref; + const formattedBranch = branchName.replace(/\//g, '-'); + const cfFrontendUrl = `https://${formattedBranch}.windmill.pages.dev`; + core.setOutput('commit_hash', pr.data.head.sha); - core.setOutput('pr_number', context.issue.number); + core.setOutput('pr_number', prNumber); + core.setOutput('branch_name', branchName); + core.setOutput('cf_frontend_url', cfFrontendUrl); - - name: Trigger Windmill flow - id: trigger-flow + - name: Check manager URL + id: check-manager-url run: | - RESPONSE=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \ - -H "Authorization: Bearer ${{ secrets.WINDMILL_TOKEN }}" \ - -H "Content-Type: application/json" \ - -d '{ - "manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}", - "commit_hash": "${{ steps.pr-details.outputs.commit_hash }}", - "pr_number": ${{ steps.pr-details.outputs.pr_number }} - }') - - JOB_UUID=$(echo "$RESPONSE" | jq -r '.id // empty') - - if [ -z "$JOB_UUID" ]; then - echo "Failed to get job UUID from response: $RESPONSE" - exit 1 + if [ -z "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}" ]; then + echo "manager_url_set=false" >> $GITHUB_OUTPUT + else + echo "manager_url_set=true" >> $GITHUB_OUTPUT fi - echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT - - - name: Post comment with job link + - name: Post error comment if manager not running + if: steps.check-manager-url.outputs.manager_url_set == 'false' uses: actions/github-script@v7 with: script: | - const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}'; - const jobUrl = `https://app.windmill.dev/run/${jobUuid}?workspace=windmill-labs`; + const prNumber = context.eventName === 'workflow_dispatch' + ? Number(context.payload.inputs.pr_number) + : context.issue.number; await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.issue.number, - body: `🚀 Ephemeral backend spawning started!\n\nView job progress: ${jobUrl}` + issue_number: prNumber, + body: `❌ Manager URL not set (did you start the ephemeral backend manager?)\n\nThe ephemeral backend manager needs to be running to spawn backends. Please start the manager first.` + }); + + - name: Fail if manager not running + if: steps.check-manager-url.outputs.manager_url_set == 'false' + run: | + echo "Error: EPHEMERAL_BACKEND_QUEUE_URL secret is not set" + exit 1 + + - name: Trigger Windmill flow + if: steps.check-manager-url.outputs.manager_url_set == 'true' + id: trigger-flow + run: | + JOB_UUID=$(curl -s -X POST "https://app.windmill.dev/api/w/windmill-labs/jobs/run/f/f/all/run_ephemeral_backend" \ + -H "Authorization: Bearer ${{ secrets.WINDMILL_RUN_FLOW_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d '{ + "manager_url": "${{ secrets.EPHEMERAL_BACKEND_QUEUE_URL }}", + "commit_hash": "${{ steps.pr-details.outputs.commit_hash }}", + "pr_number": ${{ steps.pr-details.outputs.pr_number }}, + "cf_frontend_url": "${{ steps.pr-details.outputs.cf_frontend_url }}" + }' | tr -d '"') + + echo "Job UUID: $JOB_UUID" + echo "job_uuid=$JOB_UUID" >> $GITHUB_OUTPUT + + - name: Post comment with job link + if: steps.check-manager-url.outputs.manager_url_set == 'true' + uses: actions/github-script@v7 + with: + script: | + const jobUuid = '${{ steps.trigger-flow.outputs.job_uuid }}'; + const appUrl = `https://app.windmill.dev/public/windmill-labs/a106bad0256c1dfa7a4f9279c42b1a4b#${jobUuid}`; + const prNumber = context.eventName === 'workflow_dispatch' + ? Number(context.payload.inputs.pr_number) + : context.issue.number; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `🚀 Spawning new ephemeral backend!\n\n${appUrl}` }); diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b05312eff..896f7ef025 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,106 @@ # Changelog +## [1.629.1](https://github.com/windmill-labs/windmill/compare/v1.629.0...v1.629.1) (2026-02-10) + + +### Bug Fixes + +* remove unecessary drop index on labeled_jobs_on_jobs ([0803164](https://github.com/windmill-labs/windmill/commit/08031640a02ebd5971793942e8534d69f4f71d28)) + +## [1.629.0](https://github.com/windmill-labs/windmill/compare/v1.628.3...v1.629.0) (2026-02-09) + + +### Features + +* customer portal extra workspace stats ([#7841](https://github.com/windmill-labs/windmill/issues/7841)) ([153dd32](https://github.com/windmill-labs/windmill/commit/153dd32187a3e32e3f26ab88b62195a0f9a359b9)) + + +### Bug Fixes + +* adapt mysql and ruby test assertions ([477832d](https://github.com/windmill-labs/windmill/commit/477832dbeeafb88fd16c174d5d1df8ed042e6f31)) +* add missing :name param to groups/is_owner route ([fa53a87](https://github.com/windmill-labs/windmill/commit/fa53a87107158c9e8a7e5b522242b0076a78ce46)) +* **backend:** prevent sqs hanging ([#7857](https://github.com/windmill-labs/windmill/issues/7857)) ([a3fc27b](https://github.com/windmill-labs/windmill/commit/a3fc27b23224aef2949c19f7c123d28e6cfaf968)) +* box push() future to prevent stack overflow in nested async chains ([67c8aef](https://github.com/windmill-labs/windmill/commit/67c8aef9faea5afe8ce330b477f16b4aed1779a5)) +* **frontend:** reorganize workspace settings ([#7788](https://github.com/windmill-labs/windmill/issues/7788)) ([dd42184](https://github.com/windmill-labs/windmill/commit/dd421845ba148bba70bcbafbc6a39f3012eb037c)) +* improve scheduling reliability in extreme pool contention conditions ([#7825](https://github.com/windmill-labs/windmill/issues/7825)) ([bbb397b](https://github.com/windmill-labs/windmill/commit/bbb397b6ad954052f0bd33cc4ff8897eed66e4db)) +* improve tracing behavior with NO_PROXY ([4cce13f](https://github.com/windmill-labs/windmill/commit/4cce13f5228a05da1bbce43bed7e856ce0bcf979)) +* incorrect raw app public workspaceStore derived ([edb0d4a](https://github.com/windmill-labs/windmill/commit/edb0d4a05da567b3b0be5d94c9b2856d68ecb0ff)) +* increase test thread stack size to 8MB in CI ([5548098](https://github.com/windmill-labs/windmill/commit/5548098e083af76a0b7d6f645a5458592d9c8ddc)) +* install mold+clang in Docker for cargo linker config ([99bc383](https://github.com/windmill-labs/windmill/commit/99bc383f9e94a415ff1dcef1c45ccc4c8dab1a9e)) +* make V8 runtime init idempotent and auto-initialize before isolate creation ([aa9f3da](https://github.com/windmill-labs/windmill/commit/aa9f3da429da92a059aaabb28481d33b8dacd37b)) +* parse Python datetime.datetime and datetime.date type annotations ([#7856](https://github.com/windmill-labs/windmill/issues/7856)) ([ff70a4e](https://github.com/windmill-labs/windmill/commit/ff70a4e9d105cac58c0fb0aba8fbec9875533aa4)) +* prevent V8 SIGSEGV by serializing isolate creation and fixing use-after-free ([05106d7](https://github.com/windmill-labs/windmill/commit/05106d7deeda92b7ae0e1708554f6dcb088c4a08)) +* reduce DB pool contention by eliminating dual-connection patterns ([#7861](https://github.com/windmill-labs/windmill/issues/7861)) ([4343b73](https://github.com/windmill-labs/windmill/commit/4343b73485843c3b482c21e60052f171ada9b843)) +* remove mold linker config that breaks Docker builds ([fea0954](https://github.com/windmill-labs/windmill/commit/fea0954f20f9f7c5a43b25b23df530faeac94999)) +* restart after empty branchone + improve UI ([#7838](https://github.com/windmill-labs/windmill/issues/7838)) ([b1d6ac9](https://github.com/windmill-labs/windmill/commit/b1d6ac91bd3af073feac0b31d97f7b4414d8786e)) +* use unprotected V8 platform to prevent SIGSEGV on x86_64 Linux ([90d0103](https://github.com/windmill-labs/windmill/commit/90d010347c65086b17f9802dd9a7d2da90dc68eb)) +* wmill workspace list to list local profiles ([#7843](https://github.com/windmill-labs/windmill/issues/7843)) ([f924a82](https://github.com/windmill-labs/windmill/commit/f924a8268461c49a0fec26e3216ec9546601b8de)) + +## [1.628.3](https://github.com/windmill-labs/windmill/compare/v1.628.2...v1.628.3) (2026-02-06) + + +### Bug Fixes + +* **bun:** `//native` not using workspace dependencies ([#7833](https://github.com/windmill-labs/windmill/issues/7833)) ([df0ae90](https://github.com/windmill-labs/windmill/commit/df0ae90a2c97de6f895142da1e189a9a7279f3fb)) +* mark job cleanup integration tests as ignored in CI ([4a1e61f](https://github.com/windmill-labs/windmill/commit/4a1e61f2f9a82b9279af8d0aded5683322f5f262)) + +## [1.628.2](https://github.com/windmill-labs/windmill/compare/v1.628.1...v1.628.2) (2026-02-06) + + +### Bug Fixes + +* execute CONCURRENTLY statements individually in migrations ([7a7b118](https://github.com/windmill-labs/windmill/commit/7a7b118bf36e8086b2df5a940e99e1c031f57c81)) + +## [1.628.1](https://github.com/windmill-labs/windmill/compare/v1.628.0...v1.628.1) (2026-02-06) + + +### Bug Fixes + +* prevent deadlock in consolidate live index migration ([f39b28a](https://github.com/windmill-labs/windmill/commit/f39b28ac416cfdc2420a58b549ee07479f316493)) +* use concurrent index ops to prevent deadlock on upgrade ([9967f83](https://github.com/windmill-labs/windmill/commit/9967f835ab0cba04bdad4f72b7df786bd1b02fa0)) + +## [1.628.0](https://github.com/windmill-labs/windmill/compare/v1.627.0...v1.628.0) (2026-02-06) + + +### Features + +* kafka trigger kerberos/gssapi support ([#7815](https://github.com/windmill-labs/windmill/issues/7815)) ([795e2be](https://github.com/windmill-labs/windmill/commit/795e2bebe65db9c6f721e7cd24af1446aeb896ab)) + + +### Bug Fixes + +* make notify_event trigger functions SECURITY DEFINER ([#7826](https://github.com/windmill-labs/windmill/issues/7826)) ([33fb08c](https://github.com/windmill-labs/windmill/commit/33fb08cf3d08c4a6b86f32b3ae8bf2df8c1adcaa)) +* prevent schedule pool connection exhaustion ([#7821](https://github.com/windmill-labs/windmill/issues/7821)) ([e655a06](https://github.com/windmill-labs/windmill/commit/e655a065637b288080118661650bc14641dd0c6f)) + +## [1.627.0](https://github.com/windmill-labs/windmill/compare/v1.626.0...v1.627.0) (2026-02-05) + + +### Features + +* mssql integrated auth (gssapi) ([#7760](https://github.com/windmill-labs/windmill/issues/7760)) ([afa6e7a](https://github.com/windmill-labs/windmill/commit/afa6e7ab5bb26972acbfe19af41dd3e6ac5df363)) +* restriction rulesets for workspaces ([#7791](https://github.com/windmill-labs/windmill/issues/7791)) ([a1cd02d](https://github.com/windmill-labs/windmill/commit/a1cd02d7f80c97eb07eda04113f3aae815fada69)) + + +### Bug Fixes + +* allow unauthed private pwsh repo ([#7817](https://github.com/windmill-labs/windmill/issues/7817)) ([476e6fd](https://github.com/windmill-labs/windmill/commit/476e6fd4bd2cdb062fa15a8c4048371ef4b31845)) +* fix asset grant ([e28c5b1](https://github.com/windmill-labs/windmill/commit/e28c5b18af25710b0ed3a3ffbceb18f3da76cd75)) + +## [1.626.0](https://github.com/windmill-labs/windmill/compare/v1.625.0...v1.626.0) (2026-02-05) + + +### Features + +* **local-dev:** create Claude skills when doing `wmill init` ([#7699](https://github.com/windmill-labs/windmill/issues/7699)) ([a7ce548](https://github.com/windmill-labs/windmill/commit/a7ce5484b8ec386af59f501c36e5ffc147e1d34a)) + + +### Bug Fixes + +* fix DB Manager not working with db resources with 4+ path segments ([#7809](https://github.com/windmill-labs/windmill/issues/7809)) ([3476ef4](https://github.com/windmill-labs/windmill/commit/3476ef4b9c795fb8511a83f2297154a4f55aa829)) +* fix indexer select performances busiying the db ([c3815c8](https://github.com/windmill-labs/windmill/commit/c3815c8c99d5b7d6b2dfc0e3b59d1ba51022ee39)) +* **frontend:** dedicated worker broken runnable select ([#7808](https://github.com/windmill-labs/windmill/issues/7808)) ([6f6ff9d](https://github.com/windmill-labs/windmill/commit/6f6ff9d4217e99901562b01eb258c7ccdcb0e3f4)) +* python client oidc pass session token ([#7799](https://github.com/windmill-labs/windmill/issues/7799)) ([b468603](https://github.com/windmill-labs/windmill/commit/b468603f6bc52961057fbd88539eb379a19efd9d)) + ## [1.625.0](https://github.com/windmill-labs/windmill/compare/v1.624.0...v1.625.0) (2026-02-04) diff --git a/Dockerfile b/Dockerfile index 572e84e265..5a1271df46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN make FROM ${RUST_IMAGE} AS rust_base -RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm +RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm mold clang RUN apt-get -y update \ && apt-get install -y \ @@ -97,7 +97,7 @@ ARG features="" COPY --from=planner /windmill/recipe.json recipe.json -RUN apt-get update && apt-get install -y libxml2-dev=2.9.* libxmlsec1-dev=1.2.* clang=1:14.0-55.* libclang-dev=1:14.0-55.* cmake=3.25.* && \ +RUN apt-get update && apt-get install -y libxml2-dev=2.9.* libxmlsec1-dev=1.2.* libkrb5-dev libsasl2-dev libcurl4-openssl-dev clang=1:14.0-55.* libclang-dev=1:14.0-55.* cmake=3.25.* && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -149,7 +149,7 @@ ENV PATH /usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH RUN apt-get update \ - && apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common tini \ + && apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common tini libsasl2-modules-gssapi-mit \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/backend/.cargo/config.toml b/backend/.cargo/config.toml index 234ac9a50f..f8b0a5e32a 100644 --- a/backend/.cargo/config.toml +++ b/backend/.cargo/config.toml @@ -1,6 +1,14 @@ [build] incremental = true +[target.x86_64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-C", "link-arg=-fuse-ld=mold"] + +[target.aarch64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-C", "link-arg=-fuse-ld=mold"] + [target.x86_64-apple-darwin] rustflags = [ "-C", "link-arg=-undefined", diff --git a/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json b/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json index 5b58cedc72..d3f1c39c7a 100644 --- a/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json +++ b/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-0300afc35a880eef163dfdfd9d5299fac14562ee8595c792f3c30d042fa2d3eb.json b/backend/.sqlx/query-0300afc35a880eef163dfdfd9d5299fac14562ee8595c792f3c30d042fa2d3eb.json new file mode 100644 index 0000000000..cf0856b775 --- /dev/null +++ b/backend/.sqlx/query-0300afc35a880eef163dfdfd9d5299fac14562ee8595c792f3c30d042fa2d3eb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO postgres_trigger (\n path, script_path, is_flow, workspace_id, edited_by, email,\n postgres_resource_path, replication_slot_name, publication_name\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "0300afc35a880eef163dfdfd9d5299fac14562ee8595c792f3c30d042fa2d3eb" +} diff --git a/backend/.sqlx/query-036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d.json b/backend/.sqlx/query-036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d.json deleted file mode 100644 index dedd3fd272..0000000000 --- a/backend/.sqlx/query-036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_9 ON v2_job (workspace_id, created_at DESC) where kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "036c84bb9ce72748956bc9c18fbe276444fab025a281dc4784596b0e31c1cb9d" -} diff --git a/backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json b/backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json new file mode 100644 index 0000000000..c957d28526 --- /dev/null +++ b/backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n capture\n WHERE\n workspace_id = $1\n AND created_at <= (\n SELECT\n created_at\n FROM\n capture\n WHERE\n workspace_id = $1\n ORDER BY\n created_at DESC\n OFFSET $2\n LIMIT 1\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2" +} diff --git a/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json b/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json index 8ecc409fc8..97e39aa0e5 100644 --- a/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json +++ b/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json @@ -17,7 +17,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-079b7f09da952cee6fa594c91a932781327f2b10a6dc4c6c82414ef0a610a10c.json b/backend/.sqlx/query-079b7f09da952cee6fa594c91a932781327f2b10a6dc4c6c82414ef0a610a10c.json new file mode 100644 index 0000000000..a6f9a5468a --- /dev/null +++ b/backend/.sqlx/query-079b7f09da952cee6fa594c91a932781327f2b10a6dc4c6c82414ef0a610a10c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE http_trigger SET script_path = $1 WHERE workspace_id = $2 AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "079b7f09da952cee6fa594c91a932781327f2b10a6dc4c6c82414ef0a610a10c" +} diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index a032a87239..b925141065 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -30,7 +30,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-0b238fcc4737fa31312bdd2baa7f42617f0727eea4228364f01c3b9c7056da3e.json b/backend/.sqlx/query-0b238fcc4737fa31312bdd2baa7f42617f0727eea4228364f01c3b9c7056da3e.json new file mode 100644 index 0000000000..66b26b7e76 --- /dev/null +++ b/backend/.sqlx/query-0b238fcc4737fa31312bdd2baa7f42617f0727eea4228364f01c3b9c7056da3e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM websocket_trigger WHERE workspace_id = $1 AND mode = 'disabled'::trigger_mode", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0b238fcc4737fa31312bdd2baa7f42617f0727eea4228364f01c3b9c7056da3e" +} diff --git a/backend/.sqlx/query-0cb84cbb9083d967cc8be1cccab5be61080c1003eef51eea41862b25c2b93de6.json b/backend/.sqlx/query-0cb84cbb9083d967cc8be1cccab5be61080c1003eef51eea41862b25c2b93de6.json deleted file mode 100644 index 35e2731f9a..0000000000 --- a/backend/.sqlx/query-0cb84cbb9083d967cc8be1cccab5be61080c1003eef51eea41862b25c2b93de6.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO metrics (id, value) VALUES ('telemetry', $1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0cb84cbb9083d967cc8be1cccab5be61080c1003eef51eea41862b25c2b93de6" -} diff --git a/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json b/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json deleted file mode 100644 index a867ace2cb..0000000000 --- a/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nats_trigger \n SET \n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE \n workspace_id = $11 AND path = $12\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd" -} diff --git a/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json b/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json deleted file mode 100644 index 2f17b5e8db..0000000000 --- a/backend/.sqlx/query-0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS queue_sort", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "0efb16cbf130ec6e9922ecc82a95b252449bd569df374e40ce8820fc3d75a0f0" -} diff --git a/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json b/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json deleted file mode 100644 index ec215e0f29..0000000000 --- a/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE \n workspace_id = $12 AND \n path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "JsonbArray", - { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - }, - "Varchar", - "Jsonb", - "Jsonb", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592" -} diff --git a/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json b/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json new file mode 100644 index 0000000000..de4dd4a7e3 --- /dev/null +++ b/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by)\n VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2" +} diff --git a/backend/.sqlx/query-1074c6c98e6a0c83ac04172a39abea21c793f58947051d39931d4da0868a1d77.json b/backend/.sqlx/query-1074c6c98e6a0c83ac04172a39abea21c793f58947051d39931d4da0868a1d77.json new file mode 100644 index 0000000000..74535ae3c7 --- /dev/null +++ b/backend/.sqlx/query-1074c6c98e6a0c83ac04172a39abea21c793f58947051d39931d4da0868a1d77.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO email_trigger (\n path, local_part, workspaced_local_part, script_path,\n is_flow, workspace_id, edited_by, email\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1074c6c98e6a0c83ac04172a39abea21c793f58947051d39931d4da0868a1d77" +} diff --git a/backend/.sqlx/query-10cdcf3155d8e58350e7df9acdbc73893c57716663a579b8bd120b542cc186f7.json b/backend/.sqlx/query-10cdcf3155d8e58350e7df9acdbc73893c57716663a579b8bd120b542cc186f7.json new file mode 100644 index 0000000000..2a03a4d05d --- /dev/null +++ b/backend/.sqlx/query-10cdcf3155d8e58350e7df9acdbc73893c57716663a579b8bd120b542cc186f7.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM v2_job_completed WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "10cdcf3155d8e58350e7df9acdbc73893c57716663a579b8bd120b542cc186f7" +} diff --git a/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json b/backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json similarity index 50% rename from backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json rename to backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json index 9fe2580534..b02b877e92 100644 --- a/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json +++ b/backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_completed (\n id,\n workspace_id,\n started_at,\n completed_at,\n duration_ms,\n result,\n deleted,\n canceled_by,\n canceled_reason,\n flow_status,\n memory_peak,\n status,\n worker,\n workflow_as_code_status,\n result_columns,\n retries,\n extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ", + "query": "\n INSERT INTO v2_job_completed (\n id, workspace_id, started_at, completed_at, duration_ms, result, deleted,\n canceled_by, canceled_reason, flow_status, memory_peak, status, worker,\n workflow_as_code_status, result_columns, retries, extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -38,5 +38,5 @@ }, "nullable": [] }, - "hash": "73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca" + "hash": "141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e" } diff --git a/backend/.sqlx/query-14276a040cb4db88d71fccdc3579e8c0bb132b70668301b535872d1632753e30.json b/backend/.sqlx/query-14276a040cb4db88d71fccdc3579e8c0bb132b70668301b535872d1632753e30.json index d4f7afa966..bed99ef1b7 100644 --- a/backend/.sqlx/query-14276a040cb4db88d71fccdc3579e8c0bb132b70668301b535872d1632753e30.json +++ b/backend/.sqlx/query-14276a040cb4db88d71fccdc3579e8c0bb132b70668301b535872d1632753e30.json @@ -122,7 +122,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069.json b/backend/.sqlx/query-15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069.json deleted file mode 100644 index 543f8c634c..0000000000 --- a/backend/.sqlx/query-15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069" -} diff --git a/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json b/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json new file mode 100644 index 0000000000..1fdc78c472 --- /dev/null +++ b/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email)\n VALUES ($1, $2, $3, $4::trigger_kind, $5, $6)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec" +} diff --git a/backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json b/backend/.sqlx/query-19478c2a3c05b2ed17b110f8ff58d48ce3ede5fdd7f8d64840f8c62cf89f99e7.json similarity index 63% rename from backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json rename to backend/.sqlx/query-19478c2a3c05b2ed17b110f8ff58d48ce3ede5fdd7f8d64840f8c62cf89f99e7.json index eafb17af4e..d59f60daf4 100644 --- a/backend/.sqlx/query-48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8.json +++ b/backend/.sqlx/query-19478c2a3c05b2ed17b110f8ff58d48ce3ede5fdd7f8d64840f8c62cf89f99e7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n script_lang as \"language: _\", \n COUNT(*) as \"count!\", \n SUM(duration_ms)::BIGINT as \"total_duration!\" \n FROM \n v2_job_completed\n JOIN \n v2_job \n ON \n v2_job.id = v2_job_completed.id \n WHERE \n created_at > NOW() - INTERVAL '48 hours' \n GROUP BY \n script_lang\n ", + "query": "\n SELECT\n script_lang as \"language: _\",\n COUNT(*) as \"count!\",\n SUM(duration_ms)::BIGINT as \"total_duration!\"\n FROM\n v2_job_completed\n JOIN\n v2_job\n ON\n v2_job.id = v2_job_completed.id\n WHERE\n created_at >= CURRENT_DATE - INTERVAL '1 day'\n AND created_at < CURRENT_DATE\n GROUP BY\n script_lang\n ", "describe": { "columns": [ { @@ -59,5 +59,5 @@ null ] }, - "hash": "48242fc46cb8bc9e4d41c788568634619565e4ef342051af23760c11edac5ee8" + "hash": "19478c2a3c05b2ed17b110f8ff58d48ce3ede5fdd7f8d64840f8c62cf89f99e7" } diff --git a/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json b/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json index 43c68f8c5a..30e071f8ce 100644 --- a/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json +++ b/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json @@ -40,7 +40,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json b/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json deleted file mode 100644 index 62c47f2503..0000000000 --- a/backend/.sqlx/query-19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS v2_job_queue_suspend ON v2_job_queue (workspace_id, suspend) WHERE suspend > 0;", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "19f0ccadd3ee44719a781ea0d73ea4e45f5b2c3d5c0aa5dbecf9ea9838881b74" -} diff --git a/backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json b/backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json deleted file mode 100644 index a50cd946c5..0000000000 --- a/backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n mode = $9\n WHERE \n workspace_id = $10 AND path = $11\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb", - { - "Custom": { - "name": "trigger_mode", - "kind": { - "Enum": [ - "enabled", - "disabled", - "suspended" - ] - } - } - }, - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca" -} diff --git a/backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json b/backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json new file mode 100644 index 0000000000..36707c3421 --- /dev/null +++ b/backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs, workflow_as_code_status)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962" +} diff --git a/backend/.sqlx/query-1bceaf6e9f25745b7f70128054ca81d68f3d56d4782e99e05b4f1cb362683514.json b/backend/.sqlx/query-1bceaf6e9f25745b7f70128054ca81d68f3d56d4782e99e05b4f1cb362683514.json deleted file mode 100644 index 1be74d47f2..0000000000 --- a/backend/.sqlx/query-1bceaf6e9f25745b7f70128054ca81d68f3d56d4782e99e05b4f1cb362683514.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH inserted_concurrency_counter AS (\n INSERT INTO concurrency_counter (concurrency_id, job_uuids) \n VALUES ($1, '{}'::jsonb)\n ON CONFLICT DO NOTHING\n )\n INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "1bceaf6e9f25745b7f70128054ca81d68f3d56d4782e99e05b4f1cb362683514" -} diff --git a/backend/.sqlx/query-1cad2ebfbdc46f9c0d93329897a71701f17a33b708b334d909563c9a0dcc9c23.json b/backend/.sqlx/query-1cad2ebfbdc46f9c0d93329897a71701f17a33b708b334d909563c9a0dcc9c23.json new file mode 100644 index 0000000000..49ae463657 --- /dev/null +++ b/backend/.sqlx/query-1cad2ebfbdc46f9c0d93329897a71701f17a33b708b334d909563c9a0dcc9c23.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id, script_path,\n is_flow, workspace_id, edited_by, email\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1cad2ebfbdc46f9c0d93329897a71701f17a33b708b334d909563c9a0dcc9c23" +} diff --git a/backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json b/backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json similarity index 72% rename from backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json rename to backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json index 8eb4477540..851c98634b 100644 --- a/backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json +++ b/backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n gcp_resource_path, \n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE \n ", + "query": "\n SELECT\n gcp_resource_path,\n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE\n ", "describe": { "columns": [ { @@ -96,5 +96,5 @@ true ] }, - "hash": "f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7" + "hash": "1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d" } diff --git a/backend/.sqlx/query-1d4bb4f53574ef95ef1016b760f849ec2372ac6a21bb2556d17a96dc72ea4980.json b/backend/.sqlx/query-1d4bb4f53574ef95ef1016b760f849ec2372ac6a21bb2556d17a96dc72ea4980.json new file mode 100644 index 0000000000..2257984d66 --- /dev/null +++ b/backend/.sqlx/query-1d4bb4f53574ef95ef1016b760f849ec2372ac6a21bb2556d17a96dc72ea4980.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id,\n edited_by, email, server_id, error\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d4bb4f53574ef95ef1016b760f849ec2372ac6a21bb2556d17a96dc72ea4980" +} diff --git a/backend/.sqlx/query-2031c5138a785367e5127180ccb6734efa41ee6cb3b4819c1c517798b2b23e4a.json b/backend/.sqlx/query-2031c5138a785367e5127180ccb6734efa41ee6cb3b4819c1c517798b2b23e4a.json new file mode 100644 index 0000000000..b691e2ad23 --- /dev/null +++ b/backend/.sqlx/query-2031c5138a785367e5127180ccb6734efa41ee6cb3b4819c1c517798b2b23e4a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE email_trigger SET script_path = $1, local_part = $2 WHERE workspace_id = $3 AND path = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2031c5138a785367e5127180ccb6734efa41ee6cb3b4819c1c517798b2b23e4a" +} diff --git a/backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json b/backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json deleted file mode 100644 index fb8302c8e6..0000000000 --- a/backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE \n workspace_id = $12 AND \n path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - { - "Custom": { - "name": "delivery_mode", - "kind": { - "Enum": [ - "push", - "pull" - ] - } - } - }, - "Jsonb", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - { - "Custom": { - "name": "trigger_mode", - "kind": { - "Enum": [ - "enabled", - "disabled", - "suspended" - ] - } - } - }, - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb", - "Bool", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229" -} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json new file mode 100644 index 0000000000..b0354db034 --- /dev/null +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT id, runnable_path, trigger_kind AS \"trigger_kind: String\",\n args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE runnable_path = $1\n AND trigger_kind = $2::job_trigger_kind\n ORDER BY created_at DESC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "trigger_kind: String", + "type_info": { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "job_trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "schedule", + "app", + "ui", + "postgres", + "sqs", + "gcp", + "mqtt", + "nextcloud", + "google" + ] + } + } + } + ] + }, + "nullable": [ + false, + true, + true, + true + ] + }, + "hash": "212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8" +} diff --git a/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json b/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json new file mode 100644 index 0000000000..235f255dd1 --- /dev/null +++ b/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email)\n VALUES ($1, $2, $3, $4::trigger_kind, $5, $6)\n ON CONFLICT (workspace_id, path, is_flow, trigger_kind)\n DO UPDATE SET owner = $5, email = $6, server_id = NULL, error = NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b" +} diff --git a/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json b/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json index 242b358ff5..7bd7367d8d 100644 --- a/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json +++ b/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json @@ -40,7 +40,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json b/backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json deleted file mode 100644 index 1bb99fad8e..0000000000 --- a/backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1\n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17" -} diff --git a/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json b/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json index 4e48ae281c..c51175e921 100644 --- a/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json +++ b/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json @@ -16,7 +16,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json b/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json index 128a435c8b..5a4cd89224 100644 --- a/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json +++ b/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json @@ -11,7 +11,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json b/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json index 9abe8af9af..08c6c1cbc2 100644 --- a/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json +++ b/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json @@ -11,7 +11,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json b/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json index 5d7c0e2abc..1199e9441f 100644 --- a/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json +++ b/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json @@ -15,7 +15,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json b/backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json deleted file mode 100644 index 6e58d3d386..0000000000 --- a/backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM email_trigger \n WHERE \n ((workspaced_local_part IS TRUE AND workspace_id || '-' || local_part = $1) \n OR (workspaced_local_part IS FALSE AND local_part = $1))\n AND ($2::TEXT IS NULL OR path != $2)\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07" -} diff --git a/backend/.sqlx/query-2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d.json b/backend/.sqlx/query-2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d.json deleted file mode 100644 index 236c2fa597..0000000000 --- a/backend/.sqlx/query-2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS log_file_hostname_log_ts_idx", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d" -} diff --git a/backend/.sqlx/query-dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5.json b/backend/.sqlx/query-2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac.json similarity index 64% rename from backend/.sqlx/query-dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5.json rename to backend/.sqlx/query-2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac.json index 101a63dca6..3007e7435b 100644 --- a/backend/.sqlx/query-dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5.json +++ b/backend/.sqlx/query-2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, worker_group, vcpus, memory, ping_at, started_at, custom_tags FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", + "query": "SELECT worker, worker_instance, worker_group, vcpus, memory, ping_at, started_at, custom_tags, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", "describe": { "columns": [ { @@ -42,6 +42,21 @@ "ordinal": 7, "name": "custom_tags", "type_info": "TextArray" + }, + { + "ordinal": 8, + "name": "occupancy_rate_15s", + "type_info": "Float4" + }, + { + "ordinal": 9, + "name": "occupancy_rate_5m", + "type_info": "Float4" + }, + { + "ordinal": 10, + "name": "occupancy_rate_30m", + "type_info": "Float4" } ], "parameters": { @@ -55,8 +70,11 @@ true, false, false, + true, + true, + true, true ] }, - "hash": "dbe2025b2d7dfc985e8e5e1119fc5e5ab77c873acae357c4adaf5b82fe8f4bc5" + "hash": "2b3b634b15eb58b95ce26b5a591258b54fb7bf21ae85e7a390ad73489c2247ac" } diff --git a/backend/.sqlx/query-2f7ff8bf66826101546e11066039bd647f9016cec246117a8fa1bd76c44db6a3.json b/backend/.sqlx/query-2f7ff8bf66826101546e11066039bd647f9016cec246117a8fa1bd76c44db6a3.json new file mode 100644 index 0000000000..96a9e9be19 --- /dev/null +++ b/backend/.sqlx/query-2f7ff8bf66826101546e11066039bd647f9016cec246117a8fa1bd76c44db6a3.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT server_id, error FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "2f7ff8bf66826101546e11066039bd647f9016cec246117a8fa1bd76c44db6a3" +} diff --git a/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json b/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json index 8f6c8edf25..afd0f503bf 100644 --- a/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json +++ b/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-302bc55d0c227c5b12458ccde6569c4b531ff494d1ede0a655872ae05215f8a6.json b/backend/.sqlx/query-302bc55d0c227c5b12458ccde6569c4b531ff494d1ede0a655872ae05215f8a6.json new file mode 100644 index 0000000000..6bcd70915a --- /dev/null +++ b/backend/.sqlx/query-302bc55d0c227c5b12458ccde6569c4b531ff494d1ede0a655872ae05215f8a6.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM variable WHERE workspace_id = $1 AND path = $2 AND is_secret = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "302bc55d0c227c5b12458ccde6569c4b531ff494d1ede0a655872ae05215f8a6" +} diff --git a/backend/.sqlx/query-30fc636d349e046e4dce61a1db964e7e5a857b39584a4bd5b62fc4703215cfb8.json b/backend/.sqlx/query-30fc636d349e046e4dce61a1db964e7e5a857b39584a4bd5b62fc4703215cfb8.json new file mode 100644 index 0000000000..cdf46f5b67 --- /dev/null +++ b/backend/.sqlx/query-30fc636d349e046e4dce61a1db964e7e5a857b39584a4bd5b62fc4703215cfb8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT owner FROM capture_config WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "30fc636d349e046e4dce61a1db964e7e5a857b39584a4bd5b62fc4703215cfb8" +} diff --git a/backend/.sqlx/query-33fff66efe810ad7e92b36ca9b287938437182d7817387707e800519d9f5bafc.json b/backend/.sqlx/query-33fff66efe810ad7e92b36ca9b287938437182d7817387707e800519d9f5bafc.json new file mode 100644 index 0000000000..bc30d6189e --- /dev/null +++ b/backend/.sqlx/query-33fff66efe810ad7e92b36ca9b287938437182d7817387707e800519d9f5bafc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3 AND is_secret = true", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "33fff66efe810ad7e92b36ca9b287938437182d7817387707e800519d9f5bafc" +} diff --git a/backend/.sqlx/query-b2b2b7251be2b80207f47b10eeff78d61c84161caaf16b8fd25d82b97aac1186.json b/backend/.sqlx/query-3481e65196e500ad914f10b87884c19d8a3636ae955788c3366365194fe2ef57.json similarity index 52% rename from backend/.sqlx/query-b2b2b7251be2b80207f47b10eeff78d61c84161caaf16b8fd25d82b97aac1186.json rename to backend/.sqlx/query-3481e65196e500ad914f10b87884c19d8a3636ae955788c3366365194fe2ef57.json index 89f74fac61..cc01e8b2e5 100644 --- a/backend/.sqlx/query-b2b2b7251be2b80207f47b10eeff78d61c84161caaf16b8fd25d82b97aac1186.json +++ b/backend/.sqlx/query-3481e65196e500ad914f10b87884c19d8a3636ae955788c3366365194fe2ef57.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2", + "query": "DELETE FROM mqtt_trigger WHERE workspace_id = $1 AND path = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "b2b2b7251be2b80207f47b10eeff78d61c84161caaf16b8fd25d82b97aac1186" + "hash": "3481e65196e500ad914f10b87884c19d8a3636ae955788c3366365194fe2ef57" } diff --git a/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json b/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json deleted file mode 100644 index d804949078..0000000000 --- a/backend/.sqlx/query-3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS queue_sort_2", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "3738096c29ab9d964be8a74bfd14ff1d599049ebefdaf97a017c9cef8d52ce20" -} diff --git a/backend/.sqlx/query-37937656035a487aa252bad3bdba28c129943e1c3ccd5bc3c8f652435d1a0021.json b/backend/.sqlx/query-37937656035a487aa252bad3bdba28c129943e1c3ccd5bc3c8f652435d1a0021.json new file mode 100644 index 0000000000..9c13c608ef --- /dev/null +++ b/backend/.sqlx/query-37937656035a487aa252bad3bdba28c129943e1c3ccd5bc3c8f652435d1a0021.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE((SELECT value::bigint FROM global_settings WHERE name = 'retention_period_secs'), 2592000)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "37937656035a487aa252bad3bdba28c129943e1c3ccd5bc3c8f652435d1a0021" +} diff --git a/backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json b/backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json new file mode 100644 index 0000000000..3f5bd4aefb --- /dev/null +++ b/backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE email_trigger\n SET\n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n mode = $11\n WHERE\n workspace_id = $12 AND path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315" +} diff --git a/backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json b/backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json similarity index 52% rename from backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json rename to backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json index 6dd26cdb5a..0df550b57c 100644 --- a/backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json +++ b/backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_integrations \n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ", + "query": "\n UPDATE workspace_integrations\n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ", "describe": { "columns": [], "parameters": { @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -21,5 +22,5 @@ }, "nullable": [] }, - "hash": "3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f" + "hash": "3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3" } diff --git a/backend/.sqlx/query-3b5295a7c4b99aefa52c9a8ae1e0dd12bf4a0be1bf755caf7a1fa863e7950562.json b/backend/.sqlx/query-3d198a9319d8872a04f8ccdc129f8baf44d26dc97187ec08946a665d81ed89dd.json similarity index 64% rename from backend/.sqlx/query-3b5295a7c4b99aefa52c9a8ae1e0dd12bf4a0be1bf755caf7a1fa863e7950562.json rename to backend/.sqlx/query-3d198a9319d8872a04f8ccdc129f8baf44d26dc97187ec08946a665d81ed89dd.json index 82d5378e40..e0894c69eb 100644 --- a/backend/.sqlx/query-3b5295a7c4b99aefa52c9a8ae1e0dd12bf4a0be1bf755caf7a1fa863e7950562.json +++ b/backend/.sqlx/query-3d198a9319d8872a04f8ccdc129f8baf44d26dc97187ec08946a665d81ed89dd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM raw_app WHERE workspace_id = $1", + "query": "SELECT COUNT(*) FROM http_trigger WHERE workspace_id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "3b5295a7c4b99aefa52c9a8ae1e0dd12bf4a0be1bf755caf7a1fa863e7950562" + "hash": "3d198a9319d8872a04f8ccdc129f8baf44d26dc97187ec08946a665d81ed89dd" } diff --git a/backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json b/backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json new file mode 100644 index 0000000000..f83e856cec --- /dev/null +++ b/backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_queue (\n id, workspace_id, started_at, scheduled_for, running, canceled_by,\n canceled_reason, suspend, suspend_until, worker, extras, tag, priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Timestamptz", + "Timestamptz", + "Bool", + "Varchar", + "Text", + "Int4", + "Timestamptz", + "Varchar", + "Jsonb", + "Varchar", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c" +} diff --git a/backend/.sqlx/query-3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c.json b/backend/.sqlx/query-3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c.json deleted file mode 100644 index 2488154bbd..0000000000 --- a/backend/.sqlx/query-3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new_2", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "3dca0aded0ec744b084359e1a77dc4af312fda13d832f9e8b236655628e5b81c" -} diff --git a/backend/.sqlx/query-3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd.json b/backend/.sqlx/query-3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd.json deleted file mode 100644 index 7b0eb81646..0000000000 --- a/backend/.sqlx/query-3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY ix_v2_job_labels ON v2_job\n USING GIN (labels)\n WHERE labels IS NOT NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "3de870d80a3617e7b31209d922164bf957b8bac3b5c04626ec750b18f705cbfd" -} diff --git a/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json b/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json index e6dc6ff08b..fdcb10be1c 100644 --- a/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json +++ b/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json @@ -16,7 +16,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -51,7 +52,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json b/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json deleted file mode 100644 index e74dc52790..0000000000 --- a/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_queue (\n id,\n workspace_id,\n started_at,\n scheduled_for,\n running,\n canceled_by,\n canceled_reason,\n suspend,\n suspend_until,\n worker,\n extras,\n tag,\n priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Timestamptz", - "Timestamptz", - "Bool", - "Varchar", - "Text", - "Int4", - "Timestamptz", - "Varchar", - "Jsonb", - "Varchar", - "Int2" - ] - }, - "nullable": [] - }, - "hash": "3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947" -} diff --git a/backend/.sqlx/query-4012a31045d4ff7665913be12190307542c6eabbe264071a216683d18839e68c.json b/backend/.sqlx/query-4012a31045d4ff7665913be12190307542c6eabbe264071a216683d18839e68c.json new file mode 100644 index 0000000000..0b6565a28c --- /dev/null +++ b/backend/.sqlx/query-4012a31045d4ff7665913be12190307542c6eabbe264071a216683d18839e68c.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) FROM http_trigger\n WHERE workspace_id = $1 AND authentication_method = $2::authentication_method\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "4012a31045d4ff7665913be12190307542c6eabbe264071a216683d18839e68c" +} diff --git a/backend/.sqlx/query-40971d637c5b4d2af8e67872722880058cc067fca5e807ab3e1ed17d180cb7f7.json b/backend/.sqlx/query-40971d637c5b4d2af8e67872722880058cc067fca5e807ab3e1ed17d180cb7f7.json new file mode 100644 index 0000000000..c403e8ad92 --- /dev/null +++ b/backend/.sqlx/query-40971d637c5b4d2af8e67872722880058cc067fca5e807ab3e1ed17d180cb7f7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM email_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "40971d637c5b4d2af8e67872722880058cc067fca5e807ab3e1ed17d180cb7f7" +} diff --git a/backend/.sqlx/query-410e3c47a8a6144228d2200e6859ad8a15c443801eba563a7c069bcac2bd21aa.json b/backend/.sqlx/query-410e3c47a8a6144228d2200e6859ad8a15c443801eba563a7c069bcac2bd21aa.json new file mode 100644 index 0000000000..85245fdf67 --- /dev/null +++ b/backend/.sqlx/query-410e3c47a8a6144228d2200e6859ad8a15c443801eba563a7c069bcac2bd21aa.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM capture WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "410e3c47a8a6144228d2200e6859ad8a15c443801eba563a7c069bcac2bd21aa" +} diff --git a/backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json b/backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json new file mode 100644 index 0000000000..0df0167c42 --- /dev/null +++ b/backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM capture WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7" +} diff --git a/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json b/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json index e6d71f386f..537ecbeacf 100644 --- a/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json +++ b/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json @@ -24,7 +24,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-42c7f52fe822124bf22b4c300ee8014088225ca2bbf1835b6fb5e5b78e5647c8.json b/backend/.sqlx/query-42c7f52fe822124bf22b4c300ee8014088225ca2bbf1835b6fb5e5b78e5647c8.json new file mode 100644 index 0000000000..5bb6c8c8fe --- /dev/null +++ b/backend/.sqlx/query-42c7f52fe822124bf22b4c300ee8014088225ca2bbf1835b6fb5e5b78e5647c8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_client_ping FROM capture_config WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_client_ping", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "42c7f52fe822124bf22b4c300ee8014088225ca2bbf1835b6fb5e5b78e5647c8" +} diff --git a/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json b/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json new file mode 100644 index 0000000000..30c5ff7a49 --- /dev/null +++ b/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by)\n VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7)\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835" +} diff --git a/backend/.sqlx/query-458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929.json b/backend/.sqlx/query-458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929.json deleted file mode 100644 index 1acc3d1efc..0000000000 --- a/backend/.sqlx/query-458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "458e053b93b6e8fc56420fb1ad25cd2910d301b44b6e9c9a861dee3de352f929" -} diff --git a/backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json b/backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json deleted file mode 100644 index edb113673b..0000000000 --- a/backend/.sqlx/query-45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM metrics WHERE created_at < NOW() - INTERVAL '180 day'", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "45dde0dc2da12fe46b5975ed53b520fc4160c008c3d29fe7d99e609c0b6e3e6d" -} diff --git a/backend/.sqlx/query-464895f217536bf713ec381b6c3b7c0417161bf69e596e2e34fc8d9384824776.json b/backend/.sqlx/query-464895f217536bf713ec381b6c3b7c0417161bf69e596e2e34fc8d9384824776.json new file mode 100644 index 0000000000..bb7a05f73d --- /dev/null +++ b/backend/.sqlx/query-464895f217536bf713ec381b6c3b7c0417161bf69e596e2e34fc8d9384824776.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n script_lang as \"language: _\",\n COUNT(*) as \"count!\",\n SUM(duration_ms)::BIGINT as \"total_duration!\"\n FROM\n v2_job_completed\n JOIN\n v2_job\n ON\n v2_job.id = v2_job_completed.id\n WHERE\n created_at > NOW() - INTERVAL '48 hours'\n GROUP BY\n script_lang\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "language: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "total_duration!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + null, + null + ] + }, + "hash": "464895f217536bf713ec381b6c3b7c0417161bf69e596e2e34fc8d9384824776" +} diff --git a/backend/.sqlx/query-467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa.json b/backend/.sqlx/query-467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa.json new file mode 100644 index 0000000000..0b47ce7a7d --- /dev/null +++ b/backend/.sqlx/query-467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_logs WHERE job_id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "467c563cdc81dd57a01735a9f535d1096bdfd6c539d8c31462f8f2bd711ab0fa" +} diff --git a/backend/.sqlx/query-49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145.json b/backend/.sqlx/query-49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145.json deleted file mode 100644 index e2001954e1..0000000000 --- a/backend/.sqlx/query-49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_job_completed_completed_at ON v2_job_completed (completed_at DESC)", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145" -} diff --git a/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json b/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json new file mode 100644 index 0000000000..cfb975b838 --- /dev/null +++ b/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT path, owner, email, trigger_kind AS \"trigger_kind: String\"\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "trigger_kind: String", + "type_info": { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c" +} diff --git a/backend/.sqlx/query-4d4aa16b2a55e57f9376d0cb253e671525969fb5f528ae07576e9dc5e77af1f1.json b/backend/.sqlx/query-4d4aa16b2a55e57f9376d0cb253e671525969fb5f528ae07576e9dc5e77af1f1.json deleted file mode 100644 index 3dea443954..0000000000 --- a/backend/.sqlx/query-4d4aa16b2a55e57f9376d0cb253e671525969fb5f528ae07576e9dc5e77af1f1.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_job_workspace_id_created_at_new_7", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "4d4aa16b2a55e57f9376d0cb253e671525969fb5f528ae07576e9dc5e77af1f1" -} diff --git a/backend/.sqlx/query-4dc46c62177282965e207009467ff27ac005a79a052b8eb11544e85cbe3cd3c2.json b/backend/.sqlx/query-4dc46c62177282965e207009467ff27ac005a79a052b8eb11544e85cbe3cd3c2.json new file mode 100644 index 0000000000..22c3657abd --- /dev/null +++ b/backend/.sqlx/query-4dc46c62177282965e207009467ff27ac005a79a052b8eb11544e85cbe3cd3c2.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n v2_job.workspace_id,\n COUNT(*)::BIGINT as \"job_count!\"\n FROM v2_job_completed\n JOIN v2_job ON v2_job.id = v2_job_completed.id\n WHERE v2_job.created_at >= CURRENT_DATE - INTERVAL '1 day'\n AND v2_job.created_at < CURRENT_DATE\n AND v2_job.workspace_id NOT LIKE 'wm-fork%'\n GROUP BY v2_job.workspace_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "job_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "4dc46c62177282965e207009467ff27ac005a79a052b8eb11544e85cbe3cd3c2" +} diff --git a/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json index 789c0334d7..4e8ec56d12 100644 --- a/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json +++ b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json @@ -32,7 +32,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } @@ -70,7 +71,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-4f8bc8acc174e33a45eab9f046cd3e809a5026de15540a3347c464a03c079257.json b/backend/.sqlx/query-4f8bc8acc174e33a45eab9f046cd3e809a5026de15540a3347c464a03c079257.json new file mode 100644 index 0000000000..e8257f30aa --- /dev/null +++ b/backend/.sqlx/query-4f8bc8acc174e33a45eab9f046cd3e809a5026de15540a3347c464a03c079257.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT queue_url, aws_resource_path,\n aws_auth_resource_type AS \"aws_auth_resource_type: String\",\n mode AS \"mode: String\"\n FROM sqs_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "queue_url", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "aws_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "aws_auth_resource_type: String", + "type_info": { + "Custom": { + "name": "aws_auth_resource_type", + "kind": { + "Enum": [ + "oidc", + "credentials" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "4f8bc8acc174e33a45eab9f046cd3e809a5026de15540a3347c464a03c079257" +} diff --git a/backend/.sqlx/query-51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681.json b/backend/.sqlx/query-51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681.json deleted file mode 100644 index d2775c1c94..0000000000 --- a/backend/.sqlx/query-51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681" -} diff --git a/backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json b/backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json similarity index 59% rename from backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json rename to backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json index fd4e4e7f52..ebf1df39a3 100644 --- a/backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json +++ b/backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n oauth_data as \"oauth_data!: sqlx::types::Json\",\n service_name as \"service_name!: ServiceName\"\n FROM \n workspace_integrations \n WHERE \n workspace_id = $1\n ", + "query": "\n SELECT\n oauth_data as \"oauth_data!: sqlx::types::Json\",\n service_name as \"service_name!: ServiceName\"\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [ { @@ -16,7 +16,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -33,5 +34,5 @@ false ] }, - "hash": "013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e" + "hash": "5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee" } diff --git a/backend/.sqlx/query-5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958.json b/backend/.sqlx/query-5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958.json deleted file mode 100644 index 9a1b65ff52..0000000000 --- a/backend/.sqlx/query-5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_8 ON v2_job (workspace_id, created_at DESC) where kind in ('deploymentcallback') AND parent_job IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "5401c521b5e63b7d9e7bc51c19d116599f6bcedbe70f3bf346b482fe79501958" -} diff --git a/backend/.sqlx/query-572b712b9ff0231ca0f680eb3d6a7dedf30e8c98a87e861886300afbc95af6cf.json b/backend/.sqlx/query-572b712b9ff0231ca0f680eb3d6a7dedf30e8c98a87e861886300afbc95af6cf.json new file mode 100644 index 0000000000..c4b1a5d8b2 --- /dev/null +++ b/backend/.sqlx/query-572b712b9ff0231ca0f680eb3d6a7dedf30e8c98a87e861886300afbc95af6cf.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) FROM http_trigger\n WHERE workspace_id = $1\n AND (mode = 'enabled'::trigger_mode OR mode = 'suspended'::trigger_mode)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "572b712b9ff0231ca0f680eb3d6a7dedf30e8c98a87e861886300afbc95af6cf" +} diff --git a/backend/.sqlx/query-57b7236cae0b6a1940f4c2d4b202692450ee231488d9a55ca59ff53a6f674626.json b/backend/.sqlx/query-57b7236cae0b6a1940f4c2d4b202692450ee231488d9a55ca59ff53a6f674626.json new file mode 100644 index 0000000000..0b854e8bef --- /dev/null +++ b/backend/.sqlx/query-57b7236cae0b6a1940f4c2d4b202692450ee231488d9a55ca59ff53a6f674626.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id,\n edited_by, email\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "57b7236cae0b6a1940f4c2d4b202692450ee231488d9a55ca59ff53a6f674626" +} diff --git a/backend/.sqlx/query-57eca702e951f5303a74643c7ba64472e2c2a781fbb6366d998a0f1ca22fcdf2.json b/backend/.sqlx/query-57eca702e951f5303a74643c7ba64472e2c2a781fbb6366d998a0f1ca22fcdf2.json new file mode 100644 index 0000000000..e55868665f --- /dev/null +++ b/backend/.sqlx/query-57eca702e951f5303a74643c7ba64472e2c2a781fbb6366d998a0f1ca22fcdf2.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n path, route_path, route_path_key, script_path, is_flow,\n workspace_id, edited_by, email, http_method,\n authentication_method, is_static_website, workspaced_route,\n wrap_body, raw_string, mode\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method,\n $10::authentication_method, $11, $12, $13, $14, $15::trigger_mode)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Bool", + "Bool", + "Bool", + "Bool", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "57eca702e951f5303a74643c7ba64472e2c2a781fbb6366d998a0f1ca22fcdf2" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5a9cf9cc229f7b7ddbee2485cda4e9f7a91240a00f10d24d3f28d5b722f68768.json b/backend/.sqlx/query-5a9cf9cc229f7b7ddbee2485cda4e9f7a91240a00f10d24d3f28d5b722f68768.json new file mode 100644 index 0000000000..a971ecc058 --- /dev/null +++ b/backend/.sqlx/query-5a9cf9cc229f7b7ddbee2485cda4e9f7a91240a00f10d24d3f28d5b722f68768.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH inserted_concurrency_counter AS (\n INSERT INTO concurrency_counter (concurrency_id, job_uuids)\n VALUES ($1, '{}'::jsonb)\n ON CONFLICT DO NOTHING\n )\n INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "5a9cf9cc229f7b7ddbee2485cda4e9f7a91240a00f10d24d3f28d5b722f68768" +} diff --git a/backend/.sqlx/query-5bf200f2c8db25ddf231b564503c6c70f7f3958564a79bb0c6b3863b1ebb0cbf.json b/backend/.sqlx/query-5bf200f2c8db25ddf231b564503c6c70f7f3958564a79bb0c6b3863b1ebb0cbf.json index 6d3941ad69..9422e3e8d0 100644 --- a/backend/.sqlx/query-5bf200f2c8db25ddf231b564503c6c70f7f3958564a79bb0c6b3863b1ebb0cbf.json +++ b/backend/.sqlx/query-5bf200f2c8db25ddf231b564503c6c70f7f3958564a79bb0c6b3863b1ebb0cbf.json @@ -245,7 +245,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6.json b/backend/.sqlx/query-5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6.json deleted file mode 100644 index 5dc78dab56..0000000000 --- a/backend/.sqlx/query-5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_workspace_id_completed_at_all ON v2_job_completed (workspace_id, completed_at DESC)", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "5c403799772e1c44bd00e5376893a6428b3c7cbdef4af70a6660c1a5900d17a6" -} diff --git a/backend/.sqlx/query-5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505.json b/backend/.sqlx/query-5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505.json deleted file mode 100644 index c22e4bc5cf..0000000000 --- a/backend/.sqlx/query-5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_v2_job_workspace_id_created_at ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow', 'singlestepflow') AND parent_job IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "5d99d2b058d4896f9ac1cea04fa35c003bc4e897ee01746d6f17aeee387d1505" -} diff --git a/backend/.sqlx/query-c4b3ab3d87f158b20f0557f78c6307c50620425aeb43636e19931c142b5ce0d7.json b/backend/.sqlx/query-5eaa109879b27bb2eaa738fb3f359ade1a05164beabd1c1284b6d656c8c2e114.json similarity index 51% rename from backend/.sqlx/query-c4b3ab3d87f158b20f0557f78c6307c50620425aeb43636e19931c142b5ce0d7.json rename to backend/.sqlx/query-5eaa109879b27bb2eaa738fb3f359ade1a05164beabd1c1284b6d656c8c2e114.json index 8f54593038..6be3903161 100644 --- a/backend/.sqlx/query-c4b3ab3d87f158b20f0557f78c6307c50620425aeb43636e19931c142b5ce0d7.json +++ b/backend/.sqlx/query-5eaa109879b27bb2eaa738fb3f359ade1a05164beabd1c1284b6d656c8c2e114.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)", + "query": "SELECT COUNT(*) FROM capture WHERE workspace_id = $1 AND path = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "exists", - "type_info": "Bool" + "name": "count", + "type_info": "Int8" } ], "parameters": { @@ -19,5 +19,5 @@ null ] }, - "hash": "c4b3ab3d87f158b20f0557f78c6307c50620425aeb43636e19931c142b5ce0d7" + "hash": "5eaa109879b27bb2eaa738fb3f359ade1a05164beabd1c1284b6d656c8c2e114" } diff --git a/backend/.sqlx/query-5f1df080a871127172271a72ef006005e1575ad43ea7539c186730ef8ac49e01.json b/backend/.sqlx/query-5f1df080a871127172271a72ef006005e1575ad43ea7539c186730ef8ac49e01.json new file mode 100644 index 0000000000..54f4acae30 --- /dev/null +++ b/backend/.sqlx/query-5f1df080a871127172271a72ef006005e1575ad43ea7539c186730ef8ac49e01.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM capture_config WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5f1df080a871127172271a72ef006005e1575ad43ea7539c186730ef8ac49e01" +} diff --git a/backend/.sqlx/query-60a666a7cffce2d7631682095ccfd31e99997e88374496d4800eaa7b5152a464.json b/backend/.sqlx/query-60a666a7cffce2d7631682095ccfd31e99997e88374496d4800eaa7b5152a464.json new file mode 100644 index 0000000000..fe16d3a2bf --- /dev/null +++ b/backend/.sqlx/query-60a666a7cffce2d7631682095ccfd31e99997e88374496d4800eaa7b5152a464.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT delivery_type AS \"delivery_type: String\",\n delivery_config\n FROM gcp_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "delivery_type: String", + "type_info": { + "Custom": { + "name": "delivery_mode", + "kind": { + "Enum": [ + "push", + "pull" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "delivery_config", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "60a666a7cffce2d7631682095ccfd31e99997e88374496d4800eaa7b5152a464" +} diff --git a/backend/.sqlx/query-624162e98be630ccc363f10bb0f966baf7fd7d7c74a7a9d2ae08e587967575d4.json b/backend/.sqlx/query-624162e98be630ccc363f10bb0f966baf7fd7d7c74a7a9d2ae08e587967575d4.json new file mode 100644 index 0000000000..821a3927ad --- /dev/null +++ b/backend/.sqlx/query-624162e98be630ccc363f10bb0f966baf7fd7d7c74a7a9d2ae08e587967575d4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "624162e98be630ccc363f10bb0f966baf7fd7d7c74a7a9d2ae08e587967575d4" +} diff --git a/backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json b/backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json deleted file mode 100644 index fce1c4942a..0000000000 --- a/backend/.sqlx/query-6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path_2", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "6536214f31e9d600e868b01385d8c6395e2440ea27553b7ccb18d7149b106728" -} diff --git a/backend/.sqlx/query-653e7a8509194f60e442848346ee106c8c24445130880be87c12a764e7921bff.json b/backend/.sqlx/query-653e7a8509194f60e442848346ee106c8c24445130880be87c12a764e7921bff.json new file mode 100644 index 0000000000..633f88b8d6 --- /dev/null +++ b/backend/.sqlx/query-653e7a8509194f60e442848346ee106c8c24445130880be87c12a764e7921bff.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n path, route_path, script_path,\n http_method AS \"http_method: String\",\n authentication_method AS \"authentication_method: String\"\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "http_method: String", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 4, + "name": "authentication_method: String", + "type_info": { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false + ] + }, + "hash": "653e7a8509194f60e442848346ee106c8c24445130880be87c12a764e7921bff" +} diff --git a/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json b/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json new file mode 100644 index 0000000000..5f9be1cba9 --- /dev/null +++ b/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n owner,\n email\n FROM\n capture_config\n WHERE\n workspace_id = $1\n AND path = $2\n AND is_flow = $3\n AND trigger_kind = $4\n AND last_client_ping > NOW() - INTERVAL '10 seconds'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + } + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147" +} diff --git a/backend/.sqlx/query-69274380aadda7b9333f38012c1b637cfbd67d9b1088bb6c8b366bedbf220bb7.json b/backend/.sqlx/query-69274380aadda7b9333f38012c1b637cfbd67d9b1088bb6c8b366bedbf220bb7.json new file mode 100644 index 0000000000..66d0f2ace1 --- /dev/null +++ b/backend/.sqlx/query-69274380aadda7b9333f38012c1b637cfbd67d9b1088bb6c8b366bedbf220bb7.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n COUNT(DISTINCT CASE WHEN NOT operator THEN email END)::INT as \"author_count!\",\n COUNT(DISTINCT CASE WHEN operator THEN email END)::INT as \"operator_count!\"\n FROM usr\n WHERE email IN (\n SELECT DISTINCT username FROM audit\n WHERE timestamp > NOW() - INTERVAL '1 month'\n AND operation IN ('users.login', 'oauth.login', 'users.token.refresh')\n )\n AND workspace_id NOT LIKE 'wm-fork%'\n GROUP BY workspace_id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "author_count!", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "operator_count!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "69274380aadda7b9333f38012c1b637cfbd67d9b1088bb6c8b366bedbf220bb7" +} diff --git a/backend/.sqlx/query-6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca.json b/backend/.sqlx/query-6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca.json deleted file mode 100644 index bf127e8cca..0000000000 --- a/backend/.sqlx/query-6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_job_created_at", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "6ad24aef02d86ea507d232ad0bbf798240d4fcdd52ce2745532ca30f736d25ca" -} diff --git a/backend/.sqlx/query-6afa076744233fc5e92188ff978990fa3a704afe3eec523f4e203f7f6e247261.json b/backend/.sqlx/query-6afa076744233fc5e92188ff978990fa3a704afe3eec523f4e203f7f6e247261.json new file mode 100644 index 0000000000..b34e0205c9 --- /dev/null +++ b/backend/.sqlx/query-6afa076744233fc5e92188ff978990fa3a704afe3eec523f4e203f7f6e247261.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n path, route_path, route_path_key, script_path, is_flow,\n workspace_id, edited_by, email, http_method,\n authentication_method, is_static_website, workspaced_route,\n wrap_body, raw_string\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method,\n $10::authentication_method, $11, $12, $13, $14)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Bool", + "Bool", + "Bool", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "6afa076744233fc5e92188ff978990fa3a704afe3eec523f4e203f7f6e247261" +} diff --git a/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json b/backend/.sqlx/query-6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451.json similarity index 89% rename from backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json rename to backend/.sqlx/query-6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451.json index 2ce0b8015a..1e49b8405c 100644 --- a/backend/.sqlx/query-5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0.json +++ b/backend/.sqlx/query-6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5) \n RETURNING id\n ", + "query": "\n INSERT INTO workspace_dependencies(name, workspace_id, content, language, description)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id\n ", "describe": { "columns": [ { @@ -53,5 +53,5 @@ false ] }, - "hash": "5bec60e207a5933aa301f87c0afaeaa8e9a3a2c64e23ab42100d48039ba422b0" + "hash": "6c10c38a5d5e560d8d76159ce0ae118cf7191b06d72285e71256dd6d70c19451" } diff --git a/backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json b/backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json new file mode 100644 index 0000000000..659a783821 --- /dev/null +++ b/backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "JsonbArray", + "JsonbArray", + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7" +} diff --git a/backend/.sqlx/query-6f12be65a4fe3eb39292164363f557de9cef7017dcfbcd40370b849a288c52e3.json b/backend/.sqlx/query-6f12be65a4fe3eb39292164363f557de9cef7017dcfbcd40370b849a288c52e3.json deleted file mode 100644 index fb92042662..0000000000 --- a/backend/.sqlx/query-6f12be65a4fe3eb39292164363f557de9cef7017dcfbcd40370b849a288c52e3.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS labeled_jobs_on_jobs", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "6f12be65a4fe3eb39292164363f557de9cef7017dcfbcd40370b849a288c52e3" -} diff --git a/backend/.sqlx/query-70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9.json b/backend/.sqlx/query-70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9.json new file mode 100644 index 0000000000..923f1a1af5 --- /dev/null +++ b/backend/.sqlx/query-70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory WHERE workspace_id = $1 AND conversation_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "70659efcf6c06a7aabd2078829c5b07b4c7d0b47ad8ecbcf0766ece62bac36f9" +} diff --git a/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json b/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json new file mode 100644 index 0000000000..48ea42f8a2 --- /dev/null +++ b/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) FROM capture\n WHERE workspace_id = $1 AND path = $2 AND trigger_kind = $3::trigger_kind\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626" +} diff --git a/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json b/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json new file mode 100644 index 0000000000..8aacd8a805 --- /dev/null +++ b/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by)\n VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83" +} diff --git a/backend/.sqlx/query-74c11f5a0315424574fe3e7429f967f7b94f9d9db7be628f9cb411d789085711.json b/backend/.sqlx/query-74c11f5a0315424574fe3e7429f967f7b94f9d9db7be628f9cb411d789085711.json new file mode 100644 index 0000000000..1b33e89eb1 --- /dev/null +++ b/backend/.sqlx/query-74c11f5a0315424574fe3e7429f967f7b94f9d9db7be628f9cb411d789085711.json @@ -0,0 +1,54 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO http_trigger (\n path, route_path, route_path_key, script_path, is_flow,\n workspace_id, edited_by, email, http_method,\n authentication_method, is_static_website, workspaced_route,\n wrap_body, raw_string\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method,\n $10::authentication_method, $11, $12, $13, $14)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + { + "Custom": { + "name": "authentication_method", + "kind": { + "Enum": [ + "none", + "windmill", + "api_key", + "basic_http", + "custom_script", + "signature" + ] + } + } + }, + "Bool", + "Bool", + "Bool", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "74c11f5a0315424574fe3e7429f967f7b94f9d9db7be628f9cb411d789085711" +} diff --git a/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json b/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json index cd795e6fec..59b56ceda7 100644 --- a/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json +++ b/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json @@ -40,7 +40,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-75ce9753a8acc8eccb3f2e0baaa5a871f866a8d21961fac8a003bc40c406ee79.json b/backend/.sqlx/query-75ce9753a8acc8eccb3f2e0baaa5a871f866a8d21961fac8a003bc40c406ee79.json new file mode 100644 index 0000000000..64d95155d1 --- /dev/null +++ b/backend/.sqlx/query-75ce9753a8acc8eccb3f2e0baaa5a871f866a8d21961fac8a003bc40c406ee79.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO schedule (\n workspace_id, path, edited_by, schedule, enabled,\n script_path, is_flow, email, timezone\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "75ce9753a8acc8eccb3f2e0baaa5a871f866a8d21961fac8a003bc40c406ee79" +} diff --git a/backend/.sqlx/query-7c1ae9cac13d1387cfa94149f039054dd8c30c16b4657e73cdb0d7c7f1cb3b6d.json b/backend/.sqlx/query-7c1ae9cac13d1387cfa94149f039054dd8c30c16b4657e73cdb0d7c7f1cb3b6d.json new file mode 100644 index 0000000000..cf07041291 --- /dev/null +++ b/backend/.sqlx/query-7c1ae9cac13d1387cfa94149f039054dd8c30c16b4657e73cdb0d7c7f1cb3b6d.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id,\n edited_by, email, mode\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8::trigger_mode)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "7c1ae9cac13d1387cfa94149f039054dd8c30c16b4657e73cdb0d7c7f1cb3b6d" +} diff --git a/backend/.sqlx/query-7e3bfb33fb771aec39b43a7550091ce7c9b1261b52d10f4a7f3273fed3c916df.json b/backend/.sqlx/query-7e3bfb33fb771aec39b43a7550091ce7c9b1261b52d10f4a7f3273fed3c916df.json new file mode 100644 index 0000000000..23652a2571 --- /dev/null +++ b/backend/.sqlx/query-7e3bfb33fb771aec39b43a7550091ce7c9b1261b52d10f4a7f3273fed3c916df.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT kafka_resource_path, topics, group_id, mode AS \"mode: String\"\n FROM kafka_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kafka_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "topics", + "type_info": "VarcharArray" + }, + { + "ordinal": 2, + "name": "group_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "7e3bfb33fb771aec39b43a7550091ce7c9b1261b52d10f4a7f3273fed3c916df" +} diff --git a/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json b/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json index 393a920b7c..8c380df861 100644 --- a/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json +++ b/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json @@ -27,7 +27,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-8002f7f1521846056278f18e8b003fe48a4d3d827ac8b212e0f3d3bc2ed42f37.json b/backend/.sqlx/query-8002f7f1521846056278f18e8b003fe48a4d3d827ac8b212e0f3d3bc2ed42f37.json new file mode 100644 index 0000000000..130f56d5b1 --- /dev/null +++ b/backend/.sqlx/query-8002f7f1521846056278f18e8b003fe48a4d3d827ac8b212e0f3d3bc2ed42f37.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT script_path, local_part FROM email_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "local_part", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8002f7f1521846056278f18e8b003fe48a4d3d827ac8b212e0f3d3bc2ed42f37" +} diff --git a/backend/.sqlx/query-8364505b9a4d0d6f7406e4a5425360682ac2295bbf6746c7641141ed7bf47bf6.json b/backend/.sqlx/query-8364505b9a4d0d6f7406e4a5425360682ac2295bbf6746c7641141ed7bf47bf6.json new file mode 100644 index 0000000000..0df0d1fc7c --- /dev/null +++ b/backend/.sqlx/query-8364505b9a4d0d6f7406e4a5425360682ac2295bbf6746c7641141ed7bf47bf6.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM email_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8364505b9a4d0d6f7406e4a5425360682ac2295bbf6746c7641141ed7bf47bf6" +} diff --git a/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json b/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json new file mode 100644 index 0000000000..3012e9ef77 --- /dev/null +++ b/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email)\n VALUES ($1, $2, $3, $4::trigger_kind, $5, $6)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53" +} diff --git a/backend/.sqlx/query-84bbd11f7bb0c65dcfb2e12038b8a9cb51d303480ac3c5f8a5e833a53c40ed9b.json b/backend/.sqlx/query-84bbd11f7bb0c65dcfb2e12038b8a9cb51d303480ac3c5f8a5e833a53c40ed9b.json deleted file mode 100644 index 73a9ddd438..0000000000 --- a/backend/.sqlx/query-84bbd11f7bb0c65dcfb2e12038b8a9cb51d303480ac3c5f8a5e833a53c40ed9b.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO raw_app\n (workspace_id, path, summary, extra_perms, data)\n VALUES ($1, $2, $3, '{}', $4)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "84bbd11f7bb0c65dcfb2e12038b8a9cb51d303480ac3c5f8a5e833a53c40ed9b" -} diff --git a/backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json b/backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json new file mode 100644 index 0000000000..f42d028128 --- /dev/null +++ b/backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n gcp_trigger\n SET\n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE\n workspace_id = $12 AND\n path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "delivery_mode", + "kind": { + "Enum": [ + "push", + "pull" + ] + } + } + }, + "Jsonb", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb", + "Bool", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f" +} diff --git a/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json b/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json index 9350442134..5a61f82be3 100644 --- a/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json +++ b/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json @@ -35,7 +35,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-899a162192d93c7fb7d47d38f4fb0fadcafc2bb0c61efa8ee496af7252d18270.json b/backend/.sqlx/query-899a162192d93c7fb7d47d38f4fb0fadcafc2bb0c61efa8ee496af7252d18270.json new file mode 100644 index 0000000000..73e8150580 --- /dev/null +++ b/backend/.sqlx/query-899a162192d93c7fb7d47d38f4fb0fadcafc2bb0c61efa8ee496af7252d18270.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT mqtt_resource_path, client_version AS \"client_version: String\",\n script_path, mode AS \"mode: String\"\n FROM mqtt_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mqtt_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "client_version: String", + "type_info": { + "Custom": { + "name": "mqtt_client_version", + "kind": { + "Enum": [ + "v3", + "v5" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "899a162192d93c7fb7d47d38f4fb0fadcafc2bb0c61efa8ee496af7252d18270" +} diff --git a/backend/.sqlx/query-8c3cc09c1bbb6209467c75723dd02e97dddb99789422012e85e86c3151a9f2e9.json b/backend/.sqlx/query-8c3cc09c1bbb6209467c75723dd02e97dddb99789422012e85e86c3151a9f2e9.json new file mode 100644 index 0000000000..1ef649375d --- /dev/null +++ b/backend/.sqlx/query-8c3cc09c1bbb6209467c75723dd02e97dddb99789422012e85e86c3151a9f2e9.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id,\n script_path, is_flow, workspace_id, edited_by, email\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8c3cc09c1bbb6209467c75723dd02e97dddb99789422012e85e86c3151a9f2e9" +} diff --git a/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json b/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json deleted file mode 100644 index 5979389314..0000000000 --- a/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO job_logs (\n job_id,\n workspace_id,\n logs,\n log_offset,\n log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Text", - "Int4", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c" -} diff --git a/backend/.sqlx/query-6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258.json b/backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json similarity index 68% rename from backend/.sqlx/query-6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258.json rename to backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json index 85af8d8bab..497bb3ac8f 100644 --- a/backend/.sqlx/query-6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258.json +++ b/backend/.sqlx/query-8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n j.parent_job,\n j.created_by,\n cj.duration_ms,\n cj.status = 'success' OR cj.status = 'skipped' AS success,\n j.runnable_id AS script_hash,\n j.runnable_path AS script_path,\n j.args,\n cj.result,\n cj.deleted,\n cj.status = 'canceled' AS canceled,\n cj.canceled_by,\n cj.canceled_reason,\n j.kind AS job_kind,\n CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path,\n j.permissioned_as,\n j.flow_step_id IS NOT NULL AS is_flow_step,\n j.script_lang AS language,\n cj.status = 'skipped' AS is_skipped,\n j.permissioned_as_email AS email,\n j.visible_to_owner,\n cj.memory_peak AS mem_peak,\n j.tag,\n cj.completed_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(j.created_by), 0) + \n COALESCE(LENGTH(j.runnable_path), 0) + \n COALESCE(LENGTH(j.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_job_completed AS cj\n JOIN v2_job AS j ON cj.id = j.id\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.completed_at > $1 AND cj.completed_at < $3)\n OR cj.id = ANY($2)\n ORDER BY completed_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY completed_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY completed_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n completed_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4 OR row_num = 1\n ORDER BY completed_at ASC", + "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n j.parent_job,\n j.created_by,\n cj.duration_ms,\n cj.status = 'success' OR cj.status = 'skipped' AS success,\n j.runnable_id AS script_hash,\n j.runnable_path AS script_path,\n j.args,\n cj.result,\n cj.deleted,\n cj.status = 'canceled' AS canceled,\n cj.canceled_by,\n cj.canceled_reason,\n j.kind AS job_kind,\n CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path,\n j.permissioned_as,\n j.flow_step_id IS NOT NULL AS is_flow_step,\n j.script_lang AS language,\n cj.status = 'skipped' AS is_skipped,\n j.permissioned_as_email AS email,\n j.visible_to_owner,\n cj.memory_peak AS mem_peak,\n j.tag,\n cj.completed_at,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(j.created_by), 0) + \n COALESCE(LENGTH(j.runnable_path), 0) + \n COALESCE(pg_column_size(j.args), 0) +\n COALESCE(pg_column_size(cj.result), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_job_completed AS cj\n JOIN v2_job AS j ON cj.id = j.id\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE ((cj.completed_at > $1 AND cj.completed_at < $3)\n OR cj.id = ANY($2))\n AND COALESCE(pg_column_size(cj.result), 0) <= 512000\n AND COALESCE(pg_column_size(j.args), 0) <= 512000\n ORDER BY completed_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY completed_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY completed_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n completed_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $4 OR row_num = 1\n ORDER BY completed_at ASC", "describe": { "columns": [ { @@ -250,5 +250,5 @@ true ] }, - "hash": "6c03fc7e623afcdb11b55390ac79d9dd236e694c0af5af0ebd90df940d893258" + "hash": "8d444380516faf56369cdbff645904e20477aa597e30891926af42eee56ab081" } diff --git a/backend/.sqlx/query-8e3684fd0d1670733d4d758ea94aef9bbf8a544826d80e6c7a41fb2c30478064.json b/backend/.sqlx/query-8e3684fd0d1670733d4d758ea94aef9bbf8a544826d80e6c7a41fb2c30478064.json new file mode 100644 index 0000000000..d40f6db54e --- /dev/null +++ b/backend/.sqlx/query-8e3684fd0d1670733d4d758ea94aef9bbf8a544826d80e6c7a41fb2c30478064.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM global_settings WHERE name = 'workspace_telemetry_enabled'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "8e3684fd0d1670733d4d758ea94aef9bbf8a544826d80e6c7a41fb2c30478064" +} diff --git a/backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json b/backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json similarity index 58% rename from backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json rename to backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json index 7e1e4b10ef..62f8d3100f 100644 --- a/backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json +++ b/backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n oauth_data \n FROM \n workspace_integrations \n WHERE \n workspace_id = $1 AND\n service_name = $2\n ", + "query": "\n SELECT\n oauth_data\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1 AND\n service_name = $2\n ", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -28,5 +29,5 @@ false ] }, - "hash": "1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f" + "hash": "9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e" } diff --git a/backend/.sqlx/query-9238c54d7cf53e2812936d36d98dc3e2d2b1bf4d7884720fafcbb531790f0c64.json b/backend/.sqlx/query-9238c54d7cf53e2812936d36d98dc3e2d2b1bf4d7884720fafcbb531790f0c64.json new file mode 100644 index 0000000000..e82d09a905 --- /dev/null +++ b/backend/.sqlx/query-9238c54d7cf53e2812936d36d98dc3e2d2b1bf4d7884720fafcbb531790f0c64.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE websocket_trigger SET error = NULL WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9238c54d7cf53e2812936d36d98dc3e2d2b1bf4d7884720fafcbb531790f0c64" +} diff --git a/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json b/backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json similarity index 54% rename from backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json rename to backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json index 5f7957c57f..09aadbda1e 100644 --- a/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json +++ b/backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = NULL\n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = $4 AND \n server_id IS NULL\n ", + "query": "\n UPDATE capture_config SET last_client_ping = NOW()\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4::trigger_kind\n ", "describe": { "columns": [], "parameters": { @@ -24,7 +24,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } @@ -33,5 +34,5 @@ }, "nullable": [] }, - "hash": "dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e" + "hash": "940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a" } diff --git a/backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json b/backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json new file mode 100644 index 0000000000..529824e4b2 --- /dev/null +++ b/backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM\n http_trigger\n WHERE\n workspace_id = $1 AND\n path = $2\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707" +} diff --git a/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json b/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json deleted file mode 100644 index 088d2317c9..0000000000 --- a/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_runtime (\n id, ping, memory_peak\n ) VALUES (\n $1, $2, $3\n )\n ON CONFLICT (id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Timestamptz", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739" -} diff --git a/backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json b/backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json similarity index 70% rename from backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json rename to backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json index 9cdbef80a5..a189bc4da2 100644 --- a/backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json +++ b/backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2 \n AND oauth_data IS NOT NULL\n )\n ", + "query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2\n AND oauth_data IS NOT NULL\n )\n ", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -28,5 +29,5 @@ null ] }, - "hash": "92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5" + "hash": "95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7" } diff --git a/backend/.sqlx/query-97c60a4193a75b611db41543e7fe6da81fc631cc6ac43576f8a18afedad4d4a4.json b/backend/.sqlx/query-97c60a4193a75b611db41543e7fe6da81fc631cc6ac43576f8a18afedad4d4a4.json new file mode 100644 index 0000000000..b5c19d2017 --- /dev/null +++ b/backend/.sqlx/query-97c60a4193a75b611db41543e7fe6da81fc631cc6ac43576f8a18afedad4d4a4.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO nats_trigger (\n path, nats_resource_path, subjects, script_path,\n is_flow, workspace_id, edited_by, email, use_jetstream\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "97c60a4193a75b611db41543e7fe6da81fc631cc6ac43576f8a18afedad4d4a4" +} diff --git a/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json index 9759bad4d4..66a487695f 100644 --- a/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json +++ b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json @@ -32,7 +32,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-9c7580fb38351518cb57ed6207495a7e6e17b49e4315a8ed0e36bfee16f49929.json b/backend/.sqlx/query-9c7580fb38351518cb57ed6207495a7e6e17b49e4315a8ed0e36bfee16f49929.json new file mode 100644 index 0000000000..d4c9ce7f56 --- /dev/null +++ b/backend/.sqlx/query-9c7580fb38351518cb57ed6207495a7e6e17b49e4315a8ed0e36bfee16f49929.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT error FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "9c7580fb38351518cb57ed6207495a7e6e17b49e4315a8ed0e36bfee16f49929" +} diff --git a/backend/.sqlx/query-9dd0cf1627f9e767af8758f26d8bf9e96a48e45daf4a3c54f7b81a7b92d39431.json b/backend/.sqlx/query-9dd0cf1627f9e767af8758f26d8bf9e96a48e45daf4a3c54f7b81a7b92d39431.json deleted file mode 100644 index ff9881bb4e..0000000000 --- a/backend/.sqlx/query-9dd0cf1627f9e767af8758f26d8bf9e96a48e45daf4a3c54f7b81a7b92d39431.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT pg_try_advisory_lock(4242)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "pg_try_advisory_lock", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "9dd0cf1627f9e767af8758f26d8bf9e96a48e45daf4a3c54f7b81a7b92d39431" -} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index 79c8f0b45d..b6fe1b7fa0 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -30,7 +30,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json b/backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json new file mode 100644 index 0000000000..776ff7b5e3 --- /dev/null +++ b/backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE nats_trigger\n SET\n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9" +} diff --git a/backend/.sqlx/query-a086a5fa28dcff80911c7e5de73f204fc42625e58b967b0376ff7b382e38fa11.json b/backend/.sqlx/query-a086a5fa28dcff80911c7e5de73f204fc42625e58b967b0376ff7b382e38fa11.json new file mode 100644 index 0000000000..bd61858de6 --- /dev/null +++ b/backend/.sqlx/query-a086a5fa28dcff80911c7e5de73f204fc42625e58b967b0376ff7b382e38fa11.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT script_path FROM mqtt_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a086a5fa28dcff80911c7e5de73f204fc42625e58b967b0376ff7b382e38fa11" +} diff --git a/backend/.sqlx/query-a1745a4f525b251d2f5a602ab2b2ede46b4471e21b11f607573a844013911abe.json b/backend/.sqlx/query-a1745a4f525b251d2f5a602ab2b2ede46b4471e21b11f607573a844013911abe.json index a25f845f91..33636da608 100644 --- a/backend/.sqlx/query-a1745a4f525b251d2f5a602ab2b2ede46b4471e21b11f607573a844013911abe.json +++ b/backend/.sqlx/query-a1745a4f525b251d2f5a602ab2b2ede46b4471e21b11f607573a844013911abe.json @@ -155,7 +155,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-a3e75f0309be42aca0fd74834f34b3f18dbb388bd8b9bc88b99aebedae9c3fec.json b/backend/.sqlx/query-a3e75f0309be42aca0fd74834f34b3f18dbb388bd8b9bc88b99aebedae9c3fec.json new file mode 100644 index 0000000000..66af36c0fd --- /dev/null +++ b/backend/.sqlx/query-a3e75f0309be42aca0fd74834f34b3f18dbb388bd8b9bc88b99aebedae9c3fec.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_result_stream_v2 WHERE job_id NOT IN (SELECT id FROM v2_job_queue) RETURNING job_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "a3e75f0309be42aca0fd74834f34b3f18dbb388bd8b9bc88b99aebedae9c3fec" +} diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index da6d213748..03ee58ca3f 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -185,7 +185,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-a80a17cda8ad722a97f16526d8576b76c7377066a06dba98a2921a68928aa644.json b/backend/.sqlx/query-a80a17cda8ad722a97f16526d8576b76c7377066a06dba98a2921a68928aa644.json new file mode 100644 index 0000000000..cd334ca6dd --- /dev/null +++ b/backend/.sqlx/query-a80a17cda8ad722a97f16526d8576b76c7377066a06dba98a2921a68928aa644.json @@ -0,0 +1,84 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT gcp_resource_path, topic_id, subscription_id,\n delivery_type AS \"delivery_type: String\",\n subscription_mode AS \"subscription_mode: String\",\n mode AS \"mode: String\"\n FROM gcp_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "gcp_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "topic_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "subscription_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "delivery_type: String", + "type_info": { + "Custom": { + "name": "delivery_mode", + "kind": { + "Enum": [ + "push", + "pull" + ] + } + } + } + }, + { + "ordinal": 4, + "name": "subscription_mode: String", + "type_info": { + "Custom": { + "name": "gcp_subscription_mode", + "kind": { + "Enum": [ + "create_update", + "existing" + ] + } + } + } + }, + { + "ordinal": 5, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "a80a17cda8ad722a97f16526d8576b76c7377066a06dba98a2921a68928aa644" +} diff --git a/backend/.sqlx/query-a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8.json b/backend/.sqlx/query-a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8.json deleted file mode 100644 index 1d4913e8d9..0000000000 --- a/backend/.sqlx/query-a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_by_workspace ON alerts (workspace_id);", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8" -} diff --git a/backend/.sqlx/query-a9ce39c0327607a1523424c838858fce213ef8976e3ffdc3b4bd88e92799e0af.json b/backend/.sqlx/query-a9ce39c0327607a1523424c838858fce213ef8976e3ffdc3b4bd88e92799e0af.json new file mode 100644 index 0000000000..2b376b621a --- /dev/null +++ b/backend/.sqlx/query-a9ce39c0327607a1523424c838858fce213ef8976e3ffdc3b4bd88e92799e0af.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT postgres_resource_path, replication_slot_name, publication_name, mode AS \"mode: String\"\n FROM postgres_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "postgres_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "replication_slot_name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "publication_name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "a9ce39c0327607a1523424c838858fce213ef8976e3ffdc3b4bd88e92799e0af" +} diff --git a/backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json b/backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json similarity index 59% rename from backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json rename to backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json index 33e0d6b0ff..24efa05c77 100644 --- a/backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json +++ b/backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n mode, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", + "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n email,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n )\n VALUES (\n $1,\n $2,\n $3,\n $4,\n $5,\n $6,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", "describe": { "columns": [], "parameters": { @@ -47,5 +47,5 @@ }, "nullable": [] }, - "hash": "447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba" + "hash": "aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9" } diff --git a/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json b/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json deleted file mode 100644 index 7587b063d9..0000000000 --- a/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool", - { - "Custom": { - "name": "trigger_kind", - "kind": { - "Enum": [ - "webhook", - "http", - "websocket", - "kafka", - "email", - "nats", - "postgres", - "sqs", - "mqtt", - "gcp", - "default_email", - "nextcloud" - ] - } - } - } - ] - }, - "nullable": [ - null - ] - }, - "hash": "ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30" -} diff --git a/backend/.sqlx/query-afed809b7dc7dc3c6c5b21c9899f9c46d11f8891807c2dac10b8b7edf72a197e.json b/backend/.sqlx/query-afed809b7dc7dc3c6c5b21c9899f9c46d11f8891807c2dac10b8b7edf72a197e.json new file mode 100644 index 0000000000..513e083511 --- /dev/null +++ b/backend/.sqlx/query-afed809b7dc7dc3c6c5b21c9899f9c46d11f8891807c2dac10b8b7edf72a197e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM websocket_trigger WHERE workspace_id = $1 AND mode = 'enabled'::trigger_mode", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "afed809b7dc7dc3c6c5b21c9899f9c46d11f8891807c2dac10b8b7edf72a197e" +} diff --git a/backend/.sqlx/query-34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be.json b/backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json similarity index 68% rename from backend/.sqlx/query-34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be.json rename to backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json index 8b01b0af3d..054521168e 100644 --- a/backend/.sqlx/query-34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be.json +++ b/backend/.sqlx/query-b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n j.parent_job,\n j.created_by,\n cj.duration_ms,\n cj.status = 'success' OR cj.status = 'skipped' AS success,\n j.runnable_id AS script_hash,\n j.runnable_path AS script_path,\n j.args,\n cj.completed_at,\n cj.result,\n cj.deleted,\n cj.status = 'canceled' AS canceled,\n cj.canceled_by,\n cj.canceled_reason,\n j.kind AS job_kind,\n CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path,\n j.permissioned_as,\n j.flow_step_id IS NOT NULL AS is_flow_step,\n j.script_lang AS language,\n cj.status = 'skipped' AS is_skipped,\n j.permissioned_as_email AS email,\n j.visible_to_owner,\n cj.memory_peak AS mem_peak,\n j.tag,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(j.created_by), 0) + \n COALESCE(LENGTH(j.runnable_path), 0) + \n COALESCE(LENGTH(j.args::text), 0) + \n COALESCE(LENGTH(cj.result::text), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_job_completed AS cj\n JOIN v2_job AS j ON cj.id = j.id\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.completed_at < $1\n ORDER BY completed_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT \n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY completed_at ASC \n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY completed_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n completed_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2 OR row_num = 1\n ORDER BY completed_at ASC", + "query": "WITH job_sizes AS (\n SELECT \n cj.id,\n cj.workspace_id,\n j.parent_job,\n j.created_by,\n cj.duration_ms,\n cj.status = 'success' OR cj.status = 'skipped' AS success,\n j.runnable_id AS script_hash,\n j.runnable_path AS script_path,\n j.args,\n cj.completed_at,\n cj.result,\n cj.deleted,\n cj.status = 'canceled' AS canceled,\n cj.canceled_by,\n cj.canceled_reason,\n j.kind AS job_kind,\n CASE WHEN j.trigger_kind = 'schedule'::job_trigger_kind THEN j.trigger END AS schedule_path,\n j.permissioned_as,\n j.flow_step_id IS NOT NULL AS is_flow_step,\n j.script_lang AS language,\n cj.status = 'skipped' AS is_skipped,\n j.permissioned_as_email AS email,\n j.visible_to_owner,\n cj.memory_peak AS mem_peak,\n j.tag,\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset,\n job_logs.log_file_index,\n -- Estimate size in bytes based on actual data characteristics\n (36 + -- UUID\n COALESCE(LENGTH(cj.workspace_id), 0) + \n COALESCE(LENGTH(j.created_by), 0) + \n COALESCE(LENGTH(j.runnable_path), 0) + \n COALESCE(pg_column_size(j.args), 0) +\n COALESCE(pg_column_size(cj.result), 0) + \n COALESCE(LENGTH(job_logs.logs), 0) + \n 200) AS estimated_size_bytes -- Other fields overhead\n FROM v2_job_completed AS cj\n JOIN v2_job AS j ON cj.id = j.id\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.completed_at < $1\n AND COALESCE(pg_column_size(cj.result), 0) <= 512000\n AND COALESCE(pg_column_size(j.args), 0) <= 512000\n ORDER BY completed_at ASC\n LIMIT 5000\n ),\n cumulative_sizes AS (\n SELECT\n *,\n SUM(estimated_size_bytes) OVER (\n ORDER BY completed_at ASC\n ROWS UNBOUNDED PRECEDING\n ) AS cumulative_size_bytes,\n ROW_NUMBER() OVER (ORDER BY completed_at ASC) AS row_num\n FROM job_sizes\n )\n SELECT\n id AS \"id!\",\n workspace_id AS \"workspace_id!\",\n parent_job,\n created_by AS \"created_by!\",\n duration_ms AS \"duration_ms!\",\n success AS \"success!\",\n script_hash AS \"script_hash!: Option\",\n script_path,\n args AS \"args: sqlx::types::Json>>\",\n result AS \"result: sqlx::types::Json>\",\n deleted AS \"deleted!\",\n canceled AS \"canceled!\",\n canceled_by,\n canceled_reason,\n job_kind AS \"job_kind!: JobKind\",\n schedule_path,\n permissioned_as AS \"permissioned_as!\",\n is_flow_step AS \"is_flow_step!\",\n language AS \"language: ScriptLang\",\n is_skipped AS \"is_skipped!\",\n email AS \"email!\",\n visible_to_owner AS \"visible_to_owner!\",\n mem_peak,\n tag AS \"tag!\",\n completed_at AS \"created_at!\",\n started_at,\n logs,\n log_offset AS \"log_offset?\",\n log_file_index\n FROM cumulative_sizes\n WHERE cumulative_size_bytes <= $2 OR row_num = 1\n ORDER BY completed_at ASC", "describe": { "columns": [ { @@ -248,5 +248,5 @@ true ] }, - "hash": "34f4c05b844a83627a5c61edbfe070d0a52e20deaf550185206ae854cce274be" + "hash": "b23784ed2737c01aaf05efe42bcdcfc9579aa203005ead60322fb71fd33b15b0" } diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index d08af6ffdd..4fa7da00e0 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -160,7 +160,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-b3b161d077e723ce19bdcc0db0683ff3b4c0f0314fdacc417d841e0c88a1f3d1.json b/backend/.sqlx/query-b3b161d077e723ce19bdcc0db0683ff3b4c0f0314fdacc417d841e0c88a1f3d1.json new file mode 100644 index 0000000000..8409d2f932 --- /dev/null +++ b/backend/.sqlx/query-b3b161d077e723ce19bdcc0db0683ff3b4c0f0314fdacc417d841e0c88a1f3d1.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT script_path FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b3b161d077e723ce19bdcc0db0683ff3b4c0f0314fdacc417d841e0c88a1f3d1" +} diff --git a/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json b/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json index 54a4e3cd93..186822a000 100644 --- a/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json +++ b/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json @@ -24,7 +24,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json b/backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json new file mode 100644 index 0000000000..e30f73a14c --- /dev/null +++ b/backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE email_trigger\n SET\n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n mode = $9\n WHERE\n workspace_id = $10 AND path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1" +} diff --git a/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json b/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json deleted file mode 100644 index 1d8fa42e31..0000000000 --- a/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool", - "JsonbArray", - "JsonbArray", - "Jsonb", - "Varchar", - "Varchar", - "Bool", - "Bool", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa" -} diff --git a/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json b/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json index 96f59aab3f..e6ef13f492 100644 --- a/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json +++ b/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json @@ -21,7 +21,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -71,7 +72,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8.json b/backend/.sqlx/query-bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8.json deleted file mode 100644 index 88b910fb04..0000000000 --- a/backend/.sqlx/query-bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_audit_timestamps ON audit (timestamp DESC)", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "bb3b644d630dc2040fdbebfaf7649617c1e5a22fc418a2a6ee3276a09b9a8cf8" -} diff --git a/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json b/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json deleted file mode 100644 index d9ca7cfa52..0000000000 --- a/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM job_result_stream_v2 WHERE job_id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090" -} diff --git a/backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json b/backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json new file mode 100644 index 0000000000..0a314a59e3 --- /dev/null +++ b/backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM email_trigger\n WHERE\n ((workspaced_local_part IS TRUE AND workspace_id || '-' || local_part = $1)\n OR (workspaced_local_part IS FALSE AND local_part = $1))\n AND ($2::TEXT IS NULL OR path != $2)\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916" +} diff --git a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json similarity index 67% rename from backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json rename to backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 7f581135ed..4118760af2 100644 --- a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", + "query": "\n INSERT INTO v2_job (\n id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,\n permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,\n parent_job, root_job, script_lang, script_entrypoint_override, flow_step,\n flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,\n visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,\n timeout, priority, preprocessed, args, labels, pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -105,7 +105,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } @@ -125,5 +126,5 @@ }, "nullable": [] }, - "hash": "fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f" + "hash": "be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66" } diff --git a/backend/.sqlx/query-c1058d8816d139c63dd9c4a075ab63efc585942b30c9e853f2a5cff4cc9916cd.json b/backend/.sqlx/query-c1058d8816d139c63dd9c4a075ab63efc585942b30c9e853f2a5cff4cc9916cd.json deleted file mode 100644 index 450d18d524..0000000000 --- a/backend/.sqlx/query-c1058d8816d139c63dd9c4a075ab63efc585942b30c9e853f2a5cff4cc9916cd.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS script_not_archived ON script (workspace_id, path, created_at DESC) where archived = false;", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c1058d8816d139c63dd9c4a075ab63efc585942b30c9e853f2a5cff4cc9916cd" -} diff --git a/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json b/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json deleted file mode 100644 index ff4479af3f..0000000000 --- a/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_status (\n id, flow_status, flow_leaf_jobs, workflow_as_code_status\n ) VALUES (\n $1, $2, $3, $4\n )\n ON CONFLICT (id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d" -} diff --git a/backend/.sqlx/query-c239d9160efe295ee6e64759dcbbc890d887649eaf72656a8814b0c167645044.json b/backend/.sqlx/query-c239d9160efe295ee6e64759dcbbc890d887649eaf72656a8814b0c167645044.json new file mode 100644 index 0000000000..18e620e999 --- /dev/null +++ b/backend/.sqlx/query-c239d9160efe295ee6e64759dcbbc890d887649eaf72656a8814b0c167645044.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT url, script_path, mode AS \"mode: String\"\n FROM websocket_trigger WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "c239d9160efe295ee6e64759dcbbc890d887649eaf72656a8814b0c167645044" +} diff --git a/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json b/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json new file mode 100644 index 0000000000..07243717a0 --- /dev/null +++ b/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n capture_config\n SET\n server_id = $1,\n last_server_ping = now(),\n error = 'Connecting...'\n WHERE\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5 AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n RETURNING true\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e" +} diff --git a/backend/.sqlx/query-c458012c39f1327fb2bfd6b087ee6dbe1380eca532f50bab0cb89a91d42131cf.json b/backend/.sqlx/query-c458012c39f1327fb2bfd6b087ee6dbe1380eca532f50bab0cb89a91d42131cf.json deleted file mode 100644 index 86f7f8605e..0000000000 --- a/backend/.sqlx/query-c458012c39f1327fb2bfd6b087ee6dbe1380eca532f50bab0cb89a91d42131cf.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "ALTER TABLE v2_job ENABLE ROW LEVEL SECURITY", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c458012c39f1327fb2bfd6b087ee6dbe1380eca532f50bab0cb89a91d42131cf" -} diff --git a/backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json b/backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json deleted file mode 100644 index b3d20cccea..0000000000 --- a/backend/.sqlx/query-c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c481e5d63ebf1aa537cc4ce4e84f9a71af5996bc76f328b3ba1cf68a71880462" -} diff --git a/backend/.sqlx/query-c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9.json b/backend/.sqlx/query-c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9.json deleted file mode 100644 index 4d138db49c..0000000000 --- a/backend/.sqlx/query-c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON v2_job_completed USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_labels'", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c4bdbf1c6bc7d93db4cf2633105b088b781354cb7c02628d9f8ff7f9ea0e7ed9" -} diff --git a/backend/.sqlx/query-c659cfe81441bb1b62a9f23f5d9b7f5b1f354b6bb0a78d007114fed084cfe292.json b/backend/.sqlx/query-c659cfe81441bb1b62a9f23f5d9b7f5b1f354b6bb0a78d007114fed084cfe292.json new file mode 100644 index 0000000000..7036bdbb46 --- /dev/null +++ b/backend/.sqlx/query-c659cfe81441bb1b62a9f23f5d9b7f5b1f354b6bb0a78d007114fed084cfe292.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO sqs_trigger (\n path, queue_url, aws_resource_path, script_path,\n is_flow, workspace_id, edited_by, email\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c659cfe81441bb1b62a9f23f5d9b7f5b1f354b6bb0a78d007114fed084cfe292" +} diff --git a/backend/.sqlx/query-8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d.json b/backend/.sqlx/query-c6bcf0d9e211bc03e3338682295f4995e1d622917367c478742addd073245ad5.json similarity index 59% rename from backend/.sqlx/query-8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d.json rename to backend/.sqlx/query-c6bcf0d9e211bc03e3338682295f4995e1d622917367c478742addd073245ad5.json index 662bc5f6d6..c776b35994 100644 --- a/backend/.sqlx/query-8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d.json +++ b/backend/.sqlx/query-c6bcf0d9e211bc03e3338682295f4995e1d622917367c478742addd073245ad5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM _sqlx_migrations WHERE\n version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR\n version=20250201145631 OR version=20250201145632 OR version=20251006143821", + "query": "DELETE FROM _sqlx_migrations WHERE\n version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR\n version=20250201145631 OR version=20250201145632 OR version=20251006143821 OR\n version=20260207000001 OR version=20260207000002 OR version=20260207000003 OR version=20260207000004", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "8d4ad4ee75fb149c36a9f6a0c4cf5fd981473f45d1b71b4b8236e021f7c8682d" + "hash": "c6bcf0d9e211bc03e3338682295f4995e1d622917367c478742addd073245ad5" } diff --git a/backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json b/backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json new file mode 100644 index 0000000000..ef146c1cee --- /dev/null +++ b/backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE sqs_trigger\n SET\n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + { + "Custom": { + "name": "aws_auth_resource_type", + "kind": { + "Enum": [ + "oidc", + "credentials" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763" +} diff --git a/backend/.sqlx/query-c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007.json b/backend/.sqlx/query-c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007.json deleted file mode 100644 index d79a7f5465..0000000000 --- a/backend/.sqlx/query-c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_5 ON v2_job (workspace_id, created_at DESC) where kind in ('preview', 'flowpreview') AND parent_job IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "c7be5fa2eaf66147c1213046e615f5e9fd168ef1e3aba8af64b15341055d6007" -} diff --git a/backend/.sqlx/query-c825fa5c6e287068aeaad994c0b42b8ad59b9129f032c6b918c27426ab304f2b.json b/backend/.sqlx/query-c825fa5c6e287068aeaad994c0b42b8ad59b9129f032c6b918c27426ab304f2b.json new file mode 100644 index 0000000000..a68387f905 --- /dev/null +++ b/backend/.sqlx/query-c825fa5c6e287068aeaad994c0b42b8ad59b9129f032c6b918c27426ab304f2b.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_perms\nWHERE job_id NOT IN (SELECT id FROM v2_job_queue)\nRETURNING job_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "c825fa5c6e287068aeaad994c0b42b8ad59b9129f032c6b918c27426ab304f2b" +} diff --git a/backend/.sqlx/query-cabef30aa1e308bc656660c4ac0d3e77b020de05465f81a4fb4b0c07bd0d7439.json b/backend/.sqlx/query-cabef30aa1e308bc656660c4ac0d3e77b020de05465f81a4fb4b0c07bd0d7439.json new file mode 100644 index 0000000000..1e8f95d082 --- /dev/null +++ b/backend/.sqlx/query-cabef30aa1e308bc656660c4ac0d3e77b020de05465f81a4fb4b0c07bd0d7439.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT local_part, workspaced_local_part, script_path,\n mode AS \"mode: String\"\n FROM email_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "local_part", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspaced_local_part", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "cabef30aa1e308bc656660c4ac0d3e77b020de05465f81a4fb4b0c07bd0d7439" +} diff --git a/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json b/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json index 0e338633fe..8a0dacd864 100644 --- a/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json +++ b/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json @@ -11,7 +11,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json b/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json new file mode 100644 index 0000000000..2b39145512 --- /dev/null +++ b/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n capture_config\n SET\n last_server_ping = NULL\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n is_flow = $3 AND\n trigger_kind = $4 AND\n server_id IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1" +} diff --git a/backend/.sqlx/query-ce6ebdcaed35f77b1f0f7fe6601d9f4d5eb4b159051c2b7d5ff10d7e13dcac0e.json b/backend/.sqlx/query-ce6ebdcaed35f77b1f0f7fe6601d9f4d5eb4b159051c2b7d5ff10d7e13dcac0e.json deleted file mode 100644 index 9d87786d1b..0000000000 --- a/backend/.sqlx/query-ce6ebdcaed35f77b1f0f7fe6601d9f4d5eb4b159051c2b7d5ff10d7e13dcac0e.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_recent_login_activities \nON audit (timestamp, username) \nWHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "ce6ebdcaed35f77b1f0f7fe6601d9f4d5eb4b159051c2b7d5ff10d7e13dcac0e" -} diff --git a/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json b/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json index b0cef08da3..b71bf15f85 100644 --- a/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json +++ b/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json b/backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json deleted file mode 100644 index 1060c7af74..0000000000 --- a/backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n mode = $11\n WHERE \n workspace_id = $12 AND path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb", - { - "Custom": { - "name": "trigger_mode", - "kind": { - "Enum": [ - "enabled", - "disabled", - "suspended" - ] - } - } - }, - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd" -} diff --git a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json similarity index 66% rename from backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json rename to backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index 94acdfb2d8..9d2f8d9a9f 100644 --- a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", + "query": "\n INSERT INTO v2_job (\n id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,\n permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,\n parent_job, root_job, script_lang, script_entrypoint_override, flow_step,\n flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,\n visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,\n timeout, priority, preprocessed, args, labels, pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -105,7 +105,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } @@ -125,5 +126,5 @@ }, "nullable": [] }, - "hash": "b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b" + "hash": "d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016" } diff --git a/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json b/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json similarity index 55% rename from backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json rename to backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json index 7ba4c70a26..67a38d1904 100644 --- a/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json +++ b/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5\n ", + "query": "\n UPDATE\n capture_config\n SET\n error = $1,\n server_id = NULL,\n last_server_ping = NULL\n WHERE\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5\n ", "describe": { "columns": [], "parameters": { @@ -25,7 +25,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } @@ -34,5 +35,5 @@ }, "nullable": [] }, - "hash": "db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899" + "hash": "d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106" } diff --git a/backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json b/backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json similarity index 64% rename from backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json rename to backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json index 1ce9e58cd4..af5ff0153c 100644 --- a/backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json +++ b/backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n route_path_key = $1\n AND workspace_id = $2 \n AND http_method = $3 \n AND ($4::TEXT IS NULL OR path != $4)\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM http_trigger\n WHERE\n route_path_key = $1\n AND workspace_id = $2\n AND http_method = $3\n AND ($4::TEXT IS NULL OR path != $4)\n )\n ", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294" + "hash": "d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033" } diff --git a/backend/.sqlx/query-d618ff98a2c7aceb6820007411ba7be2968d982ae84fe3294a6d78914afe3ecc.json b/backend/.sqlx/query-d618ff98a2c7aceb6820007411ba7be2968d982ae84fe3294a6d78914afe3ecc.json new file mode 100644 index 0000000000..6e735913c3 --- /dev/null +++ b/backend/.sqlx/query-d618ff98a2c7aceb6820007411ba7be2968d982ae84fe3294a6d78914afe3ecc.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT nats_resource_path, subjects, use_jetstream, mode AS \"mode: String\"\n FROM nats_trigger\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "nats_resource_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "subjects", + "type_info": "VarcharArray" + }, + { + "ordinal": 2, + "name": "use_jetstream", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "mode: String", + "type_info": { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "d618ff98a2c7aceb6820007411ba7be2968d982ae84fe3294a6d78914afe3ecc" +} diff --git a/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json b/backend/.sqlx/query-d7d11b2c288fa0282539edde0c5450e4b5f8355821617b6c1ce3db64f6ceaf69.json similarity index 58% rename from backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json rename to backend/.sqlx/query-d7d11b2c288fa0282539edde0c5450e4b5f8355821617b6c1ce3db64f6ceaf69.json index 7ddd14bb5c..e825c74a26 100644 --- a/backend/.sqlx/query-052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336.json +++ b/backend/.sqlx/query-d7d11b2c288fa0282539edde0c5450e4b5f8355821617b6c1ce3db64f6ceaf69.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'", + "query": "SELECT COUNT(*) FROM variable WHERE is_secret = true", "describe": { "columns": [ { @@ -16,5 +16,5 @@ null ] }, - "hash": "052d42b46d5faba6b41f1fdcbf6a012db51b9e5a255ec0da9a8a0999d668d336" + "hash": "d7d11b2c288fa0282539edde0c5450e4b5f8355821617b6c1ce3db64f6ceaf69" } diff --git a/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json b/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json new file mode 100644 index 0000000000..03b0c561e1 --- /dev/null +++ b/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188" +} diff --git a/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json b/backend/.sqlx/query-dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9.json similarity index 52% rename from backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json rename to backend/.sqlx/query-dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9.json index ed3665b584..2a627e06b4 100644 --- a/backend/.sqlx/query-1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708.json +++ b/backend/.sqlx/query-dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n importer_path,\n importer_kind::text as \"importer_kind!\", -- sqlx thinks this is nullable somehow, so enfore with !\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map \n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ", + "query": "\n SELECT\n importer_path,\n importer_kind::text as \"importer_kind!\",\n array_agg(importer_node_id) as importer_node_ids\n FROM dependency_map\n WHERE workspace_id = $1 AND imported_path = $2\n GROUP BY importer_path, importer_kind\n ", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "1e285da98ac08999f0ad489f1b81885d682de0d7c6ff3e09a9fddef8bb682708" + "hash": "dcc50c70ac8ecbcb0d79ea7ae0cacad6894605fd2fa391c928b35fbc37b680f9" } diff --git a/backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json b/backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json similarity index 69% rename from backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json rename to backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json index 1879fa69ec..f5a1215aa7 100644 --- a/backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json +++ b/backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n mode, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )", + "query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n email,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )", "describe": { "columns": [], "parameters": { @@ -46,5 +46,5 @@ }, "nullable": [] }, - "hash": "80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f" + "hash": "e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c" } diff --git a/backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json b/backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json similarity index 53% rename from backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json rename to backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json index 7be240422d..7c994ca763 100644 --- a/backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json +++ b/backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM email_trigger \n WHERE \n workspace_id = $1\n AND local_part = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM email_trigger\n WHERE\n workspace_id = $1\n AND local_part = $2\n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab" + "hash": "e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a" } diff --git a/backend/.sqlx/query-e485c82978d10379c6d1b7cd850f3bf764e6c5bf775f3d710ba36a79f5e424eb.json b/backend/.sqlx/query-e485c82978d10379c6d1b7cd850f3bf764e6c5bf775f3d710ba36a79f5e424eb.json new file mode 100644 index 0000000000..fc85891f33 --- /dev/null +++ b/backend/.sqlx/query-e485c82978d10379c6d1b7cd850f3bf764e6c5bf775f3d710ba36a79f5e424eb.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id,\n edited_by, email, initial_messages\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "JsonbArray" + ] + }, + "nullable": [] + }, + "hash": "e485c82978d10379c6d1b7cd850f3bf764e6c5bf775f3d710ba36a79f5e424eb" +} diff --git a/backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json b/backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json new file mode 100644 index 0000000000..a9d57f3132 --- /dev/null +++ b/backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n mqtt_trigger\n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND\n path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "JsonbArray", + { + "Custom": { + "name": "mqtt_client_version", + "kind": { + "Enum": [ + "v3", + "v5" + ] + } + } + }, + "Varchar", + "Jsonb", + "Jsonb", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 5b07bcd9c9..7309b03a02 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -185,7 +185,8 @@ "sqs", "gcp", "mqtt", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json b/backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json new file mode 100644 index 0000000000..d85c08815d --- /dev/null +++ b/backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO job_logs (\n job_id, workspace_id, logs, log_offset, log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text", + "Int4", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881" +} diff --git a/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json b/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json deleted file mode 100644 index f19d888a67..0000000000 --- a/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE postgres_trigger \n SET \n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b" -} diff --git a/backend/.sqlx/query-e5d1963829e06bfcfe413a583d229714291ce0901b757e8c799c7a6d1021c679.json b/backend/.sqlx/query-e5d1963829e06bfcfe413a583d229714291ce0901b757e8c799c7a6d1021c679.json new file mode 100644 index 0000000000..8fcc14cf15 --- /dev/null +++ b/backend/.sqlx/query-e5d1963829e06bfcfe413a583d229714291ce0901b757e8c799c7a6d1021c679.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT owner, email FROM capture_config WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "e5d1963829e06bfcfe413a583d229714291ce0901b757e8c799c7a6d1021c679" +} diff --git a/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json b/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json similarity index 55% rename from backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json rename to backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json index b0da0f0e69..dd65a586d1 100644 --- a/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json +++ b/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = now(), error = $1\n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n server_id = $6 AND \n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ", + "query": "\n UPDATE\n capture_config\n SET\n last_server_ping = now(), error = $1\n WHERE\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5 AND\n server_id = $6 AND\n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ", "describe": { "columns": [ { @@ -31,7 +31,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } @@ -43,5 +44,5 @@ null ] }, - "hash": "615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae" + "hash": "e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022" } diff --git a/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json b/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json deleted file mode 100644 index 1e67891f48..0000000000 --- a/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE sqs_trigger \n SET \n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE \n workspace_id = $10 AND path = $11\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "TextArray", - { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - }, - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e" -} diff --git a/backend/.sqlx/query-ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b.json b/backend/.sqlx/query-ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b.json deleted file mode 100644 index ab648f579d..0000000000 --- a/backend/.sqlx/query-ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_v2_job_root_by_path", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "ea9385509319f66b9330221eb50b85edafb9408d0306f17bb78b65a5d81c570b" -} diff --git a/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json index 57607ae052..7c45c44b13 100644 --- a/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json +++ b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json @@ -24,7 +24,8 @@ "mqtt", "gcp", "default_email", - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json b/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json index f80b75b5ec..16ccdd10e9 100644 --- a/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json +++ b/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json @@ -21,7 +21,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -71,7 +72,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-ecb51b6bd85218116e4d02bc92cf1cd0bbecdf51c18b4fc19ab3ef303331d51d.json b/backend/.sqlx/query-ecb51b6bd85218116e4d02bc92cf1cd0bbecdf51c18b4fc19ab3ef303331d51d.json new file mode 100644 index 0000000000..1f440a42ad --- /dev/null +++ b/backend/.sqlx/query-ecb51b6bd85218116e4d02bc92cf1cd0bbecdf51c18b4fc19ab3ef303331d51d.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT path, schedule, enabled, script_path, timezone\n FROM schedule\n WHERE workspace_id = $1 AND path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "schedule", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "timezone", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false + ] + }, + "hash": "ecb51b6bd85218116e4d02bc92cf1cd0bbecdf51c18b4fc19ab3ef303331d51d" +} diff --git a/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json b/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json new file mode 100644 index 0000000000..1876db927e --- /dev/null +++ b/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n capture (\n workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud", + "google" + ] + } + } + }, + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd" +} diff --git a/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json b/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json index 4ce08c9e1c..536175599d 100644 --- a/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json +++ b/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json @@ -21,7 +21,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -71,7 +72,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json b/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json similarity index 57% rename from backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json rename to backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json index 70fbb752eb..b31e532fc2 100644 --- a/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json +++ b/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571" + "hash": "f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4" } diff --git a/backend/.sqlx/query-f0eb9d49021668673880aa20227c24a130cada07fba3f43e32d904fde97106d3.json b/backend/.sqlx/query-f0eb9d49021668673880aa20227c24a130cada07fba3f43e32d904fde97106d3.json new file mode 100644 index 0000000000..7300f26200 --- /dev/null +++ b/backend/.sqlx/query-f0eb9d49021668673880aa20227c24a130cada07fba3f43e32d904fde97106d3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE mqtt_trigger SET script_path = $1 WHERE workspace_id = $2 AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f0eb9d49021668673880aa20227c24a130cada07fba3f43e32d904fde97106d3" +} diff --git a/backend/.sqlx/query-f149d740301a65a62664fd407abe89c972e50837b6c54ac7a3ccd5f60ca05d3c.json b/backend/.sqlx/query-f149d740301a65a62664fd407abe89c972e50837b6c54ac7a3ccd5f60ca05d3c.json new file mode 100644 index 0000000000..a0a8b47539 --- /dev/null +++ b/backend/.sqlx/query-f149d740301a65a62664fd407abe89c972e50837b6c54ac7a3ccd5f60ca05d3c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM mqtt_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f149d740301a65a62664fd407abe89c972e50837b6c54ac7a3ccd5f60ca05d3c" +} diff --git a/backend/.sqlx/query-f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df.json b/backend/.sqlx/query-f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df.json deleted file mode 100644 index a68b5dd969..0000000000 --- a/backend/.sqlx/query-f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_job_v2_job_root_by_path_2 ON v2_job (workspace_id, runnable_path) WHERE parent_job IS NULL;", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "f17c9fed09897191ee70214223d4b83a05f1d649bfedff9f24cc3e6c702d42df" -} diff --git a/backend/.sqlx/query-f21c608aa332b1600d2236ceba2be64c825e8bf5a2cb6446a2d973d028d6311f.json b/backend/.sqlx/query-f21c608aa332b1600d2236ceba2be64c825e8bf5a2cb6446a2d973d028d6311f.json new file mode 100644 index 0000000000..e5e8c91e6d --- /dev/null +++ b/backend/.sqlx/query-f21c608aa332b1600d2236ceba2be64c825e8bf5a2cb6446a2d973d028d6311f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f21c608aa332b1600d2236ceba2be64c825e8bf5a2cb6446a2d973d028d6311f" +} diff --git a/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json similarity index 76% rename from backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json rename to backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json index 43d54b5a1c..26cf59d375 100644 --- a/backend/.sqlx/query-37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b.json +++ b/backend/.sqlx/query-f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_dependencies\n SET archived = true \n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ", + "query": "\n UPDATE workspace_dependencies\n SET archived = true\n WHERE archived = false\n AND name IS NOT DISTINCT FROM $1\n AND workspace_id = $2\n AND language = $3\n RETURNING description\n ", "describe": { "columns": [ { @@ -51,5 +51,5 @@ false ] }, - "hash": "37409e147cf69c39dad848b117bdb77654c167a254053bfd3682c7d9add30b6b" + "hash": "f2bd875385052618533c868bee309de8863a5d10a482f012ee790eda64e88211" } diff --git a/backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json b/backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json new file mode 100644 index 0000000000..42f48ab33e --- /dev/null +++ b/backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE postgres_trigger\n SET\n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE\n workspace_id = $9 AND path = $10\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358" +} diff --git a/backend/.sqlx/query-f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2.json b/backend/.sqlx/query-f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2.json deleted file mode 100644 index 99e04c9bff..0000000000 --- a/backend/.sqlx/query-f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "create index concurrently if not exists ix_job_workspace_id_created_at_new_3 ON v2_job (workspace_id, created_at DESC)", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "f64ae18811e211dbf0cb98b43d3b018b0dcc0abc7e4a1f0b45885cfe18efd9b2" -} diff --git a/backend/.sqlx/query-fa4b7688a4e83dee51d5c64b7660b824a7ec745ad62845d469a5be6219a2be29.json b/backend/.sqlx/query-fa4b7688a4e83dee51d5c64b7660b824a7ec745ad62845d469a5be6219a2be29.json new file mode 100644 index 0000000000..6f661d30aa --- /dev/null +++ b/backend/.sqlx/query-fa4b7688a4e83dee51d5c64b7660b824a7ec745ad62845d469a5be6219a2be29.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "fa4b7688a4e83dee51d5c64b7660b824a7ec745ad62845d469a5be6219a2be29" +} diff --git a/backend/.sqlx/query-faa0e401e6beebde6c3fef06151d3e73a5806f61cae4a53b5bdc888ec7164395.json b/backend/.sqlx/query-faa0e401e6beebde6c3fef06151d3e73a5806f61cae4a53b5bdc888ec7164395.json deleted file mode 100644 index 2824316ed3..0000000000 --- a/backend/.sqlx/query-faa0e401e6beebde6c3fef06151d3e73a5806f61cae4a53b5bdc888ec7164395.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DROP INDEX CONCURRENTLY IF EXISTS ix_job_workspace_id_created_at_new_6", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "faa0e401e6beebde6c3fef06151d3e73a5806f61cae4a53b5bdc888ec7164395" -} diff --git a/backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json b/backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json similarity index 58% rename from backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json rename to backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json index 1d2f993f5d..71bb00c25e 100644 --- a/backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json +++ b/backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) \n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM http_trigger\n WHERE\n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1)\n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2\n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", "describe": { "columns": [ { @@ -33,5 +33,5 @@ null ] }, - "hash": "0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58" + "hash": "fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee" } diff --git a/backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json b/backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json new file mode 100644 index 0000000000..faed40d308 --- /dev/null +++ b/backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_runtime (id, ping, memory_peak)\n VALUES ($1, $2, $3)\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index cb3f257e83..dead88df84 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" [[package]] name = "ar_archive_writer" @@ -1029,15 +1029,16 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.77.0" +version = "1.93.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f2f37fea82468fe3f5a059542c05392ef680c4f7f00e0db02df8b6e5c7d0c6" +checksum = "9dcb38bb33fc0a11f1ffc3e3e85669e0a11a37690b86f77e75306d8f369146a0" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.3", + "aws-smithy-json 0.62.3", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1045,21 +1046,23 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.78.0" +version = "1.95.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecb4f6eada20e0193450cd48b12ed05e1e66baac86f39160191651b932f2b7d9" +checksum = "2ada8ffbea7bd1be1f53df1dadb0f8fdb04badb13185b3321b929d1ee3caad09" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.3", + "aws-smithy-json 0.62.3", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1067,6 +1070,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] @@ -1632,6 +1636,26 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.9.4", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.114", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -5125,6 +5149,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + [[package]] name = "dunce" version = "1.0.5" @@ -6104,6 +6140,17 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -6113,7 +6160,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -6131,6 +6178,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + [[package]] name = "ghash" version = "0.5.1" @@ -7171,6 +7231,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -7828,6 +7894,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "levenshtein_automata" version = "0.2.1" @@ -7893,9 +7965,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.180" +version = "0.2.181" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "459427e2af2b9c839b132acb702a1c654d95e10f8c326bfc2ad11310e458b1c5" [[package]] name = "libffi" @@ -7916,6 +7988,28 @@ dependencies = [ "cc", ] +[[package]] +name = "libgssapi" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" +dependencies = [ + "bitflags 2.9.4", + "bytes", + "lazy_static", + "libgssapi-sys", +] + +[[package]] +name = "libgssapi-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7518e6902e94f92e7c7271232684b60988b4bd813529b4ef9d97aead96956ae8" +dependencies = [ + "bindgen 0.71.1", + "pkg-config", +] + [[package]] name = "libloading" version = "0.7.4" @@ -8331,6 +8425,12 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md5" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6bcd6433cff03a4bfc3d9834d504467db1f1cf6d0ea765d37d330249ed629d" + [[package]] name = "measure_time" version = "0.9.0" @@ -8342,9 +8442,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" @@ -8482,7 +8582,7 @@ checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -8493,7 +8593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -9841,9 +9941,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.5" +version = "2.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" dependencies = [ "memchr", "ucd-trie", @@ -9851,9 +9951,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.5" +version = "2.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" dependencies = [ "pest", "pest_generator", @@ -9861,9 +9961,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.5" +version = "2.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" dependencies = [ "pest", "pest_meta", @@ -9874,9 +9974,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.5" +version = "2.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" dependencies = [ "pest", "sha2 0.10.9", @@ -10434,9 +10534,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" dependencies = [ "ar_archive_writer", "cc", @@ -10632,6 +10732,19 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -10654,6 +10767,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -10674,6 +10797,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -10712,6 +10844,15 @@ dependencies = [ "rand 0.9.0", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "range-alloc" version = "0.1.4" @@ -10798,9 +10939,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.9.0+2.10.0" +version = "4.10.0+2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230dca48bc354d718269f3e4353280e188b610f7af7e2fcf54b7a79d5802872" +checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" dependencies = [ "cmake", "libc", @@ -10808,6 +10949,7 @@ dependencies = [ "num_enum", "openssl-sys", "pkg-config", + "sasl2-sys", ] [[package]] @@ -11047,9 +11189,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f42e2f48018a33d679ee7f477a446b697663a14e91ab0b3a0206792a22dd3aa8" +checksum = "199dda04a536b532d0cc04d7979e39b1c763ea749bf91507017069c00b96056f" dependencies = [ "anyhow", "async-trait", @@ -11062,9 +11204,9 @@ dependencies = [ [[package]] name = "reqwest-retry" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e3930c1c07a59122f304ff269a634a4d5013afecbe646ccad4ff98e0803c2" +checksum = "fe2412db2af7d2268e7a5406be0431f37d9eb67ff390f35b395716f5f06c2eaa" dependencies = [ "anyhow", "async-trait", @@ -11733,9 +11875,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "ryu-js" @@ -11809,6 +11951,18 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sasl2-sys" +version = "0.1.22+2.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f2a7f7efd9fc98b3a9033272df10709f5ee3fa0eabbd61a527a3a1ed6bd3c6" +dependencies = [ + "cc", + "duct", + "libc", + "pkg-config", +] + [[package]] name = "saturating" version = "0.1.0" @@ -12297,6 +12451,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shared_child" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e297bd52991bbe0686c086957bee142f13df85d1e79b0b21630a99d374ae9dc" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "shellexpand" version = "2.1.2" @@ -12871,9 +13035,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", "cfg-if", @@ -13674,12 +13838,12 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.1", "once_cell", "rustix 1.1.3", "windows-sys 0.61.2", @@ -13786,8 +13950,7 @@ dependencies = [ [[package]] name = "tiberius" version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1446cb4198848d1562301a3340424b4f425ef79f35ef9ee034769a9dd92c10d" +source = "git+https://github.com/prisma/tiberius?rev=59db57960a14b422fb3a1309aa4aa47880896ff8#59db57960a14b422fb3a1309aa4aa47880896ff8" dependencies = [ "async-trait", "asynchronous-codec", @@ -13798,6 +13961,7 @@ dependencies = [ "encoding_rs", "enumflags2", "futures-util", + "libgssapi", "num-traits", "once_cell", "pin-project-lite", @@ -13810,6 +13974,7 @@ dependencies = [ "tokio-util", "tracing", "uuid", + "winauth", ] [[package]] @@ -14822,9 +14987,9 @@ checksum = "81b79ad29b5e19de4260020f8919b443b2ef0277d242ce532ec7b7a2cc8b6007" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "537dd038a89878be9b64dd4bd1b260315c1bb94f4d784956b81e27a088d9a09e" [[package]] name = "unicode-linebreak" @@ -15131,6 +15296,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -15146,6 +15317,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasite" version = "0.1.0" @@ -15248,6 +15428,28 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.11.1", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -15271,6 +15473,18 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.9.4", + "hashbrown 0.15.5", + "indexmap 2.11.1", + "semver 1.0.27", +] + [[package]] name = "wasmtimer" version = "0.4.3" @@ -15496,37 +15710,44 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winauth" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f820cd208ce9c6b050812dc2d724ba98c6c1e9db5ce9b3f58d925ae5723a5e6" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "md5", + "rand 0.7.3", + "winapi", +] + [[package]] name = "windmill" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", - "aws-sdk-config", - "aws-sigv4", + "async-nats", + "aws-config", + "aws-credential-types", + "aws-sdk-sqs", "axum 0.7.9", "base64 0.22.1", - "bitflags 2.9.4", "chrono", "constant_time_eq 0.3.1", - "deno_core", "dotenv", "futures", "gethostname", "git-version", - "globset", - "k8s-openapi", - "kube", "lazy_static", - "libloading 0.8.9", - "memchr", "object_store", "once_cell", - "opentelemetry-proto 0.29.0", - "pep440_rs", "prometheus", - "quote", "rand 0.9.0", + "rdkafka", "reqwest 0.13.1", + "rumqttc", "rustls 0.23.35", "serde", "serde_derive", @@ -15537,7 +15758,6 @@ dependencies = [ "sql-builder", "sqlx", "strum 0.27.2", - "systemstat", "tempfile", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -15547,14 +15767,15 @@ dependencies = [ "tracing", "url", "uuid", - "v8", "windmill-api", "windmill-api-client", "windmill-autoscaling", "windmill-common", + "windmill-dep-map", "windmill-git-sync", "windmill-indexer", "windmill-queue", + "windmill-runtime-nativets", "windmill-worker", "windows-service", "windows-sys 0.52.0", @@ -15562,12 +15783,11 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "argon2", "astral-tokio-tar", - "async-nats", "async-recursion", "async-stream", "async-trait", @@ -15577,39 +15797,24 @@ dependencies = [ "aws-sdk-bedrock", "aws-sdk-bedrockruntime", "aws-sdk-config", - "aws-sdk-sqs", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", "aws-sigv4", "aws-smithy-types", "axum 0.7.9", - "backon", "base32", "base64 0.22.1", - "byteorder", "bytes", - "candle-core", - "candle-nn", - "candle-transformers", "chrono", "chrono-tz", "const_format", - "constant_time_eq 0.3.1", "cookie 0.17.0", "cron", "dashmap 6.1.0", "datafusion", - "deno_core", - "deno_error", "ed25519-dalek", "flate2", "futures", "git-version", - "google-cloud-googleapis", - "google-cloud-pubsub", "hex", - "hf-hub", "hmac", "http 1.4.0", "hyper 1.8.1", @@ -15623,50 +15828,36 @@ dependencies = [ "matchit 0.7.3", "mime_guess", "native-tls", - "nkeys", "object_store", "openidconnect", "openssl", - "pg_escape", "pin-project", - "postgres-native-tls 0.5.0", "postgres-native-tls 0.5.1", "prometheus", "quick_cache", "rand 0.9.0", - "rdkafka", - "rdkafka-sys", "regex", "reqwest 0.13.1", "rsa", - "rumqttc", "rust-embed", - "rust_decimal", "rustls 0.23.35", "samael", "serde", "serde_json", "serde_urlencoded", "serde_yml", - "sha1", "sha2 0.10.9", "sql-builder", "sqlx", "strum 0.27.2", "tar", "tempfile", - "thiserror 2.0.18", "time", - "tinyvector", - "tokenizers", "tokio", "tokio-native-tls", - "tokio-postgres 0.7.11", "tokio-postgres 0.7.13", "tokio-stream", - "tokio-tungstenite 0.24.0", "tokio-util", - "tonic", "tower 0.5.3", "tower-cookies", "tower-http", @@ -15676,12 +15867,23 @@ dependencies = [ "url", "urlencoding", "uuid", + "windmill-api-auth", + "windmill-api-embeddings", + "windmill-api-groups", + "windmill-api-jobs", + "windmill-api-scripts", + "windmill-api-sse", + "windmill-api-users", + "windmill-api-workspaces", "windmill-audit", "windmill-autoscaling", "windmill-common", + "windmill-dep-map", "windmill-git-sync", "windmill-indexer", + "windmill-jseval", "windmill-mcp", + "windmill-native-triggers", "windmill-oauth", "windmill-parser", "windmill-parser-py", @@ -15689,12 +15891,49 @@ dependencies = [ "windmill-parser-sql", "windmill-parser-ts", "windmill-queue", + "windmill-store", + "windmill-trigger", + "windmill-trigger-email", + "windmill-trigger-gcp", + "windmill-trigger-http", + "windmill-trigger-kafka", + "windmill-trigger-mqtt", + "windmill-trigger-nats", + "windmill-trigger-postgres", + "windmill-trigger-sqs", + "windmill-trigger-websocket", "windmill-worker", ] +[[package]] +name = "windmill-api-auth" +version = "1.629.1" +dependencies = [ + "anyhow", + "axum 0.7.9", + "chrono", + "http 1.4.0", + "itertools 0.14.0", + "jsonwebtoken 8.3.0", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "time", + "tokio", + "tower-cookies", + "tracing", + "uuid", + "windmill-audit", + "windmill-common", + "windmill-mcp", +] + [[package]] name = "windmill-api-client" -version = "1.625.0" +version = "1.629.1" dependencies = [ "reqwest 0.12.28", "serde", @@ -15702,9 +15941,169 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "windmill-api-embeddings" +version = "1.629.1" +dependencies = [ + "anyhow", + "axum 0.7.9", + "candle-core", + "candle-nn", + "candle-transformers", + "hf-hub", + "lazy_static", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "tinyvector", + "tokenizers", + "tokio", + "tracing", + "windmill-common", + "windmill-store", +] + +[[package]] +name = "windmill-api-groups" +version = "1.629.1" +dependencies = [ + "axum 0.7.9", + "chrono", + "lazy_static", + "regex", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "uuid", + "windmill-api-auth", + "windmill-api-workspaces", + "windmill-audit", + "windmill-common", + "windmill-git-sync", +] + +[[package]] +name = "windmill-api-jobs" +version = "1.629.1" +dependencies = [ + "anyhow", + "axum 0.7.9", + "base64 0.22.1", + "chrono", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tower-http", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-api-sse", + "windmill-common", + "windmill-queue", +] + +[[package]] +name = "windmill-api-scripts" +version = "1.629.1" +dependencies = [ + "axum 0.7.9", + "chrono", + "futures", + "http 1.4.0", + "hyper 1.8.1", + "itertools 0.14.0", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-dep-map", + "windmill-git-sync", + "windmill-parser-py", + "windmill-parser-ts", + "windmill-queue", +] + +[[package]] +name = "windmill-api-sse" +version = "1.629.1" +dependencies = [ + "lazy_static", + "serde", + "serde_json", + "sqlx", + "uuid", + "windmill-common", +] + +[[package]] +name = "windmill-api-users" +version = "1.629.1" +dependencies = [ + "argon2", + "axum 0.7.9", + "chrono", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "serde", + "serde_json", + "sqlx", + "time", + "tokio", + "tower-cookies", + "tracing", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-git-sync", +] + +[[package]] +name = "windmill-api-workspaces" +version = "1.629.1" +dependencies = [ + "axum 0.7.9", + "chrono", + "hex", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "regex", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-api-jobs", + "windmill-api-users", + "windmill-audit", + "windmill-common", + "windmill-dep-map", + "windmill-git-sync", + "windmill-queue", +] + [[package]] name = "windmill-audit" -version = "1.625.0" +version = "1.629.1" dependencies = [ "chrono", "lazy_static", @@ -15718,7 +16117,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "axum 0.7.9", @@ -15737,8 +16136,9 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.625.0" +version = "1.629.1" dependencies = [ + "aes-gcm", "anyhow", "async-recursion", "async-stream", @@ -15797,6 +16197,7 @@ dependencies = [ "reqwest 0.13.1", "reqwest-middleware", "reqwest-retry", + "rsa", "semver 1.0.27", "serde", "serde_json", @@ -15829,11 +16230,31 @@ dependencies = [ "windmill-parser-py", "windmill-parser-sql", "windmill-parser-ts", + "windmill-types", +] + +[[package]] +name = "windmill-dep-map" +version = "1.629.1" +dependencies = [ + "chrono", + "itertools 0.14.0", + "lazy_static", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-common", + "windmill-parser-py-imports", + "windmill-parser-ts", + "windmill-queue", ] [[package]] name = "windmill-git-sync" -version = "1.625.0" +version = "1.629.1" dependencies = [ "regex", "serde", @@ -15848,7 +16269,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15870,9 +16291,26 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-jseval" +version = "1.629.1" +dependencies = [ + "anyhow", + "futures", + "lazy_static", + "mappable-rc", + "regex", + "rquickjs", + "serde_json", + "tokio", + "tracing", + "uuid", + "windmill-common", +] + [[package]] name = "windmill-macros" -version = "1.625.0" +version = "1.629.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15888,10 +16326,11 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "async-trait", + "chrono", "futures", "http 1.4.0", "oauth2", @@ -15906,9 +16345,40 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-native-triggers" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "backon", + "base64 0.22.1", + "chrono", + "hmac", + "http 1.4.0", + "itertools 0.14.0", + "lazy_static", + "reqwest 0.13.1", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx", + "strum 0.27.2", + "tokio", + "tracing", + "urlencoding", + "uuid", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-oauth", + "windmill-queue", +] + [[package]] name = "windmill-oauth" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "async-oauth2", @@ -15932,7 +16402,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.625.0" +version = "1.629.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15941,7 +16411,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "lazy_static", @@ -15953,7 +16423,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "serde_json", @@ -15965,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "gosyn", @@ -15977,7 +16447,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "lazy_static", @@ -15989,7 +16459,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "serde_json", @@ -16001,7 +16471,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "nu-parser", @@ -16012,7 +16482,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16023,7 +16493,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16036,7 +16506,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "async-recursion", @@ -16060,7 +16530,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "lazy_static", @@ -16074,7 +16544,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16091,7 +16561,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "lazy_static", @@ -16105,7 +16575,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "lazy_static", @@ -16124,7 +16594,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "serde", @@ -16135,7 +16605,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "async-recursion", @@ -16170,9 +16640,47 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-runtime-nativets" +version = "1.629.1" +dependencies = [ + "anyhow", + "const_format", + "deno_ast", + "deno_console", + "deno_core", + "deno_error", + "deno_fetch", + "deno_io", + "deno_net", + "deno_permissions", + "deno_runtime", + "deno_telemetry", + "deno_tls", + "deno_url", + "deno_web", + "deno_webidl", + "futures", + "itertools 0.14.0", + "lazy_static", + "regex", + "reqwest 0.13.1", + "rustls 0.23.35", + "serde", + "serde_json", + "sqlx", + "tokio", + "tracing", + "uuid", + "winapi", + "windmill-common", + "windmill-parser-ts", + "windmill-queue", +] + [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.625.0" +version = "1.629.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -16180,9 +16688,333 @@ dependencies = [ "windmill-parser-sql", ] +[[package]] +name = "windmill-store" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-recursion", + "axum 0.7.9", + "chrono", + "futures", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-mcp", + "windmill-oauth", +] + +[[package]] +name = "windmill-trigger" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "chrono", + "http 1.4.0", + "hyper 1.8.1", + "itertools 0.14.0", + "lazy_static", + "rand 0.9.0", + "reqwest 0.13.1", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-api-jobs", + "windmill-api-sse", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-parser", + "windmill-parser-py", + "windmill-parser-ts", + "windmill-queue", +] + +[[package]] +name = "windmill-trigger-email" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "lazy_static", + "regex", + "serde", + "serde_json", + "sqlx", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-gcp" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "bytes", + "chrono", + "google-cloud-googleapis", + "google-cloud-pubsub", + "http 1.4.0", + "itertools 0.14.0", + "jsonwebtoken 8.3.0", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tonic", + "tower-http", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-http" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "constant_time_eq 0.3.1", + "futures", + "hex", + "hmac", + "http 1.4.0", + "hyper 1.8.1", + "itertools 0.14.0", + "lazy_static", + "matchit 0.7.3", + "quick_cache", + "regex", + "serde", + "serde_json", + "sha1", + "sha2 0.10.9", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-kafka" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "itertools 0.14.0", + "rdkafka", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-mqtt" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "bytes", + "itertools 0.14.0", + "rumqttc", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-nats" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-nats", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "itertools 0.14.0", + "nkeys", + "serde", + "serde_json", + "sqlx", + "tokio", + "tokio-stream", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-postgres" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "byteorder", + "bytes", + "chrono", + "futures", + "itertools 0.14.0", + "lazy_static", + "native-tls", + "pg_escape", + "postgres-native-tls 0.5.0", + "quick_cache", + "rand 0.9.0", + "rust_decimal", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tokio-postgres 0.7.11", + "tokio-stream", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-sqs" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-sqs", + "aws-sdk-sts", + "aws-smithy-types", + "axum 0.7.9", + "backon", + "chrono", + "itertools 0.14.0", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-websocket" +version = "1.629.1" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "futures", + "http 1.4.0", + "itertools 0.14.0", + "serde", + "serde_json", + "sqlx", + "tokio", + "tokio-tungstenite 0.24.0", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-queue", + "windmill-trigger", +] + +[[package]] +name = "windmill-types" +version = "1.629.1" +dependencies = [ + "anyhow", + "chrono", + "hex", + "itertools 0.14.0", + "rand 0.9.0", + "serde", + "serde_json", + "sqlx", + "strum 0.27.2", + "tracing", + "uuid", +] + [[package]] name = "windmill-worker" -version = "1.625.0" +version = "1.629.1" dependencies = [ "anyhow", "async-once-cell", @@ -16202,20 +17034,6 @@ dependencies = [ "chrono", "const_format", "convert_case 0.6.0", - "deno_ast", - "deno_console", - "deno_core", - "deno_error", - "deno_fetch", - "deno_io", - "deno_net", - "deno_permissions", - "deno_runtime", - "deno_telemetry", - "deno_tls", - "deno_url", - "deno_web", - "deno_webidl", "derive_more 1.0.0", "dotenv", "eventsource-stream", @@ -16254,9 +17072,7 @@ dependencies = [ "regex", "reqwest 0.13.1", "reqwest-middleware", - "rquickjs", "rust_decimal", - "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2 0.10.9", @@ -16273,10 +17089,11 @@ dependencies = [ "url", "urlencoding", "uuid", - "winapi", "windmill-audit", "windmill-common", + "windmill-dep-map", "windmill-git-sync", + "windmill-jseval", "windmill-macros", "windmill-mcp", "windmill-parser", @@ -16295,6 +17112,7 @@ dependencies = [ "windmill-parser-ts", "windmill-parser-yaml", "windmill-queue", + "windmill-runtime-nativets", "yaml-rust", ] @@ -16910,6 +17728,88 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap 2.11.1", + "prettyplease", + "syn 2.0.114", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.114", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.9.4", + "indexmap 2.11.1", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.11.1", + "log", + "semver 1.0.27", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" @@ -17083,18 +17983,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.38" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.38" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" dependencies = [ "proc-macro2", "quote", @@ -17177,9 +18077,9 @@ dependencies = [ [[package]] name = "zip" -version = "7.3.0" +version = "7.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268bf6f9ceb991e07155234071501490bb41fd1e39c6a588106dad10ae2a5804" +checksum = "cc12baa6db2b15a140161ce53d72209dacea594230798c24774139b54ecaa980" dependencies = [ "crc32fast", "indexmap 2.11.1", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 88d16e2809..43ec791585 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.625.0" +version = "1.629.1" authors.workspace = true edition.workspace = true @@ -8,9 +8,33 @@ edition.workspace = true resolver = "2" members = [ "./windmill-api", + "./windmill-api-scripts", + "./windmill-api-users", + "./windmill-api-workspaces", + "./windmill-api-groups", + "./windmill-api-auth", + "./windmill-api-sse", + "./windmill-api-jobs", + "./windmill-trigger", + "./windmill-trigger-kafka", + "./windmill-trigger-postgres", + "./windmill-trigger-mqtt", + "./windmill-trigger-websocket", + "./windmill-trigger-email", + "./windmill-trigger-nats", + "./windmill-trigger-sqs", + "./windmill-trigger-gcp", + "./windmill-trigger-http", + "./windmill-native-triggers", + "./windmill-api-embeddings", + "./windmill-store", "./windmill-queue", "./windmill-worker", + "./windmill-dep-map", + "./windmill-types", "./windmill-common", + "./windmill-jseval", + "./windmill-runtime-nativets", "./windmill-mcp", "./windmill-audit", "./windmill-git-sync", @@ -35,7 +59,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.625.0" +version = "1.629.1" authors = ["Ruben Fiszel "] edition = "2021" @@ -46,6 +70,10 @@ path = "./src/main.rs" [profile.dev] opt-level = 0 incremental = true +split-debuginfo = "unpacked" + +[profile.dev.package."*"] +debug = false [profile.release] lto = "thin" @@ -55,6 +83,7 @@ default = [] private = ["windmill-api/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-queue/private", "windmill-worker/private"] agent_worker_server = ["windmill-api/agent_worker_server"] enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"] +local_reports = ["windmill-common/local_reports"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] stripe = ["windmill-api/stripe"] benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] @@ -62,15 +91,16 @@ embedding = ["windmill-api/embedding"] parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "dep:object_store"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus", "dep:prometheus"] flow_testing = ["windmill-worker/flow_testing"] +quickjs = ["windmill-worker/quickjs", "windmill-api/quickjs"] openidconnect = ["windmill-api/openidconnect", "windmill-common/openidconnect"] cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud", "windmill-api/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] -deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"] -quickjs = ["windmill-worker/quickjs"] +deno_core = ["windmill-worker/deno_core", "dep:windmill-runtime-nativets"] deno_core_mac = ["deno_core", "windmill-worker/libffi_mac"] kafka = ["windmill-api/kafka"] +kafka-gssapi = ["windmill-api/kafka-gssapi"] nats = ["windmill-api/nats"] otel = ["windmill-common/otel", "windmill-worker/otel"] dind = ["windmill-worker/dind"] @@ -89,7 +119,7 @@ oauth2 = ["windmill-api/oauth2"] zip = ["windmill-api/zip"] static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] -test_job_debouncing = [] +no_auth = ["windmill-api/no_auth"] private_registry_test = [] # Languages python = ["windmill-worker/python", "windmill-api/python"] @@ -98,22 +128,50 @@ mysql = ["windmill-worker/mysql"] oracledb = ["windmill-worker/oracledb"] duckdb = ["windmill-worker/duckdb"] mssql = ["windmill-worker/mssql"] +mssql-kerberos = ["windmill-worker/mssql-kerberos"] # Linux/Unix integrated auth +mssql-winauth = ["windmill-worker/mssql-winauth"] # Windows integrated auth bigquery = ["windmill-worker/bigquery"] php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] ruby = ["windmill-worker/ruby"] -all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java", "ruby"] +all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby"] # For windows we have another set of languages enabled -all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"] +# Edition meta-features: shared groups +inline_preview = ["windmill-api/inline_preview"] +oss_core = [ + "embedding", "parquet", "openidconnect", "license", + "http_trigger", "zip", "oauth2", "postgres_trigger", + "mqtt_trigger", "websocket", "smtp", "native_trigger", + "static_frontend", "mcp", "bedrock", "inline_preview", + "quickjs" +] +ce_core = ["oss_core", "private"] +ee_core = [ + "enterprise", "stripe", "prometheus", "cloud", + "kafka", "sqs_trigger", "nats", "gcp_trigger", + "jemalloc", "otel" +] +ee_server = ["enterprise_saml", "tantivy", "agent_worker_server", "local_reports"] +# Edition meta-features: CE variants +oss = ["oss_core", "all_languages", "no_auth"] +ce_rpi = ["ce_core", "all_languages"] +ce = ["ce_rpi", "jemalloc", "dind", "agent_worker_server"] +# Edition meta-features: EE variants +ee = ["ce", "ee_core", "ee_server"] +ee_rhel = ["ce_core", "ee_core", "all_languages"] +ee_windows = ["ce_core", "ee_core", "all_languages_windows"] all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "embedding", "parquet", "prometheus", "flow_testing", - "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "nats", "otel", "dind", "websocket", "http_trigger", + "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "kafka-gssapi", "nats", "otel", "dind", "websocket", "http_trigger", "postgres_trigger", "mcp", "mqtt_trigger", "sqs_trigger", "gcp_trigger", "smtp", "stripe", - "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server", "bedrock", "native_trigger"] + "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server", "bedrock", "native_trigger", "quickjs"] [patch.crates-io] object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } +# Use tiberius main branch for libgssapi 0.8.1 fix (https://github.com/prisma/tiberius/issues/343) +tiberius = { git = "https://github.com/prisma/tiberius", rev = "59db57960a14b422fb3a1309aa4aa47880896ff8" } [dependencies] anyhow.workspace = true @@ -146,25 +204,12 @@ serde_json.workspace = true serde_derive.workspace = true serde_yml.workspace = true serde.workspace = true -deno_core = { workspace = true, optional = true } +windmill-runtime-nativets = { workspace = true, optional = true } object_store = { workspace = true, optional = true } sha1 = { workspace = true, optional = true } constant_time_eq = { workspace = true, optional = true } -quote.workspace = true -memchr.workspace = true -v8 = { workspace = true, optional = true } rustls.workspace = true -pep440_rs.workspace = true strum.workspace = true -aws-sigv4.workspace = true -aws-sdk-config.workspace = true -kube.workspace = true -k8s-openapi.workspace = true -libloading.workspace = true -bitflags.workspace = true -globset.workspace = true -opentelemetry-proto.workspace = true -systemstat.workspace = true [target.'cfg(windows)'.dependencies] windows-service = "0.7" @@ -178,18 +223,26 @@ tikv-jemalloc-ctl = { optional = true, workspace = true } [dev-dependencies] serde_json.workspace = true reqwest.workspace = true -windmill-queue.workspace = true +windmill-queue = { workspace = true, features = ["failpoints"] } +windmill-dep-map.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true -deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] } tempfile.workspace = true +rumqttc.workspace = true +rdkafka.workspace = true +async-nats.workspace = true +aws-sdk-sqs.workspace = true +aws-config.workspace = true +aws-credential-types.workspace = true [workspace.dependencies] windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-dep-map = { path = "./windmill-dep-map" } +windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } @@ -198,6 +251,26 @@ windmill-indexer = {path = "./windmill-indexer"} windmill-mcp = {path = "./windmill-mcp"} windmill-oauth = {path = "./windmill-oauth"} windmill-macros = {path = "./windmill-macros"} +windmill-api-auth = { path = "./windmill-api-auth" } +windmill-api-scripts = { path = "./windmill-api-scripts" } +windmill-api-users = { path = "./windmill-api-users" } +windmill-api-workspaces = { path = "./windmill-api-workspaces" } +windmill-api-groups = { path = "./windmill-api-groups" } +windmill-api-sse = { path = "./windmill-api-sse" } +windmill-api-jobs = { path = "./windmill-api-jobs" } +windmill-trigger = { path = "./windmill-trigger" } +windmill-trigger-kafka = { path = "./windmill-trigger-kafka" } +windmill-trigger-postgres = { path = "./windmill-trigger-postgres" } +windmill-trigger-mqtt = { path = "./windmill-trigger-mqtt" } +windmill-trigger-websocket = { path = "./windmill-trigger-websocket" } +windmill-trigger-email = { path = "./windmill-trigger-email" } +windmill-trigger-nats = { path = "./windmill-trigger-nats" } +windmill-trigger-sqs = { path = "./windmill-trigger-sqs" } +windmill-trigger-gcp = { path = "./windmill-trigger-gcp" } +windmill-trigger-http = { path = "./windmill-trigger-http" } +windmill-native-triggers = { path = "./windmill-native-triggers" } +windmill-api-embeddings = { path = "./windmill-api-embeddings" } +windmill-store = { path = "./windmill-store" } windmill-parser = { path = "./parsers/windmill-parser" } windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } @@ -213,6 +286,8 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } +windmill-jseval = { path = "./windmill-jseval" } +windmill-runtime-nativets = { path = "./windmill-runtime-nativets" } windmill-api-client = { path = "./windmill-api-client" } reqwest-retry = "^0" @@ -344,6 +419,7 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } const-str = "0.5" constant_time_eq = "0.3.1" rsa = "^0" +aes-gcm = "0.10.3" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" dashmap = "6.1.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 59e112dc29..fd060cbd4e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -41264d02f64572a3ea3b81ac2833e8d466366e96 \ No newline at end of file +7596cefdba81482c0b0c0b61be26369f112d8009 diff --git a/backend/migrations/20260204181621_delete_saved_telemetry.down.sql b/backend/migrations/20260204181621_delete_saved_telemetry.down.sql new file mode 100644 index 0000000000..0435203b02 --- /dev/null +++ b/backend/migrations/20260204181621_delete_saved_telemetry.down.sql @@ -0,0 +1 @@ +-- No-op: cannot restore deleted telemetry data diff --git a/backend/migrations/20260204181621_delete_saved_telemetry.up.sql b/backend/migrations/20260204181621_delete_saved_telemetry.up.sql new file mode 100644 index 0000000000..aae45e937f --- /dev/null +++ b/backend/migrations/20260204181621_delete_saved_telemetry.up.sql @@ -0,0 +1,2 @@ +-- Delete all saved telemetry data from metrics table +DELETE FROM metrics WHERE id = 'telemetry'; diff --git a/backend/migrations/20260205193914_grant_asset_id_seq.down.sql b/backend/migrations/20260205193914_grant_asset_id_seq.down.sql new file mode 100644 index 0000000000..273ec4299e --- /dev/null +++ b/backend/migrations/20260205193914_grant_asset_id_seq.down.sql @@ -0,0 +1,2 @@ +REVOKE ALL ON SEQUENCE asset_id_seq FROM windmill_user; +REVOKE ALL ON SEQUENCE asset_id_seq FROM windmill_admin; diff --git a/backend/migrations/20260205193914_grant_asset_id_seq.up.sql b/backend/migrations/20260205193914_grant_asset_id_seq.up.sql new file mode 100644 index 0000000000..ddc052c117 --- /dev/null +++ b/backend/migrations/20260205193914_grant_asset_id_seq.up.sql @@ -0,0 +1,2 @@ +GRANT ALL ON SEQUENCE asset_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE asset_id_seq TO windmill_admin; diff --git a/backend/migrations/20260206000000_consolidate_live_index_migrations.down.sql b/backend/migrations/20260206000000_consolidate_live_index_migrations.down.sql new file mode 100644 index 0000000000..a5c930142d --- /dev/null +++ b/backend/migrations/20260206000000_consolidate_live_index_migrations.down.sql @@ -0,0 +1,2 @@ +-- No-op: indexes and windmill_migrations entries are safe to leave in place. +-- Rolling back this migration does not require removing the indexes. diff --git a/backend/migrations/20260206000000_consolidate_live_index_migrations.up.sql b/backend/migrations/20260206000000_consolidate_live_index_migrations.up.sql new file mode 100644 index 0000000000..2a12c11403 --- /dev/null +++ b/backend/migrations/20260206000000_consolidate_live_index_migrations.up.sql @@ -0,0 +1,89 @@ +-- Consolidate live index migrations into a regular SQL migration. +-- All statements are idempotent (IF EXISTS / IF NOT EXISTS). + +-- === DROP obsolete indexes === +DROP INDEX IF EXISTS ix_completed_job_workspace_id_created_at; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_created_at_new; +DROP INDEX IF EXISTS index_completed_job_on_schedule_path; +DROP INDEX IF EXISTS concurrency_limit_stats_queue; +DROP INDEX IF EXISTS root_job_index; +DROP INDEX IF EXISTS index_completed_on_created; +DROP INDEX IF EXISTS root_job_index_by_path_2; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_created_at_new_2; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_started_at_new; +DROP INDEX IF EXISTS root_job_index_by_path; +DROP INDEX IF EXISTS labeled_jobs_on_jobs; +DROP INDEX IF EXISTS ix_job_workspace_id_created_at_new_6; +DROP INDEX IF EXISTS ix_job_workspace_id_created_at_new_7; +DROP INDEX IF EXISTS queue_sort; +DROP INDEX IF EXISTS queue_sort_2; +DROP INDEX IF EXISTS log_file_hostname_log_ts_idx; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_started_at_new_2; +DROP INDEX IF EXISTS ix_job_created_at; +DROP INDEX IF EXISTS ix_v2_job_root_by_path; + +-- === CREATE indexes === +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_3 + ON v2_job (workspace_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_8 + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('deploymentcallback') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_9 + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_5 + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('preview', 'flowpreview') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS labeled_jobs_on_jobs + ON v2_job_completed USING GIN ((result -> 'wm_labels')) + WHERE result ? 'wm_labels'; + +CREATE INDEX IF NOT EXISTS ix_v2_job_labels + ON v2_job USING GIN (labels) + WHERE labels IS NOT NULL; + +ALTER TABLE v2_job ENABLE ROW LEVEL SECURITY; + +CREATE INDEX IF NOT EXISTS ix_v2_job_workspace_id_created_at + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('script', 'flow', 'singlestepflow') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS queue_sort_v2 + ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) + WHERE running = false; + +CREATE INDEX IF NOT EXISTS ix_audit_timestamps + ON audit (timestamp DESC); + +CREATE INDEX IF NOT EXISTS ix_job_completed_completed_at + ON v2_job_completed (completed_at DESC); + +CREATE INDEX IF NOT EXISTS alerts_by_workspace + ON alerts (workspace_id); + +CREATE INDEX IF NOT EXISTS v2_job_queue_suspend + ON v2_job_queue (workspace_id, suspend) + WHERE suspend > 0; + +CREATE INDEX IF NOT EXISTS idx_audit_recent_login_activities + ON audit (timestamp, username) + WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh'); + +CREATE INDEX IF NOT EXISTS script_not_archived + ON script (workspace_id, path, created_at DESC) + WHERE archived = false; + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_completed_at_all + ON v2_job_completed (workspace_id, completed_at DESC); + +CREATE INDEX IF NOT EXISTS idx_job_v2_job_root_by_path_2 + ON v2_job (workspace_id, runnable_path) + WHERE parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_job_root_job_index_by_path_2 + ON v2_job (workspace_id, runnable_path, created_at DESC) + WHERE parent_job IS NULL; diff --git a/backend/migrations/20260206060555_notify_triggers_security_definer.down.sql b/backend/migrations/20260206060555_notify_triggers_security_definer.down.sql new file mode 100644 index 0000000000..29cd3a7a48 --- /dev/null +++ b/backend/migrations/20260206060555_notify_triggers_security_definer.down.sql @@ -0,0 +1,12 @@ +ALTER FUNCTION notify_config_change() SECURITY INVOKER; +ALTER FUNCTION notify_global_setting_change() SECURITY INVOKER; +ALTER FUNCTION notify_global_setting_delete() SECURITY INVOKER; +ALTER FUNCTION notify_webhook_change() SECURITY INVOKER; +ALTER FUNCTION notify_workspace_envs_change() SECURITY INVOKER; +ALTER FUNCTION notify_workspace_premium_change() SECURITY INVOKER; +ALTER FUNCTION notify_team_plan_status_change() SECURITY INVOKER; +ALTER FUNCTION notify_runnable_version_change() SECURITY INVOKER; +ALTER FUNCTION notify_http_trigger_change() SECURITY INVOKER; +ALTER FUNCTION notify_token_invalidation() SECURITY INVOKER; +ALTER FUNCTION notify_workspace_key_change() SECURITY INVOKER; +ALTER FUNCTION notify_workspace_rate_limit_change() SECURITY INVOKER; diff --git a/backend/migrations/20260206060555_notify_triggers_security_definer.up.sql b/backend/migrations/20260206060555_notify_triggers_security_definer.up.sql new file mode 100644 index 0000000000..137aa8183e --- /dev/null +++ b/backend/migrations/20260206060555_notify_triggers_security_definer.up.sql @@ -0,0 +1,18 @@ +-- Make all notify_event trigger functions SECURITY DEFINER so that +-- INSERT INTO notify_event runs as the function owner (typically the +-- superuser that created the function) rather than the invoking role. +-- This prevents "permission denied for table notify_event" errors when +-- windmill_user or windmill_admin fire these triggers. + +ALTER FUNCTION notify_config_change() SECURITY DEFINER; +ALTER FUNCTION notify_global_setting_change() SECURITY DEFINER; +ALTER FUNCTION notify_global_setting_delete() SECURITY DEFINER; +ALTER FUNCTION notify_webhook_change() SECURITY DEFINER; +ALTER FUNCTION notify_workspace_envs_change() SECURITY DEFINER; +ALTER FUNCTION notify_workspace_premium_change() SECURITY DEFINER; +ALTER FUNCTION notify_team_plan_status_change() SECURITY DEFINER; +ALTER FUNCTION notify_runnable_version_change() SECURITY DEFINER; +ALTER FUNCTION notify_http_trigger_change() SECURITY DEFINER; +ALTER FUNCTION notify_token_invalidation() SECURITY DEFINER; +ALTER FUNCTION notify_workspace_key_change() SECURITY DEFINER; +ALTER FUNCTION notify_workspace_rate_limit_change() SECURITY DEFINER; diff --git a/backend/migrations/20260207000001_concurrent_indexes_v2_job.down.sql b/backend/migrations/20260207000001_concurrent_indexes_v2_job.down.sql new file mode 100644 index 0000000000..ce52b8b53b --- /dev/null +++ b/backend/migrations/20260207000001_concurrent_indexes_v2_job.down.sql @@ -0,0 +1 @@ +-- no-op: indexes are safe to leave in place diff --git a/backend/migrations/20260207000001_concurrent_indexes_v2_job.up.sql b/backend/migrations/20260207000001_concurrent_indexes_v2_job.up.sql new file mode 100644 index 0000000000..b5c035c4d0 --- /dev/null +++ b/backend/migrations/20260207000001_concurrent_indexes_v2_job.up.sql @@ -0,0 +1,41 @@ +-- v2_job: drop obsolete indexes and create new ones +DROP INDEX IF EXISTS root_job_index; +DROP INDEX IF EXISTS root_job_index_by_path_2; +DROP INDEX IF EXISTS root_job_index_by_path; +DROP INDEX IF EXISTS ix_job_workspace_id_created_at_new_6; +DROP INDEX IF EXISTS ix_job_workspace_id_created_at_new_7; +DROP INDEX IF EXISTS ix_job_created_at; +DROP INDEX IF EXISTS ix_v2_job_root_by_path; + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_3 + ON v2_job (workspace_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_8 + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('deploymentcallback') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_9 + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_created_at_new_5 + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('preview', 'flowpreview') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_v2_job_labels + ON v2_job USING GIN (labels) + WHERE labels IS NOT NULL; + +ALTER TABLE v2_job ENABLE ROW LEVEL SECURITY; + +CREATE INDEX IF NOT EXISTS ix_v2_job_workspace_id_created_at + ON v2_job (workspace_id, created_at DESC) + WHERE kind IN ('script', 'flow', 'singlestepflow') AND parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS idx_job_v2_job_root_by_path_2 + ON v2_job (workspace_id, runnable_path) + WHERE parent_job IS NULL; + +CREATE INDEX IF NOT EXISTS ix_job_root_job_index_by_path_2 + ON v2_job (workspace_id, runnable_path, created_at DESC) + WHERE parent_job IS NULL; diff --git a/backend/migrations/20260207000002_concurrent_indexes_v2_job_completed.down.sql b/backend/migrations/20260207000002_concurrent_indexes_v2_job_completed.down.sql new file mode 100644 index 0000000000..ce52b8b53b --- /dev/null +++ b/backend/migrations/20260207000002_concurrent_indexes_v2_job_completed.down.sql @@ -0,0 +1 @@ +-- no-op: indexes are safe to leave in place diff --git a/backend/migrations/20260207000002_concurrent_indexes_v2_job_completed.up.sql b/backend/migrations/20260207000002_concurrent_indexes_v2_job_completed.up.sql new file mode 100644 index 0000000000..b5b953f2ad --- /dev/null +++ b/backend/migrations/20260207000002_concurrent_indexes_v2_job_completed.up.sql @@ -0,0 +1,19 @@ +-- v2_job_completed: drop obsolete indexes and create new ones +DROP INDEX IF EXISTS ix_completed_job_workspace_id_created_at; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_created_at_new; +DROP INDEX IF EXISTS index_completed_job_on_schedule_path; +DROP INDEX IF EXISTS index_completed_on_created; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_created_at_new_2; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_started_at_new; +DROP INDEX IF EXISTS ix_completed_job_workspace_id_started_at_new_2; +DROP INDEX IF EXISTS labeled_jobs_on_jobs; + +CREATE INDEX IF NOT EXISTS labeled_jobs_on_jobs + ON v2_job_completed USING GIN ((result -> 'wm_labels')) + WHERE result ? 'wm_labels'; + +CREATE INDEX IF NOT EXISTS ix_job_completed_completed_at + ON v2_job_completed (completed_at DESC); + +CREATE INDEX IF NOT EXISTS ix_job_workspace_id_completed_at_all + ON v2_job_completed (workspace_id, completed_at DESC); diff --git a/backend/migrations/20260207000003_concurrent_indexes_v2_job_queue.down.sql b/backend/migrations/20260207000003_concurrent_indexes_v2_job_queue.down.sql new file mode 100644 index 0000000000..ce52b8b53b --- /dev/null +++ b/backend/migrations/20260207000003_concurrent_indexes_v2_job_queue.down.sql @@ -0,0 +1 @@ +-- no-op: indexes are safe to leave in place diff --git a/backend/migrations/20260207000003_concurrent_indexes_v2_job_queue.up.sql b/backend/migrations/20260207000003_concurrent_indexes_v2_job_queue.up.sql new file mode 100644 index 0000000000..71048b4a09 --- /dev/null +++ b/backend/migrations/20260207000003_concurrent_indexes_v2_job_queue.up.sql @@ -0,0 +1,12 @@ +-- v2_job_queue: drop obsolete indexes and create new ones +DROP INDEX IF EXISTS concurrency_limit_stats_queue; +DROP INDEX IF EXISTS queue_sort; +DROP INDEX IF EXISTS queue_sort_2; + +CREATE INDEX IF NOT EXISTS queue_sort_v2 + ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) + WHERE running = false; + +CREATE INDEX IF NOT EXISTS v2_job_queue_suspend + ON v2_job_queue (workspace_id, suspend) + WHERE suspend > 0; diff --git a/backend/migrations/20260207000004_concurrent_indexes_other.down.sql b/backend/migrations/20260207000004_concurrent_indexes_other.down.sql new file mode 100644 index 0000000000..ce52b8b53b --- /dev/null +++ b/backend/migrations/20260207000004_concurrent_indexes_other.down.sql @@ -0,0 +1 @@ +-- no-op: indexes are safe to leave in place diff --git a/backend/migrations/20260207000004_concurrent_indexes_other.up.sql b/backend/migrations/20260207000004_concurrent_indexes_other.up.sql new file mode 100644 index 0000000000..4febf6f4e7 --- /dev/null +++ b/backend/migrations/20260207000004_concurrent_indexes_other.up.sql @@ -0,0 +1,16 @@ +-- audit, alerts, script, log_file: drop obsolete indexes and create new ones +DROP INDEX IF EXISTS log_file_hostname_log_ts_idx; + +CREATE INDEX IF NOT EXISTS ix_audit_timestamps + ON audit (timestamp DESC); + +CREATE INDEX IF NOT EXISTS alerts_by_workspace + ON alerts (workspace_id); + +CREATE INDEX IF NOT EXISTS idx_audit_recent_login_activities + ON audit (timestamp, username) + WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh'); + +CREATE INDEX IF NOT EXISTS script_not_archived + ON script (workspace_id, path, created_at DESC) + WHERE archived = false; diff --git a/backend/parsers/windmill-parser-py-imports/src/mapping.rs b/backend/parsers/windmill-parser-py-imports/src/mapping.rs index 533fb76371..cb9a14b27a 100644 --- a/backend/parsers/windmill-parser-py-imports/src/mapping.rs +++ b/backend/parsers/windmill-parser-py-imports/src/mapping.rs @@ -381,5 +381,6 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! { "docx" => "python-docx", "vt" => "vt-py", "grpc" => "grpcio", + "serpapi" => "google-search-results", // Add new entry here ^ }; diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 159b54ffb4..c299bff4af 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -422,11 +422,15 @@ fn parse_expr( match e.as_ref() { Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false), Expr::Attribute(x) => { - if x.value - .as_name_expr() - .is_some_and(|x| x.id.as_str() == "wmill") - { - (parse_typ(x.attr.as_str(), enums, module), false) + if let Some(name) = x.value.as_name_expr() { + match name.id.as_str() { + "wmill" => (parse_typ(x.attr.as_str(), enums, module), false), + "datetime" => { + let full_name = format!("datetime.{}", x.attr.as_str()); + (parse_typ(&full_name, enums, module), false) + } + _ => (Typ::Unknown, false), + } } else { (Typ::Unknown, false) } @@ -493,6 +497,8 @@ fn parse_typ(id: &str, enums: &HashMap, module: Option<&[Stmt] "bytes" => Typ::Bytes, "datetime" => Typ::Datetime, "datetime.datetime" => Typ::Datetime, + "date" => Typ::Date, + "datetime.date" => Typ::Date, "Sql" | "sql" => Typ::Sql, x @ _ if x.starts_with("DynSelect_") => { Typ::DynSelect(x.strip_prefix("DynSelect_").unwrap().to_string()) @@ -620,7 +626,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt Arg { otyp: None, name: "name".to_string(), - typ: Typ::Unknown, + typ: Typ::Datetime, default: Some(json!("")), has_default: true, oidx: None @@ -709,7 +715,7 @@ def main(test1: str, Arg { otyp: None, name: "name".to_string(), - typ: Typ::Unknown, + typ: Typ::Datetime, default: Some(json!("")), has_default: true, oidx: None diff --git a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs index cff5598810..e229d97a8a 100644 --- a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs +++ b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs @@ -295,6 +295,7 @@ fn extract_field_type( "bool" => Typ::Bool, "bytes" => Typ::Bytes, "datetime" => Typ::Datetime, + "date" => Typ::Date, "Any" => Typ::Unknown, // typing.Any maps to Unknown // Custom class - check if it's a model custom_type => { diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql/src/asset_parser.rs index 51b90b379a..9db4e8a0d0 100644 --- a/backend/parsers/windmill-parser-sql/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql/src/asset_parser.rs @@ -87,7 +87,14 @@ impl AssetCollector { .map(|id| id.as_ident().map(|id| id.value.clone())) .collect::>>()? .join("."); - let path = format!("{}/{}", path, specific_table); + + // For Resource assets, use ?table= query parameter syntax + // For Ducklake and DataTable, maintain /table syntax + let path = if *kind == AssetKind::Resource { + format!("{}?table={}", path, specific_table) + } else { + format!("{}/{}", path, specific_table) + }; return Some(ParseAssetsResult { kind: *kind, access_type, path, columns: None }); } } @@ -104,7 +111,14 @@ impl AssetCollector { .map(|id| id.as_ident().map(|id| id.value.clone())) .collect::>>()? .join("."); - format!("{}/{}", path, specific_table) + + // For Resource assets, use ?table= query parameter syntax + // For Ducklake and DataTable, maintain /table syntax + if *kind == AssetKind::Resource { + format!("{}?table={}", path, specific_table) + } else { + format!("{}/{}", path, specific_table) + } } else { path.clone() }; @@ -845,7 +859,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::Resource, - path: "u/user/pg_resource/table1".to_string(), + path: "u/user/pg_resource?table=table1".to_string(), access_type: Some(R), columns: None },]) @@ -870,6 +884,75 @@ mod tests { ); } + #[test] + fn test_sql_asset_parser_resource_vs_ducklake_syntax() { + // Test that Resource uses ?table= while Ducklake uses /table + let input_resource = r#" + ATTACH 'res://u/user/pg_resource' AS db (TYPE postgres); + SELECT * FROM db.users; + "#; + let s = parse_assets(input_resource).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/user/pg_resource?table=users".to_string(), + access_type: Some(R), + columns: None + },]) + ); + + let input_ducklake = r#" + ATTACH 'ducklake://my_lake' AS dl; + SELECT * FROM dl.users; + "#; + let s = parse_assets(input_ducklake).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "my_lake/users".to_string(), + access_type: Some(R), + columns: None + },]) + ); + + let input_datatable = r#" + ATTACH 'datatable://dt1' AS dt; + SELECT * FROM dt.users; + "#; + let s = parse_assets(input_datatable).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt1/users".to_string(), + access_type: Some(R), + columns: None + },]) + ); + } + + #[test] + fn test_sql_asset_parser_resource_with_long_path() { + // Test that Resource works with paths longer than 3 components + let input = r#" + ATTACH 'res://u/diego/a/b/c/my_postgres_resource' AS db (TYPE postgres); + USE db; + SELECT * FROM my_table; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/diego/a/b/c/my_postgres_resource?table=my_table".to_string(), + access_type: Some(R), + columns: None + },]) + ); + } + #[test] fn test_sql_asset_parser_table_with_schema() { let input = r#" diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 4f88fe17c1..2463995606 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -114,13 +114,14 @@ fn merge_access_types(a: AssetUsageAccessType, b: AssetUsageAccessType) -> Asset // But never used it. In that case we don't know which table is being used, // but we still want to add the main datatable as an asset with unknown access type. // -// This function takes care of the fact that assets can be suffixed (e.g. "main/users") +// This function takes care of the fact that assets can be suffixed (e.g. "main/users" or "u/user/resource?table=table1") pub fn asset_was_used(assets: &Vec, (kind, path): (AssetKind, &String)) -> bool { assets.iter().any(|a| { let a_path = a.path.as_str(); + // Check for /table suffix (Ducklake, DataTable) or ?table= suffix (Resource) let has_same_path_base = a_path .strip_prefix(path) - .map(|p| p.starts_with('/')) + .map(|p| p.starts_with('/') || p.starts_with('?')) .unwrap_or(false); (has_same_path_base || a_path == path) && a.kind == kind }) diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index c8f19f33e1..ea062356ce 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -65,6 +65,7 @@ pub enum Typ { List(Box), Bytes, Datetime, + Date, Resource(String), Email, Sql, diff --git a/backend/parsers/windmill-sql-datatype-parser-wasm/src/lib.rs b/backend/parsers/windmill-sql-datatype-parser-wasm/src/lib.rs index 3517e0c450..2424a7859c 100644 --- a/backend/parsers/windmill-sql-datatype-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-sql-datatype-parser-wasm/src/lib.rs @@ -10,6 +10,7 @@ fn to_str(typ: Typ) -> String { Typ::List(t) => format!("list-{}", to_str(*t)), Typ::Bytes => "bytes".to_string(), Typ::Datetime => "datetime".to_string(), + Typ::Date => "date".to_string(), _ => "unknown".to_string(), } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 389bfff71a..be424a6712 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -127,27 +127,8 @@ mod monitor; mod windows_service_ee; pub fn setup_deno_runtime() -> anyhow::Result<()> { - // https://github.com/denoland/deno/blob/main/cli/main.rs#L477 #[cfg(feature = "deno_core")] - let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ - "--stack-size=1024".to_string(), - // TODO(bartlomieju): I think this can be removed as it's handled by `deno_core` - // and its settings. - // deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript - // TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast - "--no-harmony-import-assertions".to_string(), - ]) - .into_iter() - .skip(1) - .collect::>(); - - #[cfg(feature = "deno_core")] - if !unrecognized_v8_flags.is_empty() { - println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags); - } - - #[cfg(feature = "deno_core")] - deno_core::JsRuntime::init_platform(None, false); + windmill_runtime_nativets::setup_deno_runtime()?; Ok(()) } @@ -1661,6 +1642,19 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alert UI setting"); } } + "workspace_telemetry_enabled" => { + // Read the new value from the database and log it + let enabled = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = 'workspace_telemetry_enabled'" + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .and_then(|v| v.as_bool()) + .unwrap_or(false); + tracing::info!("Workspace telemetry setting changed: enabled={}", enabled); + } _ => { tracing::info!("Unrecognized Global Setting Change Payload: {:?}", payload); } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 07a4c5069b..bfbfaa2d87 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -84,10 +84,11 @@ use windmill_common::{ use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload}; use windmill_worker::{ - handle_job_error, JobCompletedSender, OtelTracingProxySettings, SameWorkerSender, - BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, - NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, - PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, + result_processor::handle_job_error, JobCompletedSender, OtelTracingProxySettings, + SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, + KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, + OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, + POWERSHELL_REPO_URL, }; #[cfg(feature = "parquet")] @@ -1102,7 +1103,6 @@ async fn delete_expired_jobs_batch( job_retention_secs ); - // Delete related records for this batch if let Err(e) = sqlx::query!( "DELETE FROM job_stats WHERE job_id = ANY($1)", &deleted_jobs @@ -1782,6 +1782,19 @@ pub async fn monitor_db( } }; + let cleanup_job_live_rows_f = async { + if server_mode && !initial_load { + if let Some(db) = conn.as_sql() { + if let Err(e) = cleanup_job_perms_orphaned(&db).await { + tracing::error!("Error cleaning up orphaned job_perms: {:?}", e); + } + if let Err(e) = cleanup_job_result_stream_orphaned_jobs(&db).await { + tracing::error!("Error cleaning up orphaned job_result_stream_v2: {:?}", e); + } + } + } + }; + // run every hour (60 minutes / 30 seconds = 120) let cleanup_worker_group_stats_f = async { if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) { @@ -1883,7 +1896,6 @@ pub async fn monitor_db( }; let update_min_worker_version_f = async { - #[cfg(not(feature = "test_job_debouncing"))] windmill_common::min_version::update_min_version( conn, _worker_mode, @@ -1956,6 +1968,7 @@ pub async fn monitor_db( cleanup_debounce_keys_f, cleanup_debounce_keys_completed_f, cleanup_flow_iterator_data_f, + cleanup_job_live_rows_f, cleanup_worker_group_stats_f, native_triggers_sync_f, cleanup_notify_events_f, @@ -1963,7 +1976,7 @@ pub async fn monitor_db( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") .execute(db) .await?; Ok(()) @@ -2103,7 +2116,11 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b } else { let wc = WORKER_CONFIG.read().await; let config = config.unwrap(); - let has_dedicated = config.dedicated_worker.is_some() || config.dedicated_workers.as_ref().is_some_and(|dws| !dws.is_empty()); + let has_dedicated = config.dedicated_worker.is_some() + || config + .dedicated_workers + .as_ref() + .is_some_and(|dws| !dws.is_empty()); if *wc != config || has_dedicated { if kill_if_change { if has_dedicated @@ -3058,6 +3075,37 @@ RETURNING key,job_id Ok(()) } +async fn cleanup_job_perms_orphaned(db: &DB) -> error::Result<()> { + let result = sqlx::query_scalar!( + "DELETE FROM job_perms +WHERE job_id NOT IN (SELECT id FROM v2_job_queue) +RETURNING job_id" + ) + .fetch_all(db) + .await?; + + if !result.is_empty() { + tracing::info!("Cleaned up {} orphaned job_perms rows", result.len()); + } + Ok(()) +} + +async fn cleanup_job_result_stream_orphaned_jobs(db: &DB) -> error::Result<()> { + let result = sqlx::query!( + "DELETE FROM job_result_stream_v2 WHERE job_id NOT IN (SELECT id FROM v2_job_queue) RETURNING job_id", + ) + .fetch_all(db) + .await?; + + if result.len() > 0 { + tracing::info!( + "Cleaned up {} orphaned job_result_stream_v2 rows", + result.len() + ); + } + Ok(()) +} + async fn cleanup_flow_iterator_data_orphaned_jobs(db: &DB) -> error::Result<()> { let result = sqlx::query!( " diff --git a/backend/tests/apps.rs b/backend/tests/apps.rs new file mode 100644 index 0000000000..869858a265 --- /dev/null +++ b/backend/tests/apps.rs @@ -0,0 +1,282 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn app_url(port: u16, endpoint: &str, path: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/apps/{endpoint}/{path}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn authed_get(port: u16, endpoint: &str, path: &str) -> reqwest::Response { + authed(client().get(app_url(port, endpoint, path))) + .send() + .await + .unwrap() +} + +fn new_app(path: &str, summary: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": summary, + "value": { + "type": "rawapp", + "inline_script": null + }, + "policy": { + "execution_mode": "anonymous", + "triggerables": {}, + "on_behalf_of": null, + "on_behalf_of_email": null + } + }) +} + +#[sqlx::test(fixtures("base"))] +async fn test_app_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/apps"); + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_app("u/test-user/test_app", "Test app")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + // create second app + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_app("u/test-user/another_app", "Another app")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create another: {}", resp.text().await?); + + // --- exists --- + let resp = authed_get(port, "exists", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "exists", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get by path --- + let resp = authed_get(port, "get/p", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_app"); + assert_eq!(body["summary"], "Test app"); + + // get not found + let resp = authed_get(port, "get/p", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- get draft --- + let resp = authed_get(port, "get/draft", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_app"); + + // --- get lite --- + let resp = authed_get(port, "get/lite", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!( + list.len() >= 2, + "expected at least 2 apps, got {}", + list.len() + ); + assert!(list.iter().any(|a| a["path"] == "u/test-user/test_app")); + + // --- list_search --- + let resp = authed(client().get(format!("{base}/list_search"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + + // --- history --- + let resp = authed_get(port, "history/p", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + let history = resp.json::>().await?; + assert!(!history.is_empty()); + + // --- get by version --- + let version = &history[0]["version"]; + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/apps/get/v/{version}" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- custom_path_exists --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/apps/custom_path_exists/nonexistent" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- secret_of --- + let resp = authed_get(port, "secret_of", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + let secret_id = resp.text().await?; + assert!(!secret_id.is_empty()); + + // --- get_latest_version --- + let resp = authed_get(port, "get_latest_version", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + + // --- public_app (unauthed, by secret) --- + let resp = client() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/apps_u/public_app/{secret_id}" + )) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "public_app: {}", + resp.text().await? + ); + + // --- secret_of_latest_version --- + let resp = authed_get(port, "secret_of_latest_version", "u/test-user/test_app").await; + assert_eq!(resp.status(), 200); + let secret = resp.text().await?; + assert!(!secret.is_empty()); + + // --- list_paths_from_workspace_runnable --- + let resp = authed(client().get(format!( + "{base}/list_paths_from_workspace_runnable/script/u/test-user/test_app" + ))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "list_paths_from_workspace_runnable: {}", + resp.text().await? + ); + + + // --- history_update --- + let app_body = authed_get(port, "get/p", "u/test-user/test_app").await; + let app = app_body.json::().await?; + let app_id = &app["id"]; + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/apps/history_update/a/{app_id}/v/{version}" + ))) + .json(&json!({"deployment_msg": "deployed v1"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "history_update: {}", + resp.text().await? + ); + + // --- update --- + let resp = authed(client().post(app_url(port, "update", "u/test-user/test_app"))) + .json(&json!({ + "summary": "Updated app", + "policy": { + "execution_mode": "anonymous", + "triggerables": {}, + "on_behalf_of": null, + "on_behalf_of_email": null + } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + + // verify update + let resp = authed_get(port, "get/p", "u/test-user/test_app").await; + let body = resp.json::().await?; + assert_eq!(body["summary"], "Updated app"); + + // --- delete --- + let resp = authed(client().delete(app_url(port, "delete", "u/test-user/another_app"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "exists", "u/test-user/another_app").await; + assert_eq!(resp.json::().await?, false); + + // ===== Hub endpoints (require external network, expect 500 or 200) ===== + + // --- hub/list --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/apps/hub/list" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/list: unexpected status {}", + resp.status() + ); + + // --- hub/get --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/apps/hub/get/1" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/get: unexpected status {}", + resp.status() + ); + + // --- hub/get_raw --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/apps/hub/get_raw/1" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/get_raw: unexpected status {}", + resp.status() + ); + + Ok(()) +} diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index ba89de3019..d19fdac786 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -87,6 +87,12 @@ impl ApiServer { Self::start_inner(db, true).await } + /// Start the API server with server_mode=true so trigger listeners are active. + /// Alias for `start_agent_mode` with a clearer name for trigger e2e tests. + pub async fn start_with_listeners(db: Pool) -> anyhow::Result { + Self::start_inner(db, true).await + } + async fn start_inner(db: Pool, agent_mode: bool) -> anyhow::Result { let (tx, rx) = tokio::sync::broadcast::channel::<()>(1); @@ -143,11 +149,17 @@ pub struct RunJob { pub payload: JobPayload, pub args: serde_json::Map, pub scheduled_for_o: Option>, + pub email: String, } impl From for RunJob { fn from(payload: JobPayload) -> Self { - Self { payload, args: Default::default(), scheduled_for_o: None } + Self { + payload, + args: Default::default(), + scheduled_for_o: None, + email: "test@windmill.dev".to_string(), + } } } @@ -165,8 +177,13 @@ impl RunJob { self } + pub fn email(mut self, email: impl Into) -> Self { + self.email = email.into(); + self + } + pub async fn push(self, db: &Pool) -> Uuid { - let RunJob { payload, args, scheduled_for_o } = self; + let RunJob { payload, args, scheduled_for_o, email } = self; let mut hm_args = std::collections::HashMap::new(); for (k, v) in args { hm_args.insert(k, windmill_common::worker::to_raw_value(&v)); @@ -180,7 +197,7 @@ impl RunJob { payload, windmill_queue::PushArgs::from(&hm_args), /* user */ "test-user", - /* email */ "test@windmill.dev", + /* email */ &email, /* permissioned_as */ "u/test-user".to_string(), /* token_prefix */ None, scheduled_for_o, @@ -313,6 +330,9 @@ pub fn spawn_test_worker( conn: &Connection, port: u16, ) -> (KillpillSender, tokio::task::JoinHandle<()>) { + #[cfg(feature = "deno_core")] + windmill_runtime_nativets::setup_deno_runtime().expect("V8 init failed"); + std::fs::DirBuilder::new() .recursive(true) .create(windmill_worker::GO_BIN_CACHE_DIR) @@ -684,8 +704,10 @@ pub async fn run_deployed_relative_imports( language, priority: None, apply_preprocessor: false, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), - debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), }) .push(&db2) .await; @@ -734,8 +756,10 @@ pub async fn run_preview_relative_imports( cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default().into(), - debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + concurrency_settings: + windmill_common::runnable_settings::ConcurrencySettings::default().into(), + debouncing_settings: + windmill_common::runnable_settings::DebouncingSettings::default(), })) .push(&db2) .await; diff --git a/backend/tests/debouncing.rs b/backend/tests/debouncing.rs deleted file mode 100644 index 0f7c68b7b8..0000000000 --- a/backend/tests/debouncing.rs +++ /dev/null @@ -1,2809 +0,0 @@ -mod common; - -#[cfg(feature = "test_job_debouncing")] -use windmill_api_client::types::NewScript; - -#[cfg(feature = "test_job_debouncing")] -fn quick_ns( - content: &str, - language: windmill_api_client::types::ScriptLang, - path: &str, - lock: Option, - parent_hash: Option, -) -> NewScript { - NewScript { - content: content.into(), - language, - lock, - parent_hash, - path: path.into(), - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - description: "".to_string(), - draft_only: None, - envs: vec![], - is_template: None, - kind: None, - summary: "".to_string(), - tag: None, - schema: std::collections::HashMap::new(), - ws_error_handler_muted: Some(false), - priority: None, - delete_after_use: None, - timeout: None, - restart_unless_cancelled: None, - deployment_message: None, - concurrency_key: None, - visible_to_runner_only: None, - no_main_func: None, - codebase: None, - has_preprocessor: None, - on_behalf_of_email: None, - assets: vec![], - } -} - -#[cfg(feature = "test_job_debouncing")] -mod dependency_job_debouncing { - async fn trigger_djob_for( - client: &windmill_api_client::Client, - path: &str, - parent_hash: &str, - content: Option, - ) { - use super::quick_ns; - use windmill_api_client::types::ScriptLang; - client - .create_script( - "test-workspace", - &quick_ns( - &content.unwrap_or( - " -def main(): - pass - " - .into(), - ), - ScriptLang::Python3, - path, - None, - Some(parent_hash.into()), - ), - ) - .await - .unwrap(); - } - // TODO: test workspaces specific things, - - /// # Double referenced even - /// It follows this topology: - /// - /// ┌─FLOW──────────┐ - /// │┌───┐┌───┐┌───┐│ - /// ││ A ││ B ││ C ││ - /// │└─▲─┘▲───▲└─▲─┘│ - /// └──┼──┼───┼──┼──┘ - /// ┌┴──┴┐ ┌┴──┴┐ - /// │L_LF│ │R_LF│ - /// └────┘ └────┘ - /// - /// p.s: "LF" stands for "Leaf", "L" - "Left", "R" - "Right" - mod flows { - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs}; - use crate::dependency_job_debouncing::trigger_djob_for; - use std::time::Duration; - use tokio::time::sleep; - use tokio_stream::StreamExt; - - /// 1. LLF and RLF create two djobs for flow at the same and fall into single debounce - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_1(db: sqlx::Pool) -> anyhow::Result<()> { - // This tests if debouncing and consolidation works. - // Also makes sures that dependency job does not create new flow version - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - // Verify locks are empty - { - assert_eq!( - sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") - .fetch_all(&db) - .await - .unwrap(), - vec![ - Some("# py: 3.11\n".into()), - Some("# py: 3.11\n".into()), - Some("# py: 3.11\n".into()) - ] - ); - } - - // Trigger both at the same time. - { - trigger_djob_for( - &client, - "f/dre/leaf_left", - "0000000000051658", - Some("#requirements:\n#bottle==0.13.2\ndef main():\npass".into()), - ) - .await; - - trigger_djob_for( - &client, - "f/dre/leaf_right", - "000000000005165B", - Some("#requirements:\n#tiny==0.1.3\ndef main():\npass".into()), - ) - .await; - } - - in_test_worker( - &db, - async { - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre/leaf_left" - ); - - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre/leaf_right" - ); - - // Let jobs propagate - sleep(Duration::from_secs(2)).await; - - // Verify there is only one queued job that is scheduled for atleast 3s ahead. - { - let q = sqlx::query_scalar!( - "SELECT (scheduled_for - created_at) FROM v2_job_queue" - ) - .fetch_all(&db) - .await - .unwrap(); - - assert_eq!(1, q.len()); - assert!(dbg!(q[0].unwrap().microseconds) > 1_000_000 /* 1 second */); - } - - // Verify debounce_stale_data and debounce_key - { - let q = sqlx::query!( - "SELECT - dsd.to_relock, - dk.key - FROM debounce_key dk - JOIN debounce_stale_data dsd ON dk.job_id = dsd.job_id" - ) - .fetch_all(&db) - .await - .unwrap(); - - // Should be single entry - assert!(q.len() == 1); - - // This verifies that all nodes_to_relock are consolidated correctly - // AND there is no doublicats - assert_eq!( - q[0].to_relock.clone().unwrap(), - vec!["a".to_owned(), "b".to_owned(), "c".to_owned()] - ); - - // Should be workspace specific and these specific tests cover only dependency job debouncing - assert_eq!( - q[0].key.clone(), - "test-workspace:f/dre/flow:dependency".to_owned(), - ); - } - - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre/flow" - ); - }, - port, - ) - .await; - - // Verify latest flow.version property - { - // Latest flow version should not be initial one - assert_eq!( - 1, // Automatically assigned - dbg!(sqlx::query_scalar!( - "SELECT versions[2] FROM flow WHERE path = 'f/dre/flow'" - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap()) - ); - - // Only second element should be our initial version - assert_eq!( - 1443253234253454, // < Predefined in fixture - dbg!(sqlx::query_scalar!( - "SELECT versions[1] FROM flow WHERE path = 'f/dre/flow'" - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap()) - ); - } - - // Verify that there is only two versions of flow in global flow_version - { - assert_eq!( - 2, - sqlx::query_scalar!( - "SELECT COUNT(*) FROM flow_version WHERE path = 'f/dre/flow'" - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - } - - // Verify locks - { - assert_eq!( - sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") - .fetch_all(&db) - .await - .unwrap(), - vec![ - Some("# py: 3.11\nbottle==0.13.2".into()), - Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()), - Some("# py: 3.11\ntiny==0.1.3".into()) - ] - ); - } - - // TODO: - // tracing_assertions::assert_has_events!([info("This is supposed to be called")]); - // 2025-10-06T14:31:10.832469Z WARN windmill-worker/src/worker.rs:1593: pull took more than 0.1s (0.222477345) this is a sign that the database is undersized for this load. empty: true, err: true worker=wk-default-nixos-EzDEL hostname=nixos - - // Verify cleanup - { - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - } - - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_left(db: sqlx::Pool) -> anyhow::Result<()> { - use crate::common::RunJob; - - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - // TODO: We don't care about timer. If there is no timer, it will be set automatically for djobs?? - let (_client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - // Trigger both at the same time. - { - let mut args = std::collections::HashMap::new(); - args.insert( - "dbg_djob_sleep".to_owned(), - // Execution should take this seconds - windmill_common::worker::to_raw_value(&20), - ); - - args.insert( - "triggered_by_relative_import".to_owned(), - // Execution should take this seconds - windmill_common::worker::to_raw_value(&()), - ); - - let (_flow_id, new_tx) = windmill_queue::push( - &db, - windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - "test-workspace", - windmill_common::jobs::JobPayload::FlowDependencies { - path: "f/dre/flow".to_owned(), - dedicated_worker: None, - version: 1443253234253454, - }, - windmill_queue::PushArgs { args: &args, extra: None }, - "admin", - "admin@windmill.dev", - "admin".to_owned(), - Some("trigger.dependents.to.recompute.dependencies"), - // Debounce period - Some(chrono::Utc::now() + chrono::Duration::seconds(5)), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - None, - None, - None, - ) - .await - .unwrap(); - new_tx.commit().await.unwrap(); - - // let handle = { - // // let mut completed = listen_for_completed_jobs(&db).await; - // let db2 = db.clone(); - // // let uuid = flow_id.clone(); - // tokio::spawn(async move { - // in_test_worker( - // &db2, - // tokio::time::sleep(tokio::time::Duration::from_secs(60)), - // // completed.find(&uuid), - // port, - // ) - // .await; - // }) - // }; - - let db2 = db.clone(); - in_test_worker( - &db2, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre/leaf_right".to_owned(), - hash: 333403.into(), - language: windmill_common::scripts::ScriptLang::Python3, - dedicated_worker: None, - }) - // .arg("dbg_djob_sleep", serde_json::json!(10)) - .run_until_complete(&db, false, port) - .await; - - RunJob::from(windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre/leaf_left".to_owned(), - hash: 333400.into(), - language: windmill_common::scripts::ScriptLang::Python3, - dedicated_worker: None, - }) - // So set it to this long - .arg("dbg_djob_sleep", serde_json::json!(10)) - .run_until_complete(&db, false, port) - .await; - - completed.next().await; // leaf_right - completed.next().await; // leaf_left - completed.next().await; // importer - completed.next().await; // importer - }, - port, - ) - .await; - } - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 0 - ); - - let r = sqlx::query_scalar!("SELECT runnable_id FROM v2_job ORDER BY created_at DESC") - .fetch_all(&db) - .await - .unwrap(); - - assert_eq!(r.len(), 4); - assert!(r.contains(&Some(1))); - assert!(r.contains(&Some(333400))); - assert!(r.contains(&Some(333403))); - assert!(r.contains(&Some(1443253234253454))); - - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_2(db: sqlx::Pool) -> anyhow::Result<()> { - let (_client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - { - let mut mvsd = windmill_common::worker::MIN_VERSION_IS_AT_LEAST_1_440 - .write() - .await; - *mvsd = true; - } - - // Function to create a dependency job - let create_dependency_job = - |delay, - nodes_to_relock, - db: sqlx::Pool, - version, - debounce_job_id_o| async move { - let mut args = std::collections::HashMap::new(); - args.insert( - "dbg_sleep_between_pull_and_debounce_key_removal".to_owned(), - windmill_common::worker::to_raw_value(&delay), - ); - - args.insert( - "nodes_to_relock".to_owned(), - windmill_common::worker::to_raw_value(&nodes_to_relock), - ); - - args.insert( - "triggered_by_relative_import".to_string(), - windmill_common::worker::to_raw_value(&()), - ); - - args.insert( - "dbg_create_job_for_unexistant_flow_version".to_string(), - windmill_common::worker::to_raw_value(&()), - ); - - let (job_uuid, new_tx) = windmill_queue::push( - &db, - windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - "test-workspace", - windmill_common::jobs::JobPayload::FlowDependencies { - path: "f/dre/flow".to_owned(), - dedicated_worker: None, - version, - }, - windmill_queue::PushArgs { args: &args, extra: None }, - "admin", - "admin@windmill.dev", - "admin".to_owned(), - Some("trigger.dependents.to.recompute.dependencies"), - Some(chrono::Utc::now()), // Schedule immediately - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - debounce_job_id_o, - None, - None, - ) - .await - .unwrap(); - - new_tx.commit().await.unwrap(); - job_uuid - }; - - // Push the first dependency job - let job1 = - create_dependency_job(2, vec!["a", "b"], db.clone(), 1443253234253454, None).await; - - let db2 = db.clone(); - in_test_worker( - &db2, - async { - // Small delay to ensure the job is marked as running - tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; - - // Now is the time when the job is pulled, but debounce_key is not yet cleared. - { - assert!(sqlx::query_scalar!( - "SELECT running FROM v2_job_queue WHERE id = $1", - job1 - ) - .fetch_one(&db) - .await - .unwrap()); - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM debounce_key") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 1 - ); - - dbg!(sqlx::query_scalar!("SELECT kind::text FROM v2_job") - .fetch_one(&db) - .await - .unwrap() - .unwrap()); - } - - let job2 = - create_dependency_job(0, vec!["b", "c"], db.clone(), 1, Some(job1)).await; - - // Process the first job completion, and the second job should also get debounced by this one - completed.next().await; - - // Verify that both jobs were created and processed - assert_eq!(job1, job2, "Second job should be debounced"); - }, - port, - ) - .await; - - assert_eq!( - vec![1443253234253454, 1], - sqlx::query_scalar!("SELECT versions FROM flow WHERE path = 'f/dre/flow'") - .fetch_one(&db) - .await - .unwrap() - ); - - // Verify cleanup - all debounce entries should be cleaned up - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "All debounce_key entries should be cleaned up after job completion" - ); - - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "All debounce_stale_data entries should be cleaned up after job completion" - ); - - // Verify locks - { - assert_eq!( - sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") - .fetch_all(&db) - .await - .unwrap(), - vec![ - Some("# py: 3.11\nbottle==0.13.2".into()), - Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()), - Some("# py: 3.11\ntiny==0.1.3".into()) - ] - ); - } - - Ok(()) - } - /// 2. Same as second test, however first flow djob will take longer than second debounce. - /// NOTE: This test should be ran in debug mode with `private` features enabled. In release it will not work properly. - #[cfg(all(feature = "python", feature = "private"))] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - // #[windmill::all_min_versions] - async fn test_3(db: sqlx::Pool) -> anyhow::Result<()> { - // This tests checks if concurrency limit works correcly and there is no race conditions. - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - // At this point we should have two - let mut job_ids = vec![]; - let push_job = |delay, version, db, nodes_to_relock, debounce_job_id_o| async move { - let mut args = std::collections::HashMap::new(); - args.insert( - "dbg_djob_sleep".to_owned(), - // First one will create delay for 5 seconds - // The second will have no delay at all. - windmill_common::worker::to_raw_value(&delay), - ); - - args.insert( - "nodes_to_relock".to_owned(), - windmill_common::worker::to_raw_value(&nodes_to_relock), - ); - - args.insert( - "triggered_by_relative_import".to_string(), - windmill_common::worker::to_raw_value(&()), - ); - - let (job_uuid, new_tx) = windmill_queue::push( - &db, - windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - "test-workspace", - windmill_common::jobs::JobPayload::FlowDependencies { - path: "f/dre/flow".to_owned(), - dedicated_worker: None, - // In newest versions we pass the current version to the djob - // version: 1443253234253454, - version, - }, - windmill_queue::PushArgs { args: &args, extra: None }, - "admin", - "admin@windmill.dev", - "admin".to_owned(), - Some("trigger.dependents.to.recompute.dependencies"), - // Schedule for now. - Some(chrono::Utc::now()), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - debounce_job_id_o, - None, - None, - None, - ) - .await - .unwrap(); - - new_tx.commit().await.unwrap(); - - job_uuid - }; - - // Push first - job_ids.push(push_job(5, 1443253234253454, db.clone(), ["a", "b"], None).await); - - // Verify debounce_stale_data and debounce_key - { - let q = sqlx::query!("SELECT COUNT(*) FROM debounce_key") - .fetch_all(&db) - .await - .unwrap(); - - // Should be single entry - assert_eq!(q.len(), 1); - } - - // Start the first one in the background - let handle = { - let mut completed = listen_for_completed_jobs(&db).await; - let db2 = db.clone(); - tokio::spawn(async move { - in_test_worker( - &db2, - // sleep(Duration::from_secs(7)), - completed.next(), // Only wait for the single job. We are going to spawn another worker for second one. - port, - ) - .await; - }) - }; - - // Wait for the job to be created and started - // This way next job is not going to be consumed by the first one. - sleep(Duration::from_secs(2)).await; - - // Push second - job_ids.push(push_job(0, 1, db.clone(), ["b", "c"], None).await); - - // Wait for the second one to finish in separate worker. - // in_test_worker(&db, completed.next(), port).await; - in_test_worker( - &db, - async { - // First job will be pulled - completed.next().await; - // However since we have concurrency limit enabled it will get rescheduled by creation of new djob. - // So we have to wait for that one as well. - completed.next().await; - }, - port, - ) - .await; - - // Wait for the first one - handle.await.unwrap(); - - // Verify locks - { - assert_eq!( - sqlx::query_scalar!("SELECT jsonb_array_elements(value->'modules')->'value'->>'lock' AS lock FROM flow") - .fetch_all(&db) - .await - .unwrap(), - vec![ - Some("# py: 3.11\nbottle==0.13.2".into()), - Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()), - Some("# py: 3.11\ntiny==0.1.3".into()) - ] - ); - } - // Verify that we have expected outcome - { - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job",) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 2 - ); - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_completed",) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 2 - ); - // Check that two jobs were executed sequentially - assert!(sqlx::query_scalar!( - " -SELECT - j1.completed_at < j2.started_at -FROM - v2_job_completed j1, - v2_job_completed j2 -WHERE - j1.id = $1 - AND j2.id = $2", - job_ids[0], - job_ids[1], - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap()); - } - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_min_version_supports_debouncing( - db: sqlx::Pool, - ) -> anyhow::Result<()> { - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let db = &db; - - crate::common::in_test_worker( - db, - async { - let job_template = crate::common::RunJob::from( - windmill_common::jobs::JobPayload::FlowDependencies { - path: "f/dre/flow".to_owned(), - dedicated_worker: None, - version: 1443253234253454, - }, - ) - .push_arg_scheduled_for_o(Some(chrono::Utc::now())) - .arg("triggered_by_relative_import", serde_json::json!(())); - - // This will push to the top level worker - let debounce_job_id = job_template.clone().push(db).await; - - // Will have space to run in parallel but in it's own worker - job_template - .push_arg_debounce_job_id_o(Some(debounce_job_id)) - .run_until_complete(db, false, port) - .await; - }, - port, - ) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // And there is only supposed to be one job. - assert_eq!( - sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_job_completed WHERE status = 'success'" - ) - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 1 - ); - - Ok(()) - } - // NOTE: Don't run in parallel with other tests - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - #[ignore] - async fn test_min_version_does_not_support_debouncing( - db: sqlx::Pool, - ) -> anyhow::Result<()> { - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = false; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let db = &db; - - crate::common::in_test_worker( - db, - async { - let job_template = crate::common::RunJob::from( - windmill_common::jobs::JobPayload::FlowDependencies { - path: "f/dre/flow".to_owned(), - dedicated_worker: None, - version: 1443253234253454, - }, - ) - .push_arg_scheduled_for_o(Some(chrono::Utc::now())) - .arg("triggered_by_relative_import", serde_json::json!(())); - - // This will push to the top level worker - let debounce_job_id = job_template.clone().push(db).await; - - // Will have space to run in parallel but in it's own worker - job_template - .push_arg_debounce_job_id_o(Some(debounce_job_id)) - .run_until_complete(db, false, port) - .await; - }, - port, - ) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // There are supposed to be two jobs, since debouncing is disabled. - assert_eq!( - sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_job_completed WHERE status = 'success'" - ) - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 2 - ); - - Ok(()) - } - // TODO: - // test that update or create flow that should bypass debouncing - } - - /// ## Testing for Apps - /// For apps we are going to do similar tests that we did for flows - mod apps { - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs}; - use crate::dependency_job_debouncing::trigger_djob_for; - use std::time::Duration; - use tokio::time::sleep; - use tokio_stream::StreamExt; - - /// 1. LLF and RLF create two djobs for flow at the same and fall into single debounce - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_1(db: sqlx::Pool) -> anyhow::Result<()> { - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - // This tests if debouncing and consolidation works. - // Also makes sures that dependency job does not create new flow version - - let (client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // Trigger both at the same time. - // It will create two immediate dependency jobs - { - trigger_djob_for( - &client, - "f/dre_app/leaf_left", - "0000000000069CF8", - Some("#requirements:\n#bottle==0.13.2\ndef main():\npass".into()), - ) - .await; - - trigger_djob_for( - &client, - "f/dre_app/leaf_right", - "0000000000069CFB", - Some("#requirements:\n#tiny==0.1.3\ndef main():\npass".into()), - ) - .await; - } - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 2 - ); - - // Spawn single worker. - in_test_worker( - &db, - async { - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre_app/leaf_left" - ); - - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre_app/leaf_right" - ); - - // Verify there is only one queued job that is scheduled for atleast 3s ahead. - { - let q = sqlx::query_scalar!( - "SELECT (scheduled_for - created_at) FROM v2_job_queue" - ) - .fetch_all(&db) - .await - .unwrap(); - - assert_eq!(1, q.len()); - assert!(dbg!(q[0].unwrap().microseconds) > 2_000_000); - } - - // Verify debounce_stale_data and debounce_key - { - let q = sqlx::query!( - "SELECT - dsd.to_relock, - dk.key - FROM debounce_key dk - JOIN debounce_stale_data dsd ON dk.job_id = dsd.job_id" - ) - .fetch_all(&db) - .await - .unwrap(); - - // Should be single entry - assert!(q.len() == 1); - - // This verifies that all nodes_to_relock are consolidated correctly - // AND there is no doublicats - assert_eq!( - q[0].to_relock.clone().unwrap(), - vec!["a".to_owned(), "b".to_owned(), "c".to_owned()] - ); - - // Should be workspace specific and these specific tests cover only dependency job debouncing - assert_eq!( - q[0].key.clone(), - "test-workspace:f/dre_app/app:dependency".to_owned(), - ); - } - - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre_app/app" - ); - }, - port, - ) - .await; - - // Verify App states - { - let q = dbg!(sqlx::query_scalar!( - "SELECT versions FROM app WHERE path = 'f/dre_app/app'" - ) - .fetch_one(&db) - .await - .unwrap()); - - assert_eq!(2, q.len()); - - // There is also supposed to be this amount of app_versions - assert_eq!( - 2, - sqlx::query_scalar!("SELECT COUNT(*) FROM app_version WHERE app_id = '2'") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - } - - // Verify cleanup - { - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - } - - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_left(db: sqlx::Pool) -> anyhow::Result<()> { - use crate::common::RunJob; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - // TODO: We don't care about timer. If there is no timer, it will be set automatically for djobs?? - let (_client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - let mut args = std::collections::HashMap::new(); - args.insert( - "dbg_djob_sleep".to_owned(), - // Execution should take this seconds - windmill_common::worker::to_raw_value(&20), - ); - args.insert( - "triggered_by_relative_import".to_owned(), - // Execution should take this seconds - windmill_common::worker::to_raw_value(&()), - ); - - let (_flow_id, new_tx) = windmill_queue::push( - &db, - windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - "test-workspace", - windmill_common::jobs::JobPayload::AppDependencies { - path: "f/dre_app/app".to_owned(), - version: 0, - }, - windmill_queue::PushArgs { args: &args, extra: None }, - "admin", - "admin@windmill.dev", - "admin".to_owned(), - Some("trigger.dependents.to.recompute.dependencies"), - // Debounce period - Some(chrono::Utc::now() + chrono::Duration::seconds(5)), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - None, - None, - None, - None, - ) - .await - .unwrap(); - new_tx.commit().await.unwrap(); - - // let mut handle = { - // let mut completed = listen_for_completed_jobs(&db).await; - // let db2 = db.clone(); - // let uuid = flow_id.clone(); - // tokio::spawn(async move { - // in_test_worker( - // &db2, - // // tokio::time::sleep(tokio::time::Duration::from_secs(60)), - // async move { - // completed.find(&uuid).await; - // }, - // port, - // ) - // .await; - // }) - // }; - - let db2 = db.clone(); - in_test_worker( - &db2, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre_app/leaf_right".to_owned(), - hash: 433403.into(), - language: windmill_common::scripts::ScriptLang::Python3, - dedicated_worker: None, - }) - // .arg("dbg_djob_sleep", serde_json::json!(10)) - .run_until_complete(&db, false, port) - .await; - - RunJob::from(windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre_app/leaf_left".to_owned(), - hash: 433400.into(), - language: windmill_common::scripts::ScriptLang::Python3, - dedicated_worker: None, - }) - // So set it to this long - .arg("dbg_djob_sleep", serde_json::json!(10)) - .run_until_complete(&db, false, port) - .await; - - completed.next().await; // leaf_right - completed.next().await; // leaf_left - completed.next().await; // importer - completed.next().await; // importer - }, - port, - ) - .await; - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 0 - ); - - let r = sqlx::query_scalar!("SELECT runnable_id FROM v2_job ORDER BY created_at DESC") - .fetch_all(&db) - .await - .unwrap(); - - assert_eq!(r.len(), 4); - assert!(r.contains(&Some(9))); - assert!(r.contains(&Some(433400))); - assert!(r.contains(&Some(433403))); - assert!(r.contains(&Some(0))); - - // handle.await.unwrap(); - - Ok(()) - } - /// 2. Same as second test, however first app djob will take longer than second debounce. - /// NOTE: This test should be ran in debug mode. In release it will not work properly. - #[cfg(all(feature = "python", feature = "private"))] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_3(db: sqlx::Pool) -> anyhow::Result<()> { - // This tests checks if concurrency limit works correcly and there is no race conditions. - let (_client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - // At this point we should have two - let mut job_ids = vec![]; - let push_job = |delay, db| async move { - let mut args = std::collections::HashMap::new(); - args.insert( - "dbg_djob_sleep".to_owned(), - // First one will create delay for 5 seconds - // The second will have no delay at all. - windmill_common::worker::to_raw_value(&delay), - ); - - args.insert( - "triggered_by_relative_import".to_string(), - windmill_common::worker::to_raw_value(&()), - ); - - let (job_uuid, new_tx) = windmill_queue::push( - &db, - windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - "test-workspace", - windmill_common::jobs::JobPayload::AppDependencies { - path: "f/dre_app/app".to_owned(), - // In newest versions we pass the current version to the djob - version: 0, - }, - windmill_queue::PushArgs { args: &args, extra: None }, - "admin", - "admin@windmill.dev", - "admin".to_owned(), - Some("trigger.dependents.to.recompute.dependencies"), - // Schedule for now. - Some(chrono::Utc::now()), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - None, - None, - None, - ) - .await - .unwrap(); - - new_tx.commit().await.unwrap(); - - job_uuid - }; - - // TODO: Verify concurrency key. - // Push first - job_ids.push(push_job(5, db.clone()).await); - - // Verify debounce_stale_data and debounce_key - { - let q = sqlx::query!("SELECT COUNT(*) FROM debounce_key") - .fetch_all(&db) - .await - .unwrap(); - - // Should be single entry - assert_eq!(q.len(), 1); - } - - // Start the first one in the background - let handle = { - let mut completed = listen_for_completed_jobs(&db).await; - let db2 = db.clone(); - tokio::spawn(async move { - in_test_worker( - &db2, - // sleep(Duration::from_secs(7)), - completed.next(), // Only wait for the single job. We are going to spawn another worker for second one. - port, - ) - .await; - }) - }; - - // Wait for the job to be created and started - // This way next job is not going to be consumed by the first one. - sleep(Duration::from_secs(2)).await; - - // Push second - job_ids.push(push_job(0, db.clone()).await); - - // Wait for the second one to finish in separate worker. - // in_test_worker(&db, completed.next(), port).await; - in_test_worker( - &db, - async { - // First job will be pulled - completed.next().await; - // However since we have concurrency limit enabled it will get rescheduled by creation of new djob. - // So we have to wait for that one as well. - completed.next().await; - }, - port, - ) - .await; - - // Wait for the first one - handle.await.unwrap(); - - // Verify that we have expected outcome - { - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job",) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 2 - ); - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_completed",) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 2 - ); - // Check that two jobs were executed sequentially - assert!(sqlx::query_scalar!( - " -SELECT - j1.completed_at < j2.started_at -FROM - v2_job_completed j1, - v2_job_completed j2 -WHERE - j1.id = $1 - AND j2.id = $2", - job_ids[0], - job_ids[1], - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap()); - } - Ok(()) - } - } - - // TODO: Test debounce reassignment works - - /// ## Testing for Scripts - mod scripts { - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs}; - use crate::dependency_job_debouncing::trigger_djob_for; - use std::time::Duration; - use tokio::time::sleep; - use tokio_stream::StreamExt; - - /// 1. LLF and RLF create two djobs for flow at the same and fall into single debounce - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - // TODO: Same test_but script fails. - async fn test_1(db: sqlx::Pool) -> anyhow::Result<()> { - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - // This tests if debouncing and consolidation works. - // Also makes sures that dependency job does not create new flow version - let (client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - // Verify lock is empty - { - assert_eq!( - sqlx::query_scalar!( - "SELECT lock FROM script WHERE path = 'f/dre_script/script'" - ) - .fetch_one(&db) - .await - .unwrap(), - Some("".into()) - ); - } - - // Trigger both at the same time. - { - trigger_djob_for( - &client, - "f/dre_script/leaf_left", - "0000000000082398", - Some("#requirements:\n#bottle==0.13.2\ndef main():\npass".into()), - ) - .await; - trigger_djob_for( - &client, - "f/dre_script/leaf_right", - "000000000008239B", - Some("#requirements:\n#tiny==0.1.3\ndef main():\npass".into()), - ) - .await; - } - - sleep(Duration::from_secs(1)).await; - - in_test_worker( - &db, - async { - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre_script/leaf_left" - ); - - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre_script/leaf_right" - ); - - // handle.await.unwrap(); - - // Let jobs propagate - - tokio::select!( - _ = async { - while sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue WHERE running = false") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - == 0 - { - sleep(Duration::from_secs(1)).await; - } - } => {}, - _ = sleep(Duration::from_secs(60)) => { panic!("Timeout") } - ); - // Verify there is only one queued job that is scheduled for atleast 3s ahead. - { - for r in - sqlx::query_scalar!("SELECT id FROM v2_job_queue WHERE running = false") - .fetch_all(&db) - .await - .unwrap() - { - dbg!( - sqlx::query!("SELECT runnable_path FROM v2_job WHERE id = $1", r) - .fetch_all(&db) - .await - .unwrap() - ); - } - for r in sqlx::query_scalar!("SELECT id FROM v2_job_completed") - .fetch_all(&db) - .await - .unwrap() - { - dbg!( - sqlx::query!("SELECT runnable_path FROM v2_job WHERE id = $1", r) - .fetch_all(&db) - .await - .unwrap() - ); - } - - dbg!(sqlx::query!("SELECT runnable_path FROM v2_job") - .fetch_all(&db) - .await - .unwrap()); - - let q = sqlx::query_scalar!( - "SELECT (scheduled_for - created_at) FROM v2_job_queue WHERE running = false" - ) - .fetch_all(&db) - .await - .unwrap(); - - assert_eq!(1, q.len()); - assert!(dbg!(q[0].unwrap().microseconds) > 1_000_000 /* 1 second */); - } - - // Verify debounce_stale_data and debounce_key - { - let q = sqlx::query_scalar!("SELECT key FROM debounce_key") - .fetch_all(&db) - .await - .unwrap(); - - assert_eq!(q.len(), 1); - - assert_eq!( - q[0].clone(), - "test-workspace:f/dre_script/script:dependency".to_owned(), - ); - - // Stale data is empty for scripts - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM debounce_stale_data") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 0 - ); - } - - // Wait until debounce delay is complete - // sleep(Duration::from_secs(6)).await; - - assert_eq!( - &sqlx::query_scalar!( - "SELECT runnable_path FROM v2_job WHERE id = $1", - completed.next().await.unwrap() - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - "f/dre_script/script" - ); - }, - port, - ) - .await; - - // completed.next().await.unwrap(); - - // Verify - { - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from v2_job_queue") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - assert_eq!( - vec![533404], - dbg!(sqlx::query_scalar!( - "SELECT hash FROM script WHERE path = 'f/dre_script/script' AND archived = true" - ) - .fetch_all(&db) - .await - .unwrap()) - ); - - assert_ne!( - 533404, - sqlx::query_scalar!( - "SELECT hash FROM script WHERE path = 'f/dre_script/script' AND archived = false" - ) - .fetch_one(&db) - .await - .unwrap() - ); - - assert_eq!( - vec![533404], - sqlx::query_scalar!( - "SELECT parent_hashes FROM script WHERE path = 'f/dre_script/script' AND archived = false" - ) - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - } - - // Verify cleanup - { - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_stale_data") - .fetch_one(&db) - .await - .unwrap() - .unwrap() - ); - } - - // handle.await.unwrap(); - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - async fn test_left(db: sqlx::Pool) -> anyhow::Result<()> { - use crate::common::RunJob; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - // TODO: We don't care about timer. If there is no timer, it will be set automatically for djobs?? - let (_client, port, _s) = init_client(db.clone()).await; - let mut completed = listen_for_completed_jobs(&db).await; - - // Trigger both at the same time. - { - let mut args = std::collections::HashMap::new(); - args.insert( - "dbg_djob_sleep".to_owned(), - // Execution should take this seconds - windmill_common::worker::to_raw_value(&20), - ); - - args.insert( - "triggered_by_relative_import".to_owned(), - // Execution should take this seconds - windmill_common::worker::to_raw_value(&()), - ); - - let (_flow_id, new_tx) = windmill_queue::push( - &db, - windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - "test-workspace", - windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre_script/script".to_owned(), - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - hash: 533404.into(), - }, - windmill_queue::PushArgs { args: &args, extra: None }, - "admin", - "admin@windmill.dev", - "admin".to_owned(), - Some("trigger.dependents.to.recompute.dependencies"), - // Debounce period - Some(chrono::Utc::now() + chrono::Duration::seconds(5)), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - None, - None, - None, - ) - .await - .unwrap(); - new_tx.commit().await.unwrap(); - - let db2 = db.clone(); - in_test_worker( - &db2, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre_script/leaf_right".to_owned(), - hash: 533403.into(), - language: windmill_common::scripts::ScriptLang::Python3, - dedicated_worker: None, - }) - .run_until_complete(&db, false, port) - .await; - - // This one is supposed to be started after flow djob has debounced and started but haven't finished yet. - RunJob::from(windmill_common::jobs::JobPayload::Dependencies { - path: "f/dre_script/leaf_left".to_owned(), - hash: 533400.into(), - language: windmill_common::scripts::ScriptLang::Python3, - dedicated_worker: None, - }) - // So set it to this long - .arg("dbg_djob_sleep", serde_json::json!(10)) - .run_until_complete(&db, false, port) - .await; - - completed.next().await; // leaf_right - completed.next().await; // leaf_left - completed.next().await; // importer - completed.next().await; // importer - }, - port, - ) - .await; - } - - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(&db) - .await - .unwrap() - .unwrap(), - 0 - ); - - let r = sqlx::query_scalar!("SELECT runnable_id FROM v2_job ORDER BY created_at DESC") - .fetch_all(&db) - .await - .unwrap(); - - dbg!(&r); - assert_eq!(r.len(), 4); - assert!(r.contains(&Some(-221349019907577876))); - assert!(r.contains(&Some(533400))); - assert!(r.contains(&Some(533403))); - assert!(r.contains(&Some(533404))); - - Ok(()) - } - - // // TODO: we don't need scripts to have concurrency limit - // /// 3. Same as second test, however first app djob will take longer than second debounce. - // /// NOTE: This test should be ran in debug mode. In release it will not work properly. - // #[cfg(all(feature = "python", feature = "private"))] - // #[sqlx::test(fixtures("base", "djob_debouncing"))] - // async fn test_3(db: sqlx::Pool) -> anyhow::Result<()> { - // // This tests checks if concurrency limit works correcly and there is no race conditions. - // let (client, port, _s) = init_client(db.clone()).await; - // let mut completed = listen_for_completed_jobs(&db).await; - - // // At this point we should have two - // let mut job_ids = vec![]; - // let push_job = |delay, db| async move { - // let mut args = std::collections::HashMap::new(); - // args.insert( - // "dbg_djob_sleep".to_owned(), - // // First one will create delay for 5 seconds - // // The second will have no delay at all. - // windmill_common::worker::to_raw_value(&delay), - // ); - - // args.insert( - // "triggered_by_relative_import".to_string(), - // windmill_common::worker::to_raw_value(&()), - // ); - - // let (job_uuid, new_tx) = windmill_queue::push( - // &db, - // windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()), - // "test-workspace", - // windmill_common::jobs::JobPayload::Dependencies { - // path: "f/dre_script/script".to_owned(), - // language: windmill_common::scripts::ScriptLang::Python3, - // dedicated_worker: None, - // hash: windmill_common::scripts::ScriptHash(533404), - // }, - // windmill_queue::PushArgs { args: &args, extra: None }, - // "admin", - // "admin@windmill.dev", - // "admin".to_owned(), - // Some("trigger.dependents.to.recompute.dependencies"), - // // Schedule for now. - // Some(chrono::Utc::now()), - // None, - // None, - // None, - // None, - // None, - // false, - // false, - // None, - // true, - // Some("dependency".into()), - // None, - // None, - // None, - // None, - // false, - // None, - // None, - // None, - // None, - // ) - // .await - // .unwrap(); - - // new_tx.commit().await.unwrap(); - - // job_uuid - // }; - - // // Push first - // job_ids.push(push_job(5, db.clone()).await); - // sleep(Duration::from_millis(300)).await; - - // // Verify debounce_stale_data and debounce_key - // { - // let q = sqlx::query_scalar!("SELECT key FROM debounce_key") - // .fetch_all(&db) - // .await - // .unwrap(); - - // assert_eq!(q.len(), 1); - - // assert_eq!( - // q[0].clone(), - // "test-workspace:f/dre_script/script:dependency".to_owned(), - // ); - - // // Stale data is empty for scripts - // assert_eq!( - // sqlx::query_scalar!("SELECT COUNT(*) FROM debounce_stale_data") - // .fetch_one(&db) - // .await - // .unwrap() - // .unwrap(), - // 0 - // ); - // } - - // // Start the first one in the background - // let handle = { - // let mut completed = listen_for_completed_jobs(&db).await; - // let db2 = db.clone(); - // tokio::spawn(async move { - // in_test_worker( - // &db2, - // // sleep(Duration::from_secs(7)), - // completed.next(), // Only wait for the single job. We are going to spawn another worker for second one. - // port, - // ) - // .await; - // }) - // }; - - // // Wait for the job to be created and started - // // This way next job is not going to be consumed by the first one. - // sleep(Duration::from_secs(1)).await; - - // // Push second - // job_ids.push(push_job(0, db.clone()).await); - - // // Wait for the second one to finish in separate worker. - // in_test_worker( - // &db, - // async { - // // First job will be pulled - // completed.next().await; - // // However since we have concurrency limit enabled it will get rescheduled by creation of new djob. - // // So we have to wait for that one as well. - // completed.next().await; - // }, - // port, - // ) - // .await; - - // // Wait for the first one - // handle.await.unwrap(); - - // // Verify that we have expected outcome - // { - // assert_eq!( - // sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue",) - // .fetch_one(&db) - // .await - // .unwrap() - // .unwrap(), - // 0 - // ); - // // Verify lock - // { - // assert_eq!( - // sqlx::query_scalar!( - // "SELECT lock FROM script WHERE path = 'f/dre_script/script'" - // ) - // .fetch_one(&db) - // .await - // .unwrap(), - // Some("# py: 3.11\nbottle==0.13.2\ntiny==0.1.3".into()) - // ); - // } - // assert_eq!( - // sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job",) - // .fetch_one(&db) - // .await - // .unwrap() - // .unwrap(), - // 2 - // ); - - // assert_eq!( - // sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_completed",) - // .fetch_one(&db) - // .await - // .unwrap() - // .unwrap(), - // 2 - // ); - // // Check that two jobs were executed sequentially - // assert!(sqlx::query_scalar!( - // " - // SELECT - // j1.completed_at < j2.started_at - // FROM - // v2_job_completed j1, - // v2_job_completed j2 - // WHERE - // j1.id = $1 - // AND j2.id = $2", - // job_ids[0], - // job_ids[1], - // ) - // .fetch_one(&db) - // .await - // .unwrap() - // .unwrap()); - // } - // Ok(()) - // } - } - // TODO: Test git sync -} -#[cfg(feature = "test_job_debouncing")] -mod normal_job_debouncing { - mod scripts { - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "job_debouncing"))] - async fn test_default_debounce_key(db: sqlx::Pool) -> anyhow::Result<()> { - use serde_json::json; - use windmill_common::scripts::ScriptHash; - - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let db = &db; - - in_test_worker( - db, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("33")) - // Start another worker, so we have two workers at the same time. - // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. - .run_until_complete_with(db, false, port, |id| async move { - - // Verify debounce_key - assert_eq!( - sqlx::query_scalar!( - "SELECT key FROM debounce_key WHERE job_id = $1", - id.clone() - ) - .fetch_one(db) - .await - .unwrap(), - "test-workspace/script/f/scripts/script_1#args:\"33\":\"ey\"".to_owned() - ); - - // Verify it is scheduled for future and not now. - { - assert!( - dbg!( - sqlx::query_scalar!( - "SELECT (scheduled_for - created_at) FROM v2_job_queue WHERE running = false" - ) - .fetch_one(db) - .await - .unwrap() - .unwrap() - .microseconds - ) > 1_000_000 /* 1 second */ - ); - } - - // Start another job. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("33")) - // But we only push it, one of the jobs should be debounced. - .push(db) - .await; - }) - .await; - }, - port, - ) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // And there is only supposed to be one job. - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 1 - ); - - // Verify debounce key clean up - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(db) - .await - .unwrap() - .unwrap() - ); - - Ok(()) - } - - #[cfg(all(feature = "python", feature = "agent_worker_server"))] - #[sqlx::test(fixtures("base", "job_debouncing"))] - async fn test_default_debounce_key_agent_wk( - db: sqlx::Pool, - ) -> anyhow::Result<()> { - use serde_json::json; - use windmill_common::scripts::ScriptHash; - - use crate::common::{init_client_agent_mode, RunJob}; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client_agent_mode(db.clone()).await; - let db = &db; - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("33")) - .run_until_complete(db, true, port) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // And there is only supposed to be one job. - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 1 - ); - - // And that job execute successfully - assert_eq!( - sqlx::query_scalar!("SELECT status::text FROM v2_job_completed") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - "success" - ); - - // Verify debounce key clean up - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(db) - .await - .unwrap() - .unwrap() - ); - - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "job_debouncing"))] - async fn test_custom_debounce_key(db: sqlx::Pool) -> anyhow::Result<()> { - use serde_json::json; - use windmill_common::scripts::ScriptHash; - - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let completed = listen_for_completed_jobs(&db).await; - let db = &db; - - in_test_worker( - db, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: Some("$workspace:my-custom-debounce-key:$args[x]".to_owned()), - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("1")) // 1 - // Start another worker, so we have two workers at the same time. - // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. - .run_until_complete_with(db, false, port, |id| async move { - - // Verify debounce_key - assert_eq!( - sqlx::query_scalar!( - "SELECT key FROM debounce_key WHERE job_id = $1", - id.clone() - ) - .fetch_one(db) - .await - .unwrap(), - "test-workspace:my-custom-debounce-key:ey".to_owned() - ); - - // Verify it is scheduled for future and not now. - { - assert!( - dbg!( - sqlx::query_scalar!( - "SELECT (scheduled_for - created_at) FROM v2_job_queue WHERE running = false" - ) - .fetch_one(db) - .await - .unwrap() - .unwrap() - .microseconds - ) > 1_000_000 /* 1 second */ - ); - } - - // Start another job. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - custom_debounce_key: Some("$workspace:my-custom-debounce-key:$args[x]".to_owned()), - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - // We will pass different argument. but it should still get debounced. - .arg("b", json!("2")) // 2 - // But we only push it, one of the jobs should be debounced. - .push(db) - .await; - }) - .await; - }, - port, - ) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // And there is only supposed to be one job. - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 1 - ); - - // And that job execute successfully - assert_eq!( - sqlx::query_scalar!("SELECT status::text FROM v2_job_completed") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - "success" - ); - - // Verify debounce key clean up - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(db) - .await - .unwrap() - .unwrap() - ); - - Ok(()) - } - - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "job_debouncing"))] - async fn test_no_debounce(db: sqlx::Pool) -> anyhow::Result<()> { - use serde_json::json; - use windmill_common::scripts::ScriptHash; - - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let completed = listen_for_completed_jobs(&db).await; - let db = &db; - - // different args - in_test_worker( - db, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("33")) - // Start another worker, so we have two workers at the same time. - // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. - .run_until_complete_with(db, false, port, |_id| async move { - // Start another job. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - // Different args. - .arg("x", json!("ey")) - .arg("b", json!("34")) // Different arg - .push(db) - .await; - }) - .await; - }, - port, - ) - .await; - - // no debounce delay on second - in_test_worker( - db, - async { - // This job should execute and then try to start another job that will get debounced. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - // Do not supply with custom debounce key. - // We will test if the debounce_key is created correctly. - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("33")) - // Start another worker, so we have two workers at the same time. - // We don't know which will execute the job, but we do know that if the job is executed, this worker will exit. - .run_until_complete_with(db, false, port, |_id| async move { - // Start another job. - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - custom_debounce_key: None, - debounce_delay_s: None, // Set to none to skip debouncing - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }) - .arg("x", json!("ey")) - .arg("b", json!("33")) - .push(db) - .await; - }) - .await; - }, - port, - ) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // And there is supposed to be four jobs and no debouncing. - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 4 - ); - - // Verify debounce key clean up - assert_eq!( - 0, - sqlx::query_scalar!("SELECT COUNT(*) from debounce_key") - .fetch_one(db) - .await - .unwrap() - .unwrap() - ); - - Ok(()) - } - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "djob_debouncing"))] - #[ignore = "modifies global env variable that is used by other tests"] - async fn test_min_version_does_not_support_debouncing( - db: sqlx::Pool, - ) -> anyhow::Result<()> { - use serde_json::json; - use windmill_common::scripts::ScriptHash; - - use crate::common::{in_test_worker, init_client, listen_for_completed_jobs, RunJob}; - - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = false; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let db = &db; - - crate::common::in_test_worker( - db, - async { - let job_template = - RunJob::from(windmill_common::jobs::JobPayload::ScriptHash { - hash: ScriptHash(533400), - path: "f/scripts/script_1".into(), - custom_debounce_key: None, - debounce_delay_s: None, // Set to none to skip debouncing - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - cache_ttl: None, - dedicated_worker: None, - language: windmill_common::scripts::ScriptLang::Python3, - priority: None, - apply_preprocessor: false, - }); - - // This will push to the top level worker - job_template.clone().push(db).await; - - // Will have space to run in parallel but in it's own worker - job_template.run_until_complete(db, false, port).await; - }, - port, - ) - .await; - - // Verify there is not jobs running - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job_queue") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 0 - ); - - // There are supposed to be two jobs, since debouncing is disabled. - assert_eq!( - sqlx::query_scalar!("SELECT COUNT(*) FROM v2_job") - .fetch_one(db) - .await - .unwrap() - .unwrap(), - 2 - ); - - Ok(()) - } - } - - mod flows { - #[cfg(feature = "python")] - #[sqlx::test(fixtures("base", "job_debouncing"))] - async fn test_different_kinds_top_level( - db: sqlx::Pool, - ) -> anyhow::Result<()> { - use crate::common::{init_client, listen_for_completed_jobs, RunJob}; - use serde_json::json; - { - let mut mvsd = windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING - .write() - .await; - *mvsd = true; - } - - let (_client, port, _s) = init_client(db.clone()).await; - let db = &db; - - // We want to run this for all tables related to flow be created. - RunJob::from(windmill_common::jobs::JobPayload::FlowDependencies { - path: "f/flows/flow".into(), - dedicated_worker: None, - version: 1443253234253454, - }) - .run_until_complete(db, false, port) - .await; - - dbg!(sqlx::query!("SELECT * FROM flow_node",) - .fetch_all(db) - .await - .unwrap()); - - let (j1, j2, j3) = tokio::join!( - RunJob::from(windmill_common::jobs::JobPayload::Flow { - version: 1443253234253454, - path: "f/flows/flow".into(), - dedicated_worker: None, - apply_preprocessor: false, - }) - .push(db), - RunJob::from(windmill_common::jobs::JobPayload::SingleStepFlow { - hash: None, - path: "f/flows/flow".into(), - custom_debounce_key: None, - debounce_delay_s: Some(2), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - flow_version: Some(1443253234253454), - args: std::collections::HashMap::new(), - retry: None, - error_handler_path: None, - error_handler_args: None, - skip_handler: None, - cache_ttl: None, - priority: None, - tag_override: None, - trigger_path: None, - apply_preprocessor: false, - }) - .push(db), - RunJob::from(windmill_common::jobs::JobPayload::RawFlow { - value: windmill_common::flows::FlowValue { - debounce_delay_s: Some(2), - modules: vec![windmill_common::flows::FlowModule { - id: "a".into(), - value: windmill_common::worker::to_raw_value(&json!({ - "lock": "# py: 3.11\n", - "type": "rawscript", - "assets": [], - "content": "def main(x: str, y: str):\n return x", - "language": "python3", - "debounce_delay_s": 15, - "input_transforms": { - "x": { - "type": "static", - "value": "" - }, - "y": { - "type": "static", - "value": "" - } - } - })), - ..Default::default() - }], - ..Default::default() - }, - path: Some("f/flows/flow".into()), - restarted_from: None, - }) - .push(db), - // RunJob::from(windmill_common::jobs::JobPayload::Code( - // windmill_common::jobs::RawCode { - // content: " - // def main(n: int): - // pass - // " - // .into(), - // path: Some("f/flows/flow".into()), - // hash: None, - // language: windmill_common::scripts::ScriptLang::Python3, - // custom_debounce_key: None, - // debounce_delay_s: Some(2), - // ..Default::default() - // }, - // )) - // .push(db) - ); - - assert_eq!(j1, j2); - assert_eq!(j1, j3); - // assert_eq!(j1, j4); - Ok(()) - } - } - - // TODO(ALL): - // - Check if all jobs were sucessfull. - // - // TODO: - // - [x] FlowNode (Script) - // - [x] FlowNode (Flow) - has no debouncing nor concurrency limits - // - [x] RawCode (Flow as code) - // - [x] RawFlow - // - [x] Flow - // - [x] FlowScript - // - // TODO(imperatively): - // - [x] Creation of flow - // - [x] Check entire flow - // - [x] Check it's inline scripts - // - [x] Creation of script - // - // TODO: [x] Agent workers. (and tests) - // TODO: [x] Backwards compat (and tests) - // TODO: [x] Concurrency limit is disabled if preprocessor is enabled. Investigate. - // TODO: [x] Last resort - monitor.rs to clean up debounce_keys - // TODO: [x] Catch debounce values by server if debouncing is disabled. (and tests) -} diff --git a/backend/tests/drafts.rs b/backend/tests/drafts.rs new file mode 100644 index 0000000000..e479d0b97c --- /dev/null +++ b/backend/tests/drafts.rs @@ -0,0 +1,106 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +#[sqlx::test(fixtures("base"))] +async fn test_draft_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/drafts"); + + // create a script first so the draft has a valid path + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/draft_script", + "summary": "Script for draft test", + "description": "", + "content": "export async function main() { return 1; }", + "language": "deno", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create script: {}", resp.text().await?); + + // --- create draft --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/draft_script", + "typ": "script", + "value": { + "content": "export async function main() { return 2; }", + "language": "deno" + } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create draft: {}", resp.text().await?); + + // verify draft exists via script get/draft endpoint + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/get/draft/u/test-user/draft_script" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert!(body["draft"].is_object(), "expected draft to be present"); + + // --- update draft (create with same path overwrites) --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/draft_script", + "typ": "script", + "value": { + "content": "export async function main() { return 3; }", + "language": "deno" + } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + // --- delete draft --- + let resp = authed(client().delete(format!( + "{base}/delete/script/u/test-user/draft_script" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // verify draft is gone + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/get/draft/u/test-user/draft_script" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert!(body["draft"].is_null(), "expected draft to be deleted"); + + Ok(()) +} diff --git a/backend/tests/fixtures/base.sql b/backend/tests/fixtures/base.sql index 5e96b75b99..7db9918fba 100644 --- a/backend/tests/fixtures/base.sql +++ b/backend/tests/fixtures/base.sql @@ -15,7 +15,27 @@ INSERT INTO workspace_key(workspace_id, kind, key) VALUES INSERT INTO workspace_settings (workspace_id) VALUES ('test-workspace'); +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('test-workspace', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('test@windmill.dev', 'not-a-real-hash', 'password', true, true, 'Test User', 'test-user'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('test2@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 2'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) + VALUES ('test3@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 3'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test2@windmill.dev', 'test-user-2', false, 'User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace', 'test3@windmill.dev', 'test-user-3', false, 'User'); + insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true); +insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN_2', 'test2@windmill.dev', 'test token 2', false); +insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN_3', 'test3@windmill.dev', 'test token 3', false); GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin; GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user; diff --git a/backend/tests/fixtures/job_debouncing.sql b/backend/tests/fixtures/job_debouncing.sql deleted file mode 100644 index 480d7c29a7..0000000000 --- a/backend/tests/fixtures/job_debouncing.sql +++ /dev/null @@ -1,171 +0,0 @@ --- SCRIPTS -- -INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( -'test-workspace', -'test-user', -'def main(x: str = "hey", b: int = 1): - pass -', -'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', -'', -'', -'f/scripts/script_1', 533400, 'python3', ''); --- Padded Hex: 0000000000082398 - -INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( -'test-workspace', -'test-user', -'def main(): - pass -', -'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', -'', -'', -'f/scripts/script_2', 533403, 'python3', ''); --- Padded Hex: 000000000008239B - -INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( -'test-workspace', -'test-user', -' -def main(): - pass -', -'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', -'', -'', -'f/scripts/script_3', 533404, 'python3', ''); - - --- Padded Hex: 000000000008239C -INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( -'test-workspace', -'', -'', -'f/flows/flow', -'{1443253234253454}', -'{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "properties": {}, - "required": [], - "type": "object" -}', -$tag$ -{ - "modules": [ - { - "id": "a", - "value": { - "type": "rawscript", - "assets": [], - "content": "def main(x: str, y: str):\n return x", - "language": "python3", - "debounce_delay_s": 2, - "input_transforms": { - "x": { - "type": "static", - "value": "" - }, - "y": { - "type": "static", - "value": "" - } - } - }, - "continue_on_error": false - } - ], - "debounce_delay_s": 2 -}$tag$, -'system' -); - -INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) -SELECT versions[1], workspace_id, path, schema, value, edited_by FROM flow WHERE path = 'f/flows/flow'; - --- No top level debouncing -INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( -'test-workspace', -'', -'', -'f/flows/flow_full', -'{123}', -'{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "properties": {}, - "required": [], - "type": "object" -}', -$tag$ -{ - "modules": [ - { - "id": "a", - "value": { - "lock": "# py: 3.11\n", - "type": "rawscript", - "assets": [], - "content": "import time\n\ndef main(x: str, y: str):\n time.sleep(30)\n\n return x", - "language": "python3", - "concurrent_limit": 1, - "input_transforms": { - "x": { - "type": "static", - "value": "" - }, - "y": { - "type": "static", - "value": "" - } - }, - "concurrency_time_window_s": 5 - }, - "continue_on_error": false - }, - { - "id": "b", - "value": { - "type": "whileloopflow", - "modules": [ - { - "id": "c", - "value": { - "lock": "# py: 3.11\n", - "type": "rawscript", - "assets": [], - "content": "# import wmill\n\n\ndef main(x: str):\n return x", - "language": "python3", - "input_transforms": { - "x": { - "type": "static", - "value": "" - } - } - } - }, - { - "id": "d", - "value": { - "type": "rawscript", - "assets": [], - "content": "# import wmill\n\n\ndef main(x: str):\n return x", - "language": "python3", - "input_transforms": { - "x": { - "type": "static", - "value": "" - } - } - } - } - ], - "skip_failures": false - } - } - ] -}$tag$, -'system' -); - - -INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_by) -SELECT versions[1], workspace_id, path, schema, value, edited_by FROM flow WHERE path = 'f/flows/flow_full'; diff --git a/backend/tests/fixtures/resources_test.sql b/backend/tests/fixtures/resources_test.sql new file mode 100644 index 0000000000..1d267256cd --- /dev/null +++ b/backend/tests/fixtures/resources_test.sql @@ -0,0 +1,69 @@ +-- Combined fixture for resource endpoint tests + +-- === resource type test data === + +INSERT INTO resource_type (workspace_id, name, schema, description, created_by) +VALUES ('test-workspace', 'test_db', '{"type": "object", "properties": {"host": {"type": "string"}}}', + 'Test DB type', 'test-user'); + +-- === get_value_interpolated test data === + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/simple_resource', '{"host": "localhost", "port": 5432}', + 'Simple resource', 'object', '{}', 'test-user'); + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'u/test-user/db_password', 'hunter2', false, 'DB password', '{}'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/resource_with_var', '{"host": "localhost", "password": "$var:u/test-user/db_password"}', + 'Resource with var ref', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/db_credentials', '{"user": "admin", "password": "secret123"}', + 'DB credentials', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/resource_with_res', '{"host": "localhost", "credentials": "$res:u/test-user/db_credentials"}', + 'Resource with res ref', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/resource_mixed', '{"host": "localhost", "password": "$var:u/test-user/db_password", "credentials": "$res:u/test-user/db_credentials"}', + 'Resource with mixed refs', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/null_resource', null, + 'Null resource', 'object', '{}', 'test-user'); + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'u/test-user/api_key', 'sk-abc123', false, 'API key', '{}'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/inner_resource', '{"key": "$var:u/test-user/api_key"}', + 'Inner resource', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/chained_resource', '{"service": "myapi", "auth": "$res:u/test-user/inner_resource"}', + 'Chained resource', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/resource_with_array', '{"hosts": ["host1", "host2"], "port": 5432}', + 'Resource with array', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/scalar_var_resource', '"$var:u/test-user/db_password"', + 'Scalar var ref', 'string', '{}', 'test-user'); + +-- === mcp_tools test data === + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/mcp_valid', '{"name": "test-mcp", "url": "http://127.0.0.1:19999/mcp"}', + 'Valid MCP resource (unreachable)', 'mcp_server', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/mcp_invalid_format', '{"host": "localhost", "port": 5432}', + 'Not an MCP resource', 'object', '{}', 'test-user'); + +INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by) +VALUES ('test-workspace', 'u/test-user/mcp_null', null, + 'Null MCP resource', 'mcp_server', '{}', 'test-user'); diff --git a/backend/tests/fixtures/schedule_push.sql b/backend/tests/fixtures/schedule_push.sql new file mode 100644 index 0000000000..9312d0bbd9 --- /dev/null +++ b/backend/tests/fixtures/schedule_push.sql @@ -0,0 +1,84 @@ +-- Fixture for schedule push tests +-- Sets up scripts, flows, users, and schedules needed to test push_scheduled_job + +-- Password entries for auth resolution +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name) +VALUES + ('test@windmill.dev', 'dummy_hash', 'password', false, true, 'Test User'), + ('obo@windmill.dev', 'dummy_hash', 'password', false, true, 'OBO User') +ON CONFLICT (email) DO NOTHING; + +-- OBO user in workspace +INSERT INTO usr (workspace_id, email, username, is_admin, role) +VALUES ('test-workspace', 'obo@windmill.dev', 'obo-user', false, 'Developer') +ON CONFLICT (workspace_id, username) DO NOTHING; + +-- A simple script +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind) +VALUES ( + 'test-workspace', 'test-user', + 'export async function main() { return "ok"; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'Test script', '', 'f/system/test_script', 100001, 'deno', '', 'script' +); + +-- A script with on_behalf_of_email +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, on_behalf_of_email) +VALUES ( + 'test-workspace', 'test-user', + 'export async function main() { return "obo"; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'OBO script', '', 'f/system/obo_script', 100002, 'deno', '', 'script', 'obo@windmill.dev' +); + +-- A script with a tag +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, tag) +VALUES ( + 'test-workspace', 'test-user', + 'export async function main() { return "tagged"; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'Tagged script', '', 'f/system/tagged_script', 100003, 'deno', '', 'script', 'custom-tag' +); + +-- A script with timeout +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, timeout) +VALUES ( + 'test-workspace', 'test-user', + 'export async function main() { return "timeout"; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'Timeout script', '', 'f/system/timeout_script', 100004, 'deno', '', 'script', 300 +); + +-- A flow +INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by) +VALUES ( + 'test-workspace', 'Test flow', '', 'f/system/test_flow', '{200001}', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"path": "f/system/test_script", "type": "script", "input_transforms": {}}}]}', + 'test-user' +); + +INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) +VALUES ( + 200001, 'test-workspace', 'f/system/test_flow', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"path": "f/system/test_script", "type": "script", "input_transforms": {}}}]}', + 'test-user' +); + +-- A flow with on_behalf_of_email +INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by, on_behalf_of_email) +VALUES ( + 'test-workspace', 'OBO flow', '', 'f/system/obo_flow', '{200002}', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"path": "f/system/test_script", "type": "script", "input_transforms": {}}}]}', + 'test-user', 'obo@windmill.dev' +); + +INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) +VALUES ( + 200002, 'test-workspace', 'f/system/obo_flow', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"path": "f/system/test_script", "type": "script", "input_transforms": {}}}]}', + 'test-user' +); diff --git a/backend/tests/fixtures/secret_backend.sql b/backend/tests/fixtures/secret_backend.sql index 0bc5a6f52a..1ebbc13e9b 100644 --- a/backend/tests/fixtures/secret_backend.sql +++ b/backend/tests/fixtures/secret_backend.sql @@ -14,18 +14,18 @@ INSERT INTO workspace_key(workspace_id, kind, key) VALUES ('test-workspace-2', 'cloud', 'test-key-2') ON CONFLICT DO NOTHING; --- Insert test secrets for workspace 1 --- Note: The 'value' column stores encrypted values in production, --- but for tests we'll use plain text that the migration will handle +-- Insert test variables with placeholder values. +-- Secret values are encrypted by the test setup using build_crypt + encrypt +-- with the workspace key, matching production behavior. INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) VALUES - ('test-workspace', 'u/test-user/db_password', 'encrypted-db-pass-123', true, 'Database password', '{}'), - ('test-workspace', 'u/test-user/api_key', 'encrypted-api-key-abc', true, 'API key for external service', '{}'), + ('test-workspace', 'u/test-user/db_password', 'PLACEHOLDER', true, 'Database password', '{}'), + ('test-workspace', 'u/test-user/api_key', 'PLACEHOLDER', true, 'API key for external service', '{}'), ('test-workspace', 'u/test-user/public_var', 'not-a-secret', false, 'A non-secret variable', '{}') ON CONFLICT DO NOTHING; -- Insert test secrets for workspace 2 (to test isolation) INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) VALUES - ('test-workspace-2', 'u/test-user/other_secret', 'encrypted-other-secret', true, 'Secret in workspace 2', '{}') + ('test-workspace-2', 'u/test-user/other_secret', 'PLACEHOLDER', true, 'Secret in workspace 2', '{}') ON CONFLICT DO NOTHING; diff --git a/backend/tests/fixtures/start_all_triggers.sh b/backend/tests/fixtures/start_all_triggers.sh new file mode 100755 index 0000000000..a7e5b911ca --- /dev/null +++ b/backend/tests/fixtures/start_all_triggers.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Starts (or stops) all external services needed for the trigger e2e tests. +# +# Usage: +# ./tests/fixtures/start_all_triggers.sh # start everything +# ./tests/fixtures/start_all_triggers.sh stop # stop everything +# ./tests/fixtures/start_all_triggers.sh oss # start only OSS services +set -euo pipefail + +DIR="$(cd "$(dirname "$0")" && pwd)" +ACTION="${1:-start}" + +SCRIPTS_OSS=( + "$DIR/start_mqtt.sh" + "$DIR/start_websocket.sh" + "$DIR/start_postgres_replication.sh" +) + +SCRIPTS_EE=( + "$DIR/start_kafka.sh" + "$DIR/start_nats.sh" + "$DIR/start_sqs.sh" + "$DIR/start_gcp_pubsub.sh" +) + +if [[ "$ACTION" == "stop" ]]; then + for s in "${SCRIPTS_OSS[@]}" "${SCRIPTS_EE[@]}"; do + echo "--- $(basename "$s" .sh) stop ---" + bash "$s" stop + done + exit 0 +fi + +if [[ "$ACTION" == "oss" ]]; then + SCRIPTS=("${SCRIPTS_OSS[@]}") +else + SCRIPTS=("${SCRIPTS_OSS[@]}" "${SCRIPTS_EE[@]}") +fi + +for s in "${SCRIPTS[@]}"; do + echo "--- $(basename "$s" .sh) ---" + bash "$s" + echo "" +done + +echo "============================================" +echo "All services ready. Run the e2e tests with:" +echo "" + +if [[ "$ACTION" == "oss" ]]; then + echo " cargo test --test trigger_e2e --features mqtt_trigger,websocket,postgres_trigger -- --ignored --nocapture" +else + echo " # OSS triggers" + echo " cargo test --test trigger_e2e --features mqtt_trigger,websocket,postgres_trigger -- --ignored --nocapture" + echo "" + echo " # Enterprise triggers" + echo " AWS_ENDPOINT_URL=http://localhost:4566 PUBSUB_EMULATOR_HOST=localhost:8085 cargo test --test trigger_e2e --features kafka,nats,sqs_trigger,gcp_trigger,enterprise,private -- --ignored --nocapture" +fi diff --git a/backend/tests/fixtures/start_gcp_pubsub.sh b/backend/tests/fixtures/start_gcp_pubsub.sh new file mode 100755 index 0000000000..88ef706982 --- /dev/null +++ b/backend/tests/fixtures/start_gcp_pubsub.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Starts the GCP Pub/Sub emulator for trigger_e2e::test_gcp_e2e (Enterprise) +# +# Usage: +# ./tests/fixtures/start_gcp_pubsub.sh # start +# ./tests/fixtures/start_gcp_pubsub.sh stop # stop & remove +set -euo pipefail + +NAME="windmill-test-pubsub" +PORT=8085 + +if [[ "${1:-}" == "stop" ]]; then + docker rm -f "$NAME" 2>/dev/null && echo "stopped $NAME" || echo "$NAME not running" + exit 0 +fi + +if docker ps --format '{{.Names}}' | grep -q "^${NAME}$"; then + echo "$NAME is already running" + exit 0 +fi + +docker rm -f "$NAME" 2>/dev/null || true + +docker run -d --name "$NAME" -p "${PORT}:8085" \ + gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators \ + gcloud beta emulators pubsub start --host-port="0.0.0.0:${PORT}" + +echo "Waiting for Pub/Sub emulator to become ready..." +for i in $(seq 1 30); do + if curl -sf "http://localhost:${PORT}" &>/dev/null; then + break + fi + sleep 1 +done + +# Create the test topic and subscription +curl -sX PUT "http://localhost:${PORT}/v1/projects/local-project/topics/windmill-e2e-test" >/dev/null +curl -sX PUT "http://localhost:${PORT}/v1/projects/local-project/subscriptions/windmill-e2e-sub" \ + -H "Content-Type: application/json" \ + -d '{"topic": "projects/local-project/topics/windmill-e2e-test"}' >/dev/null + +echo "GCP Pub/Sub emulator listening on localhost:${PORT}" +echo " topic: windmill-e2e-test" +echo " subscription: windmill-e2e-sub" +echo "" +echo "Run the test:" +echo " PUBSUB_EMULATOR_HOST=localhost:${PORT} cargo test --test trigger_e2e test_gcp_e2e --features gcp_trigger,enterprise,private -- --ignored --nocapture" diff --git a/backend/tests/fixtures/start_kafka.sh b/backend/tests/fixtures/start_kafka.sh new file mode 100755 index 0000000000..567c6e1893 --- /dev/null +++ b/backend/tests/fixtures/start_kafka.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Starts a Kafka broker for trigger_e2e::test_kafka_e2e (Enterprise) +# +# Usage: +# ./tests/fixtures/start_kafka.sh # start +# ./tests/fixtures/start_kafka.sh stop # stop & remove +set -euo pipefail + +NAME="windmill-test-kafka" +PORT=9092 + +if [[ "${1:-}" == "stop" ]]; then + docker rm -f "$NAME" 2>/dev/null && echo "stopped $NAME" || echo "$NAME not running" + exit 0 +fi + +if docker ps --format '{{.Names}}' | grep -q "^${NAME}$"; then + echo "$NAME is already running" + exit 0 +fi + +docker rm -f "$NAME" 2>/dev/null || true + +docker run -d --name "$NAME" -p "${PORT}:9092" \ + -e KAFKA_NODE_ID=1 \ + -e KAFKA_PROCESS_ROLES=broker,controller \ + -e KAFKA_LISTENERS="PLAINTEXT://0.0.0.0:${PORT},CONTROLLER://0.0.0.0:9093" \ + -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \ + -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \ + -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \ + -e KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://localhost:${PORT}" \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + -e KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \ + apache/kafka:latest + +echo "Waiting for Kafka to become ready..." +for i in $(seq 1 60); do + if docker exec "$NAME" /opt/kafka/bin/kafka-topics.sh --list --bootstrap-server "localhost:${PORT}" &>/dev/null; then + break + fi + sleep 1 +done + +docker exec "$NAME" /opt/kafka/bin/kafka-topics.sh --create \ + --topic windmill-e2e-test \ + --bootstrap-server "localhost:${PORT}" \ + --partitions 1 --replication-factor 1 2>/dev/null || true + +echo "Kafka broker listening on localhost:${PORT} with topic 'windmill-e2e-test'" +echo "" +echo "Run the test:" +echo " cargo test --test trigger_e2e test_kafka_e2e --features kafka,enterprise,private -- --ignored --nocapture" diff --git a/backend/tests/fixtures/start_mqtt.sh b/backend/tests/fixtures/start_mqtt.sh new file mode 100755 index 0000000000..e48e5043cf --- /dev/null +++ b/backend/tests/fixtures/start_mqtt.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Starts a Mosquitto MQTT broker for trigger_e2e::test_mqtt_e2e +# +# Usage: +# ./tests/fixtures/start_mqtt.sh # start +# ./tests/fixtures/start_mqtt.sh stop # stop & remove +set -euo pipefail + +NAME="windmill-test-mqtt" +PORT=1883 + +if [[ "${1:-}" == "stop" ]]; then + docker rm -f "$NAME" 2>/dev/null && echo "stopped $NAME" || echo "$NAME not running" + exit 0 +fi + +if docker ps --format '{{.Names}}' | grep -q "^${NAME}$"; then + echo "$NAME is already running" + exit 0 +fi + +docker rm -f "$NAME" 2>/dev/null || true + +docker run -d --name "$NAME" -p "${PORT}:1883" \ + eclipse-mosquitto:latest \ + mosquitto -c /mosquitto-no-auth.conf + +echo "MQTT broker listening on localhost:${PORT}" +echo "" +echo "Run the test:" +echo " cargo test --test trigger_e2e test_mqtt_e2e --features mqtt_trigger -- --ignored --nocapture" diff --git a/backend/tests/fixtures/start_nats.sh b/backend/tests/fixtures/start_nats.sh new file mode 100755 index 0000000000..fe6cf7be68 --- /dev/null +++ b/backend/tests/fixtures/start_nats.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Starts a NATS server for trigger_e2e::test_nats_e2e (Enterprise) +# +# Usage: +# ./tests/fixtures/start_nats.sh # start +# ./tests/fixtures/start_nats.sh stop # stop & remove +set -euo pipefail + +NAME="windmill-test-nats" +PORT=4222 + +if [[ "${1:-}" == "stop" ]]; then + docker rm -f "$NAME" 2>/dev/null && echo "stopped $NAME" || echo "$NAME not running" + exit 0 +fi + +if docker ps --format '{{.Names}}' | grep -q "^${NAME}$"; then + echo "$NAME is already running" + exit 0 +fi + +docker rm -f "$NAME" 2>/dev/null || true + +docker run -d --name "$NAME" -p "${PORT}:4222" nats:latest + +echo "NATS server listening on localhost:${PORT}" +echo "" +echo "Run the test:" +echo " cargo test --test trigger_e2e test_nats_e2e --features nats,enterprise,private -- --ignored --nocapture" diff --git a/backend/tests/fixtures/start_postgres_replication.sh b/backend/tests/fixtures/start_postgres_replication.sh new file mode 100755 index 0000000000..5bde571e23 --- /dev/null +++ b/backend/tests/fixtures/start_postgres_replication.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Configures the local PostgreSQL for logical replication trigger tests. +# +# Prerequisites: wal_level=logical must be set (requires PG restart). +# Check with: SHOW wal_level; +# +# Usage: +# ./tests/fixtures/start_postgres_replication.sh # setup +# ./tests/fixtures/start_postgres_replication.sh stop # teardown +set -euo pipefail + +PGURL="${DATABASE_URL:-postgres://postgres:changeme@localhost:5432/windmill}" + +if [[ "${1:-}" == "stop" ]]; then + psql "$PGURL" <<'SQL' +SELECT pg_drop_replication_slot('test_e2e_slot') + WHERE EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_e2e_slot'); +DROP PUBLICATION IF EXISTS test_e2e_pub; +DROP TABLE IF EXISTS test_trigger_table; +SQL + echo "Postgres replication teardown complete" + exit 0 +fi + +# Check wal_level +WAL_LEVEL=$(psql "$PGURL" -tAc "SHOW wal_level;") +if [[ "$WAL_LEVEL" != "logical" ]]; then + echo "ERROR: wal_level is '$WAL_LEVEL', must be 'logical'" + echo "" + echo "Fix with:" + echo " psql \"$PGURL\" -c \"ALTER SYSTEM SET wal_level = logical;\"" + echo " # then restart PostgreSQL" + exit 1 +fi + +psql "$PGURL" <<'SQL' +CREATE TABLE IF NOT EXISTS test_trigger_table (id serial PRIMARY KEY, data text); + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'test_e2e_pub') THEN + CREATE PUBLICATION test_e2e_pub FOR TABLE test_trigger_table; + END IF; +END $$; + +SELECT pg_create_logical_replication_slot('test_e2e_slot', 'pgoutput') + WHERE NOT EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = 'test_e2e_slot'); +SQL + +echo "Postgres logical replication ready (publication=test_e2e_pub, slot=test_e2e_slot)" +echo "" +echo "Run the test:" +echo " cargo test --test trigger_e2e test_postgres_e2e --features postgres_trigger -- --ignored --nocapture" diff --git a/backend/tests/fixtures/start_sqs.sh b/backend/tests/fixtures/start_sqs.sh new file mode 100755 index 0000000000..79563505e2 --- /dev/null +++ b/backend/tests/fixtures/start_sqs.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Starts LocalStack for SQS trigger_e2e::test_sqs_e2e (Enterprise) +# +# Usage: +# ./tests/fixtures/start_sqs.sh # start +# ./tests/fixtures/start_sqs.sh stop # stop & remove +set -euo pipefail + +NAME="windmill-test-localstack" +PORT=4566 + +if [[ "${1:-}" == "stop" ]]; then + docker rm -f "$NAME" 2>/dev/null && echo "stopped $NAME" || echo "$NAME not running" + exit 0 +fi + +if docker ps --format '{{.Names}}' | grep -q "^${NAME}$"; then + echo "$NAME is already running" + exit 0 +fi + +docker rm -f "$NAME" 2>/dev/null || true + +docker run -d --name "$NAME" -p "${PORT}:4566" \ + -e SERVICES=sqs \ + localstack/localstack + +echo "Waiting for LocalStack to become ready..." +for i in $(seq 1 30); do + if curl -sf "http://localhost:${PORT}/_localstack/health" &>/dev/null; then + break + fi + sleep 1 +done + +# Create the test queue +aws --endpoint-url="http://localhost:${PORT}" \ + --region us-east-1 \ + --no-sign-request \ + sqs create-queue --queue-name windmill-e2e-test 2>/dev/null || true + +echo "LocalStack SQS listening on localhost:${PORT} with queue 'windmill-e2e-test'" +echo "" +echo "Run the test:" +echo " AWS_ENDPOINT_URL=http://localhost:${PORT} cargo test --test trigger_e2e test_sqs_e2e --features sqs_trigger,enterprise,private -- --ignored --nocapture" diff --git a/backend/tests/fixtures/start_websocket.sh b/backend/tests/fixtures/start_websocket.sh new file mode 100755 index 0000000000..9bab96debe --- /dev/null +++ b/backend/tests/fixtures/start_websocket.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Starts a WebSocket echo server for trigger_e2e::test_websocket_e2e +# +# Usage: +# ./tests/fixtures/start_websocket.sh # start +# ./tests/fixtures/start_websocket.sh stop # stop & remove +set -euo pipefail + +NAME="windmill-test-ws-echo" +PORT=8765 + +if [[ "${1:-}" == "stop" ]]; then + docker rm -f "$NAME" 2>/dev/null && echo "stopped $NAME" || echo "$NAME not running" + exit 0 +fi + +if docker ps --format '{{.Names}}' | grep -q "^${NAME}$"; then + echo "$NAME is already running" + exit 0 +fi + +docker rm -f "$NAME" 2>/dev/null || true + +docker run -d --name "$NAME" -p "${PORT}:8080" \ + -e PORT=8080 \ + jmalloc/echo-server + +echo "WebSocket echo server listening on localhost:${PORT}" +echo "" +echo "Run the test:" +echo " cargo test --test trigger_e2e test_websocket_e2e --features websocket -- --ignored --nocapture" diff --git a/backend/tests/fixtures/variables_test.sql b/backend/tests/fixtures/variables_test.sql new file mode 100644 index 0000000000..0d3e99e313 --- /dev/null +++ b/backend/tests/fixtures/variables_test.sql @@ -0,0 +1,10 @@ +-- Fixture for variable endpoint tests + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'u/test-user/plain_var', 'hello world', false, 'A plain variable', '{}'); + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'u/test-user/secret_var', 'supersecret', true, 'A secret variable', '{}'); + +INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) +VALUES ('test-workspace', 'u/test-user/another_var', 'foobar', false, 'Another variable', '{}'); diff --git a/backend/tests/flows.rs b/backend/tests/flows.rs new file mode 100644 index 0000000000..848231f2fc --- /dev/null +++ b/backend/tests/flows.rs @@ -0,0 +1,289 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn flow_url(port: u16, endpoint: &str, path: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/flows/{endpoint}/{path}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn authed_get(port: u16, endpoint: &str, path: &str) -> reqwest::Response { + authed(client().get(flow_url(port, endpoint, path))) + .send() + .await + .unwrap() +} + +fn new_flow(path: &str, summary: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": summary, + "description": "", + "value": { + "modules": [] + }, + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + }) +} + +#[sqlx::test(fixtures("base"))] +async fn test_flow_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/flows"); + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_flow("u/test-user/test_flow", "Test flow")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + // create second flow + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_flow("u/test-user/another_flow", "Another flow")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create another: {}", resp.text().await?); + + // --- exists --- + let resp = authed_get(port, "exists", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "exists", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get by path --- + let resp = authed_get(port, "get", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_flow"); + assert_eq!(body["summary"], "Test flow"); + + // get not found + let resp = authed_get(port, "get", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- get draft --- + let resp = authed_get(port, "get/draft", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_flow"); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!( + list.len() >= 2, + "expected at least 2 flows, got {}", + list.len() + ); + assert!(list.iter().any(|f| f["path"] == "u/test-user/test_flow")); + + // --- list_search --- + let resp = authed(client().get(format!("{base}/list_search"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + + // --- list_paths --- + let resp = authed(client().get(format!("{base}/list_paths"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let paths = resp.json::>().await?; + assert!(paths.contains(&"u/test-user/test_flow".to_string())); + + // --- history --- + let resp = authed_get(port, "history/p", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + let history = resp.json::>().await?; + assert!(!history.is_empty()); + + // --- get_latest_version --- + let resp = authed_get(port, "get_latest_version", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + + // --- deployment_status --- + let resp = authed_get(port, "deployment_status/p", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + + // --- list_tokens --- + let resp = authed_get(port, "list_tokens", "u/test-user/test_flow").await; + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- list_paths_from_workspace_runnable --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/flows/list_paths_from_workspace_runnable/flow/u/test-user/test_flow" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- get by version --- + let version = &history[0]["id"]; + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/flows/get/v/{version}" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_flow"); + + // --- update --- + let resp = authed(client().post(flow_url(port, "update", "u/test-user/test_flow"))) + .json(&new_flow("u/test-user/test_flow", "Updated flow")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + + // verify update + let resp = authed_get(port, "get", "u/test-user/test_flow").await; + let body = resp.json::().await?; + assert_eq!(body["summary"], "Updated flow"); + + // history should have 2 entries + let resp = authed_get(port, "history/p", "u/test-user/test_flow").await; + let history = resp.json::>().await?; + assert!( + history.len() >= 2, + "expected at least 2 history entries, got {}", + history.len() + ); + let latest_version = &history[0]["id"]; + + // --- get by version + path --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/flows/get/v/{latest_version}/p/u/test-user/test_flow" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_flow"); + + // --- history_update --- + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/flows/history_update/v/{latest_version}" + ))) + .json(&json!({"deployment_msg": "deployed v2"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "history_update: {}", resp.text().await?); + + // --- get_triggers_count --- + let resp = authed(client().get(flow_url( + port, + "get_triggers_count", + "u/test-user/test_flow", + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- toggle_workspace_error_handler (EE-gated, expect 400 in OSS) --- + let resp = authed(client().post(flow_url( + port, + "toggle_workspace_error_handler", + "u/test-user/test_flow", + ))) + .json(&json!({})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "toggle_workspace_error_handler: unexpected status {}", + resp.status() + ); + + // --- archive --- + let resp = authed(client().post(flow_url(port, "archive", "u/test-user/another_flow"))) + .json(&json!({})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // archived flow should still be gettable + let resp = authed_get(port, "get", "u/test-user/another_flow").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["archived"], true); + + // --- delete --- + let resp = authed(client().delete(flow_url(port, "delete", "u/test-user/another_flow"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "exists", "u/test-user/another_flow").await; + assert_eq!(resp.json::().await?, false); + + // ===== Hub endpoints (require external network, expect 500 or 200) ===== + + // --- hub/list --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/flows/hub/list" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/list: unexpected status {}", + resp.status() + ); + + // --- hub/get --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/flows/hub/get/1" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/get: unexpected status {}", + resp.status() + ); + + Ok(()) +} diff --git a/backend/tests/folders.rs b/backend/tests/folders.rs new file mode 100644 index 0000000000..075eea69eb --- /dev/null +++ b/backend/tests/folders.rs @@ -0,0 +1,167 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn folder_url(port: u16, endpoint: &str, name: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/folders/{endpoint}/{name}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +#[sqlx::test(fixtures("base"))] +async fn test_folder_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/folders"); + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "name": "test_folder", + "summary": "A test folder", + "display_name": "Test Folder" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + + // create second folder + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "name": "another_folder", + "summary": "Another folder" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create another: {}", resp.text().await?); + + // --- exists --- + let resp = authed(client().get(folder_url(port, "exists", "test_folder"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed(client().get(folder_url(port, "exists", "nonexistent"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get --- + let resp = authed(client().get(folder_url(port, "get", "test_folder"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["name"], "test_folder"); + assert_eq!(body["summary"], "A test folder"); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().any(|f| f["name"] == "test_folder")); + + // --- listnames --- + let resp = authed(client().get(format!("{base}/listnames"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let names = resp.json::>().await?; + assert!(names.contains(&"test_folder".to_string())); + + // --- update --- + let resp = authed(client().post(folder_url(port, "update", "test_folder"))) + .json(&json!({"summary": "Updated summary"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed(client().get(folder_url(port, "get", "test_folder"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + assert_eq!(body["summary"], "Updated summary"); + + // --- addowner --- + let resp = authed(client().post(folder_url(port, "addowner", "test_folder"))) + .json(&json!({"owner": "u/test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "addowner: {}", resp.text().await?); + + // verify ownership + let resp = authed(client().get(folder_url(port, "get", "test_folder"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + let owners = body["owners"].as_array().unwrap(); + assert!( + owners.iter().any(|o| o.as_str() == Some("u/test-user")), + "expected u/test-user in owners, got: {:?}", + owners + ); + + // --- removeowner --- + let resp = authed(client().post(folder_url(port, "removeowner", "test_folder"))) + .json(&json!({"owner": "u/test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- getusage --- + let resp = authed(client().get(folder_url(port, "getusage", "test_folder"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- is_owner --- + let resp = authed(client().get(format!( + "{base}/is_owner/f/test_folder" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await?; + + // --- delete --- + let resp = authed(client().delete(folder_url(port, "delete", "another_folder"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed(client().get(folder_url(port, "exists", "another_folder"))) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, false); + + Ok(()) +} diff --git a/backend/tests/groups.rs b/backend/tests/groups.rs new file mode 100644 index 0000000000..453a15916e --- /dev/null +++ b/backend/tests/groups.rs @@ -0,0 +1,300 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn group_url(port: u16, endpoint: &str, name: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/groups/{endpoint}/{name}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +#[sqlx::test(fixtures("base"))] +async fn test_group_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/groups"); + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "name": "test_group", + "summary": "A test group" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + + // create second group + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "name": "another_group", + "summary": "Another group" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create another: {}", resp.text().await?); + + // create duplicate -> error + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "name": "test_group", + "summary": "Duplicate" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + // --- get --- + let resp = authed(client().get(group_url(port, "get", "test_group"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["name"], "test_group"); + assert_eq!(body["summary"], "A test group"); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().any(|g| g["name"] == "test_group")); + + // --- listnames --- + let resp = authed(client().get(format!("{base}/listnames"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let names = resp.json::>().await?; + assert!(names.contains(&"test_group".to_string())); + + // --- update --- + let resp = authed(client().post(group_url(port, "update", "test_group"))) + .json(&json!({"summary": "Updated summary"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed(client().get(group_url(port, "get", "test_group"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + assert_eq!(body["summary"], "Updated summary"); + + // --- adduser --- + let resp = authed(client().post(group_url(port, "adduser", "test_group"))) + .json(&json!({"username": "test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "adduser: {}", resp.text().await?); + + // verify membership + let resp = authed(client().get(group_url(port, "get", "test_group"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + let members = body["members"].as_array().unwrap(); + assert!( + members.iter().any(|m| m.as_str() == Some("test-user")), + "expected test-user in members, got: {:?}", + members + ); + + // --- removeuser --- + let resp = authed(client().post(group_url(port, "removeuser", "test_group"))) + .json(&json!({"username": "test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- is_owner --- + let resp = authed(client().get(group_url(port, "is_owner", "test_group"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + // --- delete --- + let resp = authed(client().delete(group_url(port, "delete", "another_group"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // verify deleted - get should 404 or the group shouldn't appear in list + let resp = authed(client().get(format!("{base}/listnames"))) + .send() + .await + .unwrap(); + let names = resp.json::>().await?; + assert!(!names.contains(&"another_group".to_string())); + + // ===== Global (instance group) endpoints ===== + let global_base = format!("http://localhost:{port}/api/groups"); + + // --- create instance group --- + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({"name": "test_igroup", "summary": "Test instance group"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "create igroup: {}", + resp.text().await? + ); + + // --- list instance groups --- + let resp = authed(client().get(format!("{global_base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().any(|g| g["name"] == "test_igroup")); + + // --- list_with_workspaces --- + let resp = authed(client().get(format!("{global_base}/list_with_workspaces"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- get instance group --- + let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["name"], "test_igroup"); + assert_eq!(body["summary"], "Test instance group"); + + // --- update instance group --- + let resp = authed(client().post(format!("{global_base}/update/test_igroup"))) + .json(&json!({"new_summary": "Updated instance group"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "update igroup: {}", + resp.text().await? + ); + + // verify update + let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + assert_eq!(body["summary"], "Updated instance group"); + + // --- adduser to instance group --- + let resp = authed(client().post(format!("{global_base}/adduser/test_igroup"))) + .json(&json!({"email": "test@windmill.dev"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "adduser igroup: {}", + resp.text().await? + ); + + // verify membership + let resp = authed(client().get(format!("{global_base}/get/test_igroup"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + let emails = body["emails"].as_array().unwrap(); + assert!( + emails + .iter() + .any(|e| e.as_str() == Some("test@windmill.dev")), + "expected test@windmill.dev in emails, got: {:?}", + emails + ); + + // --- removeuser from instance group --- + let resp = authed(client().post(format!( + "{global_base}/removeuser/test_igroup" + ))) + .json(&json!({"email": "test@windmill.dev"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- export (EE-gated) --- + let resp = authed(client().get(format!("{global_base}/export"))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "export igroups: unexpected status {}", + resp.status() + ); + + // --- overwrite (EE-gated) --- + let resp = authed(client().post(format!("{global_base}/overwrite"))) + .json(&json!([])) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "overwrite igroups: unexpected status {}", + resp.status() + ); + + // --- delete instance group --- + let resp = authed(client().delete(format!("{global_base}/delete/test_igroup"))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "delete igroup: {}", + resp.text().await? + ); + + // verify deleted + let resp = authed(client().get(format!("{global_base}/list"))) + .send() + .await + .unwrap(); + let list = resp.json::>().await?; + assert!(!list.iter().any(|g| g["name"] == "test_igroup")); + + Ok(()) +} diff --git a/backend/tests/job_cleanup_test.rs b/backend/tests/job_cleanup_test.rs new file mode 100644 index 0000000000..f72fda4cb8 --- /dev/null +++ b/backend/tests/job_cleanup_test.rs @@ -0,0 +1,299 @@ +/// Integration test for batched job cleanup +/// +/// Prerequisites: +/// - PostgreSQL running with windmill database +/// - DATABASE_URL env var or default: postgres://postgres:changeme@localhost:5432/windmill +/// +/// Run all tests: +/// cargo test --test job_cleanup_test -- --nocapture +/// +/// Run specific test: +/// cargo test --test job_cleanup_test test_batched_job_cleanup -- --nocapture +/// cargo test --test job_cleanup_test test_skip_locked_prevents_contention -- --nocapture +/// +/// Setup test data (run in psql first): +/// -- Set short retention for testing +/// UPDATE global_settings SET value = '3600' WHERE name = 'retention_period_secs'; +/// +/// -- Insert expired test jobs +/// INSERT INTO v2_job (id, tag, workspace_id, created_at, created_by, permissioned_as, permissioned_as_email, kind, same_worker, visible_to_owner) +/// SELECT gen_random_uuid(), 'deno', 'admins', now() - interval '2 hours', 'test_user', 'test_user', 'test@example.com', 'script', false, true +/// FROM generate_series(1, 10000); +/// +/// INSERT INTO v2_job_completed (id, workspace_id, duration_ms, deleted, status, completed_at, started_at) +/// SELECT j.id, j.workspace_id, 100, false, 'success', j.created_at + interval '1 second', j.created_at +/// FROM v2_job j WHERE j.created_by = 'test_user'; +use sqlx::postgres::PgPoolOptions; +use std::time::Instant; +use uuid::Uuid; + +#[tokio::test] +#[ignore] +async fn test_batched_job_cleanup() { + // Connect to database + let database_url = std::env::var("DATABASE_URL") + .unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string()); + + let db = PgPoolOptions::new() + .max_connections(5) + .connect(&database_url) + .await + .expect("Failed to connect to database"); + + // Get retention period from settings + let retention_secs: i64 = 2592000; + + println!( + "Retention period: {} seconds ({} days)", + retention_secs, + retention_secs / 86400 + ); + + // Count jobs eligible for deletion + let eligible_count: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_completed WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval", + retention_secs + ) + .fetch_one(&db) + .await + .expect("Failed to count eligible jobs") + .unwrap_or(0); + + println!("Jobs eligible for deletion: {}", eligible_count); + + if eligible_count == 0 { + println!("No jobs to delete. To create test data, run:"); + println!(" psql $DATABASE_URL -c \"UPDATE global_settings SET value = '3600' WHERE name = 'retention_period_secs';\""); + println!(" Then insert expired jobs (see test file header for SQL)"); + return; + } + + // Test different batch sizes + let batch_sizes = [1000, 5000, 10000]; + + for batch_size in batch_sizes { + let remaining: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_completed WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval", + retention_secs + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap_or(0); + + if remaining == 0 { + println!("All jobs deleted, stopping batch size tests"); + break; + } + + println!( + "\n--- Testing batch_size={} (remaining: {}) ---", + batch_size, remaining + ); + + let start = Instant::now(); + let mut total_deleted = 0u64; + let mut batch_num = 0u32; + let max_batches = 5; + + loop { + if batch_num >= max_batches { + println!("Reached max batches limit ({})", max_batches); + break; + } + + let batch_start = Instant::now(); + + // Run batched deletion with LIMIT and SKIP LOCKED (same as monitor.rs) + let deleted_jobs: Vec = sqlx::query_scalar!( + "DELETE FROM v2_job_completed + WHERE id IN ( + SELECT id FROM v2_job_completed + WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval + ORDER BY completed_at ASC + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + RETURNING id", + retention_secs, + batch_size as i64 + ) + .fetch_all(&db) + .await + .expect("Failed to delete batch"); + + let batch_count = deleted_jobs.len(); + let batch_elapsed = batch_start.elapsed(); + + if batch_count == 0 { + println!("No more jobs to delete"); + break; + } + + // Delete related records + sqlx::query!( + "DELETE FROM job_stats WHERE job_id = ANY($1)", + &deleted_jobs + ) + .execute(&db) + .await + .ok(); + + sqlx::query!("DELETE FROM job_logs WHERE job_id = ANY($1)", &deleted_jobs) + .execute(&db) + .await + .ok(); + + sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted_jobs) + .execute(&db) + .await + .ok(); + + sqlx::query!( + "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", + &deleted_jobs + ) + .execute(&db) + .await + .ok(); + + total_deleted += batch_count as u64; + batch_num += 1; + + let rate = batch_count as f64 / batch_elapsed.as_secs_f64(); + println!( + " Batch {}: deleted {} jobs in {:?} ({:.0} jobs/sec)", + batch_num, batch_count, batch_elapsed, rate + ); + } + + let total_elapsed = start.elapsed(); + let overall_rate = if total_elapsed.as_secs_f64() > 0.0 { + total_deleted as f64 / total_elapsed.as_secs_f64() + } else { + 0.0 + }; + println!( + "batch_size={}: deleted {} jobs in {} batches, total time {:?} ({:.0} jobs/sec)", + batch_size, total_deleted, batch_num, total_elapsed, overall_rate + ); + } + + let final_count: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_completed WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval", + retention_secs + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap_or(0); + + println!( + "\nFinal eligible count: {} (started with {})", + final_count, eligible_count + ); +} + +#[tokio::test] +#[ignore] +async fn test_skip_locked_prevents_contention() { + // This test verifies that SKIP LOCKED allows concurrent cleanup without deadlocks + let database_url = std::env::var("DATABASE_URL") + .unwrap_or("postgres://postgres:changeme@localhost:5432/windmill".to_string()); + + let db = PgPoolOptions::new() + .max_connections(10) + .connect(&database_url) + .await + .expect("Failed to connect to database"); + + let retention_secs: i64 = sqlx::query_scalar!( + "SELECT COALESCE((SELECT value::bigint FROM global_settings WHERE name = 'retention_period_secs'), 2592000)" + ) + .fetch_one(&db) + .await + .expect("Failed to get retention period") + .unwrap_or(2592000); + + let eligible: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_completed WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval", + retention_secs + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap_or(0); + + println!("Retention period: {} seconds", retention_secs); + println!("Jobs eligible for deletion: {}", eligible); + + if eligible == 0 { + println!("No jobs to delete, skipping concurrent test"); + return; + } + + let batch_size: i64 = 1000; + + // Run 3 concurrent cleanup tasks (simulating multiple server replicas) + let handles: Vec<_> = (0..3) + .map(|worker_id| { + let db = db.clone(); + tokio::spawn(async move { + let start = Instant::now(); + let mut total = 0u64; + + for _batch in 0..3 { + let deleted: Vec = sqlx::query_scalar!( + "DELETE FROM v2_job_completed + WHERE id IN ( + SELECT id FROM v2_job_completed + WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval + ORDER BY completed_at ASC + LIMIT $2 + FOR UPDATE SKIP LOCKED + ) + RETURNING id", + retention_secs, + batch_size + ) + .fetch_all(&db) + .await + .unwrap_or_default(); + + let count = deleted.len(); + total += count as u64; + + if count > 0 { + sqlx::query!("DELETE FROM v2_job WHERE id = ANY($1)", &deleted) + .execute(&db) + .await + .ok(); + } + + if count == 0 { + break; + } + } + + println!( + "Worker {}: deleted {} jobs in {:?}", + worker_id, + total, + start.elapsed() + ); + total + }) + }) + .collect(); + + let results: Vec = futures::future::join_all(handles) + .await + .into_iter() + .map(|r| r.unwrap_or(0)) + .collect(); + + let total: u64 = results.iter().sum(); + println!("\nTotal deleted by all workers: {}", total); + println!("Per worker: {:?}", results); + println!("\nWith SKIP LOCKED: no deadlocks, work distributed across workers"); +} diff --git a/backend/tests/nativets_jobs.rs b/backend/tests/nativets_jobs.rs new file mode 100644 index 0000000000..c654882589 --- /dev/null +++ b/backend/tests/nativets_jobs.rs @@ -0,0 +1,269 @@ +/* + * Integration tests for nativets (//native) job execution. + * + * These tests ensure that the V8 runtime is properly initialized and that + * nativets jobs execute correctly without segfaults. They cover sync functions, + * async functions, and fetch operations. + * + * All nativets tests run inside a single #[sqlx::test] with a single long-lived + * worker, because V8 isolates cannot be safely created/destroyed/recreated + * across different tokio runtimes or worker lifecycles. + * + * Run with: + * cargo test -p windmill --features "deno_core" --test nativets_jobs -- --nocapture + */ + +#[cfg(feature = "deno_core")] +mod common; + +#[cfg(feature = "deno_core")] +use common::*; + +#[cfg(feature = "deno_core")] +use futures::StreamExt; +#[cfg(feature = "deno_core")] +use sqlx::{Pool, Postgres}; +#[cfg(feature = "deno_core")] +use windmill_common::jobs::{JobPayload, RawCode}; +#[cfg(feature = "deno_core")] +use windmill_common::scripts::ScriptLang; + +#[cfg(feature = "deno_core")] +fn nativets_code(content: &str) -> JobPayload { + JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }) +} + +#[cfg(feature = "deno_core")] +async fn push_and_wait( + db: &Pool, + job: RunJob, + listener: &mut (impl StreamExt + Unpin), +) -> CompletedJob { + let uuid = job.push(db).await; + let deadline = std::time::Instant::now() + tokio::time::Duration::from_secs(60); + loop { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + panic!("Timed out waiting for job {uuid}"); + } + match tokio::time::timeout(remaining, listener.next()).await { + Ok(Some(completed_uuid)) if completed_uuid == uuid => break, + Ok(Some(_)) => continue, + Ok(None) => panic!("Listener ended while waiting for {uuid}"), + Err(_) => panic!("Timed out waiting for job {uuid}"), + } + } + completed_job(uuid, db).await +} + +/// All nativets tests share a single worker to avoid V8 isolate lifecycle issues. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_nativets_jobs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let mut listener = listen_for_completed_jobs(&db).await; + let conn = windmill_common::worker::Connection::Sql(db.clone()); + let (killpill, _worker_handle) = spawn_test_worker(&conn, port); + + // -- simple string return -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): string { + return "hello from nativets"; +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "simple_return failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!("hello from nativets") + ); + } + + // -- arithmetic with args -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(x: number, y: number): number { + return x + y; +} +"#, + )) + .arg("x", serde_json::json!(17)) + .arg("y", serde_json::json!(25)), + &mut listener, + ) + .await; + assert!(result.success, "with_args failed: {:?}", result.result); + assert_eq!(result.json_result().unwrap(), serde_json::json!(42)); + } + + // -- complex object return -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(items: {name: string, value: number}[]): {total: number, names: string[]} { + return { + total: items.reduce((sum, i) => sum + i.value, 0), + names: items.map(i => i.name).sort(), + }; +} +"#, + )) + .arg( + "items", + serde_json::json!([ + {"name": "c", "value": 30}, + {"name": "a", "value": 10}, + {"name": "b", "value": 20} + ]), + ), + &mut listener, + ) + .await; + assert!(result.success, "object_return failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!({"total": 60, "names": ["a", "b", "c"]}) + ); + } + + // -- async function (Promise) -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export async function main(): Promise { + const result = await Promise.resolve(42); + return result; +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "async failed: {:?}", result.result); + assert_eq!(result.json_result().unwrap(), serde_json::json!(42)); + } + + // -- fetch hitting the internal API server -- + { + let code = format!( + r#"//native + +export async function main(): Promise {{ + const resp = await fetch("http://localhost:{port}/api/version"); + return {{ status: resp.status, ok: resp.ok }}; +}} +"# + ); + let result = push_and_wait(&db, RunJob::from(nativets_code(&code)), &mut listener).await; + assert!(result.success, "fetch failed: {:?}", result.result); + let val = result.json_result().unwrap(); + assert_eq!(val["ok"], serde_json::json!(true)); + assert_eq!(val["status"], serde_json::json!(200)); + } + + // -- regex -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(text: string, pattern: string): string[] { + const regex = new RegExp(pattern, 'g'); + return [...text.matchAll(regex)].map(m => m[0]); +} +"#, + )) + .arg("text", serde_json::json!("foo123bar456baz789")) + .arg("pattern", serde_json::json!("\\d+")), + &mut listener, + ) + .await; + assert!(result.success, "regex failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!(["123", "456", "789"]) + ); + } + + // -- JSON roundtrip -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): string { + const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } }); + const parsed = JSON.parse(data); + return parsed.nested.arr.map((x: number) => x * 10).join(","); +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "json_roundtrip failed: {:?}", result.result); + assert_eq!(result.json_result().unwrap(), serde_json::json!("10,20,30")); + } + + // -- no args, array return -- + { + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): number[] { + return Array.from({length: 5}, (_, i) => i * i); +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "no_args failed: {:?}", result.result); + assert_eq!( + result.json_result().unwrap(), + serde_json::json!([0, 1, 4, 9, 16]) + ); + } + + killpill.send(); + Ok(()) +} diff --git a/backend/tests/nativets_stress.rs b/backend/tests/nativets_stress.rs new file mode 100644 index 0000000000..cf3710d86c --- /dev/null +++ b/backend/tests/nativets_stress.rs @@ -0,0 +1,421 @@ +/* + * Stress test for parallel nativets execution. + * + * Spawns 8 workers and pushes hundreds of fast nativets jobs concurrently. + * Tests: arithmetic, JSON manipulation, string ops, fetch to an internal HTTP server, + * and async operations. Validates all jobs complete successfully with correct results. + * + * IMPORTANT: V8 segfaults when the test harness captures stdout (default behavior). + * Always run with --nocapture: + * cargo test -p windmill --features "deno_core" --test nativets_stress -- --nocapture + */ + +#[cfg(feature = "deno_core")] +mod common; + +#[cfg(feature = "deno_core")] +use common::*; + +#[cfg(feature = "deno_core")] +use std::time::Instant; + +#[cfg(feature = "deno_core")] +use futures::StreamExt; +#[cfg(feature = "deno_core")] +use serde_json::json; +#[cfg(feature = "deno_core")] +use sqlx::{Pool, Postgres}; +#[cfg(feature = "deno_core")] +use uuid::Uuid; + +#[cfg(feature = "deno_core")] +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, + worker::{Connection, WORKER_CONFIG}, + KillpillSender, +}; +#[cfg(feature = "deno_core")] +use windmill_queue::PushIsolationLevel; + +#[cfg(feature = "deno_core")] +const NUM_WORKERS: usize = 8; + +/// Various nativets scripts that are fast to execute but cover different features. +#[cfg(feature = "deno_core")] +fn job_scripts() -> Vec<(&'static str, serde_json::Value, serde_json::Value)> { + vec![ + // (script_code, args_json, expected_result) + ( + r#"//native + +export function main(x: number, y: number): number { + return x + y; +} +"#, + json!({"x": 10, "y": 32}), + json!(42), + ), + ( + r#"//native + +export function main(s: string): string { + return s.split('').reverse().join(''); +} +"#, + json!({"s": "hello"}), + json!("olleh"), + ), + ( + r#"//native + +export function main(n: number): number[] { + return Array.from({length: n}, (_, i) => i * i); +} +"#, + json!({"n": 5}), + json!([0, 1, 4, 9, 16]), + ), + ( + r#"//native + +export function main(items: {name: string, value: number}[]): {total: number, names: string[]} { + return { + total: items.reduce((sum, i) => sum + i.value, 0), + names: items.map(i => i.name).sort(), + }; +} +"#, + json!({"items": [{"name": "c", "value": 30}, {"name": "a", "value": 10}, {"name": "b", "value": 20}]}), + json!({"total": 60, "names": ["a", "b", "c"]}), + ), + ( + r#"//native + +export function main(a: number): object { + const fib = (n: number): number => n <= 1 ? n : fib(n - 1) + fib(n - 2); + return { input: a, fib: fib(a) }; +} +"#, + json!({"a": 10}), + json!({"input": 10, "fib": 55}), + ), + ( + r#"//native + +export function main(text: string, pattern: string): string[] { + const regex = new RegExp(pattern, 'g'); + return [...text.matchAll(regex)].map(m => m[0]); +} +"#, + json!({"text": "foo123bar456baz789", "pattern": "\\d+"}), + json!(["123", "456", "789"]), + ), + ( + r#"//native + +export function main(obj: Record): Record { + return Object.fromEntries( + Object.entries(obj).map(([k, v]) => [k.toUpperCase(), String(v * 2)]) + ); +} +"#, + json!({"obj": {"a": 1, "b": 2, "c": 3}}), + json!({"A": "2", "B": "4", "C": "6"}), + ), + ( + r#"//native + +export function main(): string { + const data = JSON.stringify({ key: "value", nested: { arr: [1, 2, 3] } }); + const parsed = JSON.parse(data); + return parsed.nested.arr.map((x: number) => x * 10).join(","); +} +"#, + json!({}), + json!("10,20,30"), + ), + ] +} + +#[cfg(feature = "deno_core")] +async fn push_job(db: &Pool, content: &str, args: &serde_json::Value) -> Uuid { + let mut hm_args = std::collections::HashMap::new(); + if let Some(obj) = args.as_object() { + for (k, v) in obj { + hm_args.insert(k.clone(), windmill_common::worker::to_raw_value(v)); + } + } + + let job = JobPayload::Code(RawCode { + hash: None, + content: content.to_string(), + path: None, + language: ScriptLang::Bun, // //native annotation → nativets + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + }); + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (uuid, tx) = windmill_queue::push( + db, + tx, + "test-workspace", + job, + windmill_queue::PushArgs::from(&hm_args), + /* user */ "test-user", + /* email */ "test@windmill.dev", + /* permissioned_as */ "u/test-user".to_string(), + /* token_prefix */ None, + /* scheduled_for */ None, + /* schedule_path */ None, + /* parent_job */ None, + /* root_job */ None, + /* flow_innermost_root_job */ None, + /* job_id */ None, + /* is_flow_step */ false, + /* same_worker */ false, + None, + true, + None, + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await + .expect("push must succeed"); + tx.commit().await.unwrap(); + uuid +} + +#[cfg(feature = "deno_core")] +fn spawn_workers( + conn: &Connection, + port: u16, + n: usize, +) -> (KillpillSender, Vec>) { + use std::sync::atomic::{AtomicUsize, Ordering}; + static WORKER_ID: AtomicUsize = AtomicUsize::new(0); + + std::fs::DirBuilder::new() + .recursive(true) + .create(windmill_worker::GO_BIN_CACHE_DIR) + .expect("could not create initial worker dir"); + + let (tx, _) = KillpillSender::new(n + 1); + let mut handles = Vec::with_capacity(n); + + for i in 0..n { + let rx = tx.subscribe(); + let conn = conn.clone(); + let tx2 = tx.clone(); + let id = WORKER_ID.fetch_add(1, Ordering::SeqCst); + let worker_name = format!("{id}/stress-w{i}"); + + let future = async move { + let base_internal_url = format!("http://localhost:{}", port); + { + let mut wc = WORKER_CONFIG.write().await; + wc.worker_tags = windmill_common::worker::DEFAULT_TAGS.clone(); + wc.priority_tags_sorted = vec![windmill_common::worker::PriorityTags { + priority: 0, + tags: wc.worker_tags.clone(), + }]; + windmill_common::worker::store_suspended_pull_query(&wc).await; + windmill_common::worker::store_pull_query(&wc).await; + } + windmill_worker::run_worker( + &conn, + "test-host", + worker_name, + i as u64, + n as u32, + "127.0.0.1", + rx, + tx2, + &base_internal_url, + ) + .await; + }; + + handles.push(tokio::task::spawn(future)); + } + + (tx, handles) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_parallel_nativets_stress(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let conn = Connection::Sql(db.clone()); + + // Set up completed job listener + let mut listener = listen_for_completed_jobs(&db).await; + + // Prepare job scripts + let scripts = job_scripts(); + + // Also prepare fetch-based scripts that hit the internal API server + let fetch_script_template = |port: u16| -> Vec<(String, serde_json::Value)> { + vec![ + ( + format!( + r#"//native + +export async function main(): Promise {{ + const resp = await fetch("http://localhost:{port}/api/version"); + return {{ status: resp.status, ok: resp.ok }}; +}} +"# + ), + json!({}), + ), + ( + format!( + r#"//native + +export async function main(): Promise {{ + const resp = await fetch("http://localhost:{port}/api/version"); + const text = await resp.text(); + return typeof text; +}} +"# + ), + json!({}), + ), + ] + }; + + let fetch_scripts = fetch_script_template(port); + let total_jobs = 200; + + let start = Instant::now(); + let mut expected_results: Vec<(Uuid, Option)> = + Vec::with_capacity(total_jobs); + + for i in 0..total_jobs { + let (uuid, expected) = if i % 10 < 8 { + // 80% non-fetch jobs + let idx = i % scripts.len(); + let (code, args, expected) = &scripts[idx]; + let uuid = push_job(&db, code, args).await; + (uuid, Some(expected.clone())) + } else { + // 20% fetch jobs (we don't check exact result since version string varies) + let idx = i % fetch_scripts.len(); + let (code, args) = &fetch_scripts[idx]; + let uuid = push_job(&db, code, args).await; + (uuid, None) // Don't check exact value, just success + }; + expected_results.push((uuid, expected)); + } + + let push_duration = start.elapsed(); + tracing::info!( + "Pushed {} jobs in {:?} ({:?}/job)", + total_jobs, + push_duration, + push_duration / total_jobs as u32 + ); + + // Spawn 8 workers + let (killpill, worker_handles) = spawn_workers(&conn, port, NUM_WORKERS); + + // Wait for all jobs to complete + let mut completed: std::collections::HashSet = std::collections::HashSet::new(); + let timeout_dur = tokio::time::Duration::from_secs(120); + let deadline = Instant::now() + timeout_dur; + + while completed.len() < total_jobs { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + panic!( + "Timed out waiting for jobs: {}/{} completed", + completed.len(), + total_jobs + ); + } + + match tokio::time::timeout(remaining, listener.next()).await { + Ok(Some(uuid)) => { + completed.insert(uuid); + if completed.len() % 50 == 0 { + tracing::info!( + "Progress: {}/{} jobs completed", + completed.len(), + total_jobs + ); + } + } + Ok(None) => panic!("Listener stream ended"), + Err(_) => panic!("Timed out: {}/{} completed", completed.len(), total_jobs), + } + } + + let exec_duration = start.elapsed(); + tracing::info!( + "All {} jobs completed in {:?} with {} workers ({:?}/job avg)", + total_jobs, + exec_duration, + NUM_WORKERS, + exec_duration / total_jobs as u32 + ); + + // Kill workers + killpill.send(); + for handle in worker_handles { + let _ = tokio::time::timeout(std::time::Duration::from_secs(10), handle).await; + } + + // Verify results + let mut successes = 0; + let mut failures = 0; + + for (uuid, expected) in &expected_results { + let job = completed_job(*uuid, &db).await; + if !job.success { + failures += 1; + tracing::error!("Job {} FAILED: {:?}", uuid, job.result); + continue; + } + successes += 1; + + if let Some(expected_val) = expected { + let result = job + .json_result() + .expect("successful job should have result"); + assert_eq!( + result, *expected_val, + "Job {} produced wrong result.\nExpected: {}\nGot: {}", + uuid, expected_val, result + ); + } + } + + tracing::info!( + "Results: {} successes, {} failures out of {} total", + successes, + failures, + total_jobs + ); + + assert_eq!(failures, 0, "All jobs should succeed"); + assert_eq!(successes, total_jobs, "All jobs should be verified"); + + Ok(()) +} diff --git a/backend/tests/resources.rs b/backend/tests/resources.rs new file mode 100644 index 0000000000..a81daf029e --- /dev/null +++ b/backend/tests/resources.rs @@ -0,0 +1,444 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn resource_url(port: u16, endpoint: &str, path: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/resources/{endpoint}/{path}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn authed_get(port: u16, endpoint: &str, path: &str) -> reqwest::Response { + authed(client().get(resource_url(port, endpoint, path))) + .send() + .await + .unwrap() +} + +#[sqlx::test(fixtures("base", "resources_test"))] +async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/resources"); + + // --- exists --- + let resp = authed_get(port, "exists", "u/test-user/simple_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "exists", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get --- + let resp = authed_get(port, "get", "u/test-user/simple_resource").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/simple_resource"); + assert_eq!(body["resource_type"], "object"); + assert_eq!(body["description"], "Simple resource"); + assert_eq!(body["value"], json!({"host": "localhost", "port": 5432})); + + let resp = authed_get(port, "get", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- get_value --- + let resp = authed_get(port, "get_value", "u/test-user/simple_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"host": "localhost", "port": 5432}) + ); + + let resp = authed_get(port, "get_value", "u/test-user/null_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + serde_json::Value::Null + ); + + let resp = authed_get(port, "get_value", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- get_value_interpolated --- + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/simple_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"host": "localhost", "port": 5432}) + ); + + // $var: interpolation + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/resource_with_var").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"host": "localhost", "password": "hunter2"}) + ); + + // $res: interpolation + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/resource_with_res").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"host": "localhost", "credentials": {"user": "admin", "password": "secret123"}}) + ); + + // mixed $var: and $res: refs + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/resource_mixed").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"host": "localhost", "password": "hunter2", "credentials": {"user": "admin", "password": "secret123"}}) + ); + + // chained $res: -> $var: + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/chained_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"service": "myapi", "auth": {"key": "sk-abc123"}}) + ); + + // null value + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/null_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + serde_json::Value::Null + ); + + // not found + let resp = + authed_get(port, "get_value_interpolated", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // array passthrough + let resp = authed_get( + port, + "get_value_interpolated", + "u/test-user/resource_with_array", + ) + .await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"hosts": ["host1", "host2"], "port": 5432}) + ); + + // scalar $var: ref + let resp = authed_get( + port, + "get_value_interpolated", + "u/test-user/scalar_var_resource", + ) + .await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, json!("hunter2")); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!( + list.len() >= 10, + "expected at least 10 resources from fixture, got {}", + list.len() + ); + assert!(list.iter().any(|r| r["path"] == "u/test-user/simple_resource")); + + // list with resource_type filter + let resp = authed(client().get(format!("{base}/list?resource_type=mcp_server"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().all(|r| r["resource_type"] == "mcp_server")); + + // --- list_search --- + let resp = authed(client().get(format!("{base}/list_search"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + + // --- list_names --- + let resp = authed(client().get(format!("{base}/list_names/object"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/new_resource", + "value": {"url": "https://example.com"}, + "description": "Created in test", + "resource_type": "object" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + // verify it exists + let resp = authed_get(port, "exists", "u/test-user/new_resource").await; + assert_eq!(resp.json::().await?, true); + + // verify value + let resp = authed_get(port, "get_value", "u/test-user/new_resource").await; + assert_eq!(resp.status(), 200); + assert_eq!( + resp.json::().await?, + json!({"url": "https://example.com"}) + ); + + // create duplicate -> 400 + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/new_resource", + "value": {}, + "resource_type": "object" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + // create with update_if_exists -> 201 + let resp = authed(client().post(format!("{base}/create?update_if_exists=true"))) + .json(&json!({ + "path": "u/test-user/new_resource", + "value": {"url": "https://updated.com"}, + "description": "Updated via upsert", + "resource_type": "object" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + let resp = authed_get(port, "get_value", "u/test-user/new_resource").await; + assert_eq!( + resp.json::().await?, + json!({"url": "https://updated.com"}) + ); + + // --- update (description) --- + let resp = authed(client().post(resource_url(port, "update", "u/test-user/new_resource"))) + .json(&json!({"description": "Updated description"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get", "u/test-user/new_resource").await; + let body = resp.json::().await?; + assert_eq!(body["description"], "Updated description"); + + // --- update_value --- + let resp = authed( + client().post(resource_url(port, "update_value", "u/test-user/new_resource")), + ) + .json(&json!({"value": {"url": "https://final.com"}})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get_value", "u/test-user/new_resource").await; + assert_eq!( + resp.json::().await?, + json!({"url": "https://final.com"}) + ); + + // --- delete --- + let resp = authed( + client().delete(resource_url(port, "delete", "u/test-user/new_resource")), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "exists", "u/test-user/new_resource").await; + assert_eq!(resp.json::().await?, false); + + // delete nonexistent -> 404 + let resp = authed( + client().delete(resource_url(port, "delete", "u/test-user/new_resource")), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 404); + + // --- file_resource_type_to_file_ext_map --- + let resp = authed(client().get(format!( + "{base}/file_resource_type_to_file_ext_map" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await?; + + // --- resource types --- + + // type/exists + let resp = authed_get(port, "type/exists", "test_db").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "type/exists", "nonexistent_type").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // type/get + let resp = authed_get(port, "type/get", "test_db").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["name"], "test_db"); + assert_eq!(body["description"], "Test DB type"); + + let resp = authed_get(port, "type/get", "nonexistent_type").await; + assert_eq!(resp.status(), 404); + + // type/list + let resp = authed(client().get(format!("{base}/type/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().any(|rt| rt["name"] == "test_db")); + + // type/listnames + let resp = authed(client().get(format!("{base}/type/listnames"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let names = resp.json::>().await?; + assert!(names.contains(&"test_db".to_string())); + + // type/create + let resp = authed(client().post(format!("{base}/type/create"))) + .json(&json!({ + "name": "new_test_type", + "description": "A new type", + "schema": {"type": "object"} + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + let resp = authed_get(port, "type/exists", "new_test_type").await; + assert_eq!(resp.json::().await?, true); + + // type/create duplicate -> 400 + let resp = authed(client().post(format!("{base}/type/create"))) + .json(&json!({ + "name": "new_test_type", + "description": "Duplicate", + "schema": {"type": "object"} + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + // type/update + let resp = authed(client().post(resource_url(port, "type/update", "new_test_type"))) + .json(&json!({"description": "Updated type desc"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "type/get", "new_test_type").await; + let body = resp.json::().await?; + assert_eq!(body["description"], "Updated type desc"); + + // type/delete + let resp = authed( + client().delete(resource_url(port, "type/delete", "new_test_type")), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "type/exists", "new_test_type").await; + assert_eq!(resp.json::().await?, false); + + Ok(()) +} + +#[cfg(feature = "mcp")] +#[sqlx::test(fixtures("base", "resources_test"))] +async fn test_mcp_tools(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // unauthenticated -> 401 + let resp = client() + .get(resource_url(port, "mcp_tools", "u/test-user/mcp_valid")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 401); + + // not found -> 404 + let resp = authed_get(port, "mcp_tools", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // null value -> 400 + let resp = authed_get(port, "mcp_tools", "u/test-user/mcp_null").await; + assert_eq!(resp.status(), 400); + + // invalid format -> 400 with parse error message + let resp = authed_get(port, "mcp_tools", "u/test-user/mcp_invalid_format").await; + assert_eq!(resp.status(), 400); + let body = resp.text().await?; + assert!( + body.contains("Failed to parse MCP resource"), + "expected parse error, got: {body}" + ); + + // valid MCP resource but unreachable server -> 500 + let resp = authed_get(port, "mcp_tools", "u/test-user/mcp_valid").await; + assert_eq!(resp.status(), 500); + let body = resp.text().await?; + assert!( + body.contains("Failed to connect to MCP server"), + "expected connection error, got: {body}" + ); + + Ok(()) +} diff --git a/backend/tests/schedule_push.rs b/backend/tests/schedule_push.rs new file mode 100644 index 0000000000..b26336b637 --- /dev/null +++ b/backend/tests/schedule_push.rs @@ -0,0 +1,1615 @@ +mod common; + +mod schedule_push { + use chrono::Utc; + use sqlx::{Pool, Postgres}; + use windmill_common::db::Authed; + use windmill_common::jobs::{JobKind, JobTriggerKind}; + use windmill_common::schedule::Schedule; + use windmill_common::scripts::ScriptHash; + use windmill_common::users::username_to_permissioned_as; + use windmill_queue::jobs::{try_schedule_next_job, MiniCompletedJob}; + use windmill_queue::schedule::push_scheduled_job; + + fn make_schedule(overrides: impl FnOnce(&mut Schedule)) -> Schedule { + let mut s = Schedule { + workspace_id: "test-workspace".to_string(), + path: "f/system/test_schedule".to_string(), + edited_by: "test-user".to_string(), + edited_at: Utc::now(), + schedule: "0 0 */5 * * *".to_string(), + timezone: "UTC".to_string(), + enabled: true, + script_path: "f/system/test_script".to_string(), + is_flow: false, + args: None, + extra_perms: serde_json::json!({}), + email: "test@windmill.dev".to_string(), + error: None, + on_failure: None, + on_failure_times: None, + on_failure_exact: None, + on_failure_extra_args: None, + on_recovery: None, + on_recovery_times: None, + on_recovery_extra_args: None, + on_success: None, + on_success_extra_args: None, + ws_error_handler_muted: false, + retry: None, + no_flow_overlap: false, + summary: None, + description: None, + tag: None, + paused_until: None, + cron_version: None, + dynamic_skip: None, + }; + overrides(&mut s); + s + } + + fn make_authed() -> Authed { + Authed { + email: "test@windmill.dev".to_string(), + username: "test-user".to_string(), + is_admin: true, + is_operator: false, + groups: vec![], + folders: vec![], + scopes: None, + token_prefix: None, + } + } + + fn make_completed_job(schedule: &Schedule) -> MiniCompletedJob { + MiniCompletedJob { + id: uuid::Uuid::new_v4(), + workspace_id: schedule.workspace_id.clone(), + runnable_id: Some(ScriptHash(100001)), + scheduled_for: Utc::now() - chrono::Duration::minutes(5), + parent_job: None, + flow_innermost_root_job: None, + runnable_path: Some(schedule.script_path.clone()), + kind: JobKind::Script, + started_at: Some(Utc::now() - chrono::Duration::minutes(4)), + permissioned_as: username_to_permissioned_as(&schedule.edited_by), + created_by: schedule.edited_by.clone(), + script_lang: None, + permissioned_as_email: schedule.email.clone(), + flow_step_id: None, + trigger_kind: Some(JobTriggerKind::Schedule), + trigger: Some(schedule.path.clone()), + priority: None, + concurrent_limit: None, + tag: "deno".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + runnable_settings_handle: None, + } + } + + async fn count_queued_jobs(db: &Pool) -> i64 { + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM v2_job_queue") + .fetch_one(db) + .await + .unwrap() + } + + async fn get_queued_job( + db: &Pool, + ) -> Option<( + String, // workspace_id + Option, // runnable_path + Option, // trigger + Option, // trigger_kind as text + )> { + sqlx::query_as::<_, (String, Option, Option, Option)>( + "SELECT j.workspace_id, j.runnable_path, j.trigger, j.trigger_kind::text + FROM v2_job j JOIN v2_job_queue q ON j.id = q.id + LIMIT 1", + ) + .fetch_optional(db) + .await + .unwrap() + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: basic script schedule + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_script_schedule(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + let (ws, path, trigger, trigger_kind) = get_queued_job(&db).await.unwrap(); + assert_eq!(ws, "test-workspace"); + assert_eq!(path.as_deref(), Some("f/system/test_script")); + assert_eq!(trigger.as_deref(), Some("f/system/test_schedule")); + assert_eq!(trigger_kind.as_deref(), Some("schedule")); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: flow schedule + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_flow_schedule(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.is_flow = true; + s.script_path = "f/system/test_flow".to_string(); + s.path = "f/system/flow_schedule".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + let (_, path, trigger, _) = get_queued_job(&db).await.unwrap(); + assert_eq!(path.as_deref(), Some("f/system/test_flow")); + assert_eq!(trigger.as_deref(), Some("f/system/flow_schedule")); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: on_behalf_of_email (script) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_script_on_behalf_of_email(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.script_path = "f/system/obo_script".to_string(); + s.path = "f/system/obo_schedule".to_string(); + }); + + // No pre-computed authed: forces the obo path inside push_scheduled_job + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + + let email = sqlx::query_scalar::<_, String>( + "SELECT permissioned_as_email FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(email, "obo@windmill.dev"); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: on_behalf_of_email (flow) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_flow_on_behalf_of_email(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.is_flow = true; + s.script_path = "f/system/obo_flow".to_string(); + s.path = "f/system/obo_flow_schedule".to_string(); + }); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + + let email = sqlx::query_scalar::<_, String>( + "SELECT permissioned_as_email FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(email, "obo@windmill.dev"); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: with retry wraps in SingleStepFlow + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_script_with_retry(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.retry = Some(serde_json::json!({ + "constant": { "attempts": 3, "seconds": 10 } + })); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + + // When retry is set, the job kind is singlescriptflow (SingleStepFlow wraps it) + let kind = sqlx::query_scalar::<_, String>( + "SELECT kind::text FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(kind, "singlestepflow"); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: duplicate detection (same schedule + time = skip) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_duplicate_skipped(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let authed = make_authed(); + + // First push + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + + // Second push with same schedule — should be idempotent + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); // Still 1 + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: invalid timezone + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_invalid_timezone(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.timezone = "Invalid/Timezone".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let result = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await; + assert!(result.is_err()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: invalid cron expression + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_invalid_cron(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.schedule = "not a cron".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let result = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await; + assert!(result.is_err()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: invalid args (not a dict) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_invalid_args(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + let raw = serde_json::value::RawValue::from_string("[1,2,3]".to_string()).unwrap(); + s.args = Some(sqlx::types::Json(raw)); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let result = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await; + assert!(result.is_err()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: with schedule args passed to job + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_with_args(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + let raw = + serde_json::value::RawValue::from_string(r#"{"key":"value"}"#.to_string()).unwrap(); + s.args = Some(sqlx::types::Json(raw)); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + + let args = sqlx::query_scalar::<_, serde_json::Value>( + "SELECT args FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(args, serde_json::json!({"key": "value"})); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: script not found + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_script_not_found(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.script_path = "f/system/nonexistent".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let result = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await; + assert!(result.is_err()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: flow not found + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_flow_not_found(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.is_flow = true; + s.script_path = "f/system/nonexistent_flow".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let result = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await; + assert!(result.is_err()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: paused schedule (paused_until in future) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_paused_schedule(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.paused_until = Some(Utc::now() + chrono::Duration::hours(1)); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + // Job is still pushed, but scheduled_for will be after paused_until + assert_eq!(count_queued_jobs(&db).await, 1); + + let scheduled_for = sqlx::query_scalar::<_, chrono::DateTime>( + "SELECT scheduled_for FROM v2_job_queue LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert!(scheduled_for > Utc::now()); + Ok(()) + } + + // ----------------------------------------------------------------------- + // push_scheduled_job: clock shift detection (now_cutoff >= now) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_clock_shift(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let authed = make_authed(); + + // Pass a now_cutoff far in the future — simulates clock shift + let future_cutoff = Utc::now() + chrono::Duration::hours(24); + let tx = db.begin().await?; + let tx = + push_scheduled_job(&db, tx, &schedule, Some(&authed), Some(future_cutoff)).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + + // The scheduled_for should be after the cutoff + let scheduled_for = sqlx::query_scalar::<_, chrono::DateTime>( + "SELECT scheduled_for FROM v2_job_queue LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert!(scheduled_for > future_cutoff); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: disabled schedule does not push + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_handle_disabled_schedule(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.enabled = false; + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: script path mismatch does not push + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_handle_path_mismatch(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + "f/system/different_script", + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: enabled + matching path pushes next job + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_handle_enabled_pushes_next_job(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 1); + + let (_, path, trigger, trigger_kind) = get_queued_job(&db).await.unwrap(); + assert_eq!(path.as_deref(), Some("f/system/test_script")); + assert_eq!(trigger.as_deref(), Some("f/system/test_schedule")); + assert_eq!(trigger_kind.as_deref(), Some("schedule")); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: on_behalf_of_email via handle path + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_handle_on_behalf_of_email(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.script_path = "f/system/obo_script".to_string(); + s.path = "f/system/obo_schedule".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 1); + + let email = sqlx::query_scalar::<_, String>( + "SELECT permissioned_as_email FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(email, "obo@windmill.dev"); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: push failure returns error to caller + // (caller is responsible for retry + eventual disable) + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_handle_push_failure_disables_schedule(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + // NotFound: schedule disabled internally, no error returned (caller commits) + assert!(err.is_none()); + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 0); + + // Schedule should be disabled (NotFound is non-retryable) + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(!enabled); + assert!(error.is_some()); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: successful push is atomic with tx commit + // If the caller commits, both the next tick and any prior writes persist. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_success_atomic_with_commit(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + assert!(err.is_none()); + tx.commit().await?; + + // Next tick was pushed + assert_eq!(count_queued_jobs(&db).await, 1); + + // Schedule stayed enabled + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: successful push rolls back if tx is dropped + // Ensures no next tick leaks when the outer tx is not committed. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_success_rolls_back_on_drop(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + assert!(err.is_none()); + + // Intentionally drop tx without committing (simulates caller failure) + drop(tx); + + // Nothing should be visible — the next tick must NOT leak + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: schedule disable rolls back if tx is dropped + // The schedule must stay enabled when the caller doesn't commit. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failure_disable_rolls_back_on_drop(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + // NotFound: schedule disabled in tx, no error returned + assert!(err.is_none()); + + // Drop without commit — simulates zombie retry path + drop(tx); + + // Schedule should STILL be enabled (disable was rolled back with tx) + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "schedule must stay enabled when tx is rolled back"); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: tx remains usable after successful push + // The caller can perform additional writes on the returned tx. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_tx_usable_after_success(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (mut tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + assert!(err.is_none()); + + // Write something else on the same tx + sqlx::query("INSERT INTO global_settings (name, value) VALUES ('_test_after_push', '42'::jsonb)") + .execute(&mut *tx) + .await?; + tx.commit().await?; + + // Both the pushed job and the extra write should be visible + assert_eq!(count_queued_jobs(&db).await, 1); + let val: serde_json::Value = sqlx::query_scalar( + "SELECT value FROM global_settings WHERE name = '_test_after_push'", + ) + .fetch_one(&db) + .await?; + assert_eq!(val, serde_json::json!(42)); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: tx remains usable after push failure + disable + // The caller can still write on the returned tx after a schedule disable. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_tx_usable_after_failure(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (mut tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + // NotFound: schedule disabled in tx, no error returned + assert!(err.is_none()); + + // Write something else on the returned tx — tx is still usable + sqlx::query("INSERT INTO global_settings (name, value) VALUES ('_test_after_fail', '99'::jsonb)") + .execute(&mut *tx) + .await?; + tx.commit().await?; + + // Schedule should be disabled (NotFound is non-retryable) + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(!enabled); + assert!(error.is_some()); + + // Extra write should still be committed (tx is usable after push failure) + let val: serde_json::Value = sqlx::query_scalar( + "SELECT value FROM global_settings WHERE name = '_test_after_fail'", + ) + .fetch_one(&db) + .await?; + assert_eq!(val, serde_json::json!(99)); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: flow schedule pushes next job + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_try_schedule_flow(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.is_flow = true; + s.script_path = "f/system/test_flow".to_string(); + s.path = "f/system/flow_schedule".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 1); + + let (_, path, trigger, _) = get_queued_job(&db).await.unwrap(); + assert_eq!(path.as_deref(), Some("f/system/test_flow")); + assert_eq!(trigger.as_deref(), Some("f/system/flow_schedule")); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: script with retry wraps in SingleStepFlow + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_try_schedule_with_retry(db: Pool) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.retry = Some(serde_json::json!({ + "constant": { "attempts": 3, "seconds": 10 } + })); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 1); + + let kind = sqlx::query_scalar::<_, String>( + "SELECT kind::text FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(kind, "singlestepflow"); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: push failure error message is stored on schedule + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_failure_stores_error_message(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + // NotFound: schedule disabled in tx, no error returned (caller commits) + assert!(err.is_none()); + tx.commit().await?; + + let error: String = sqlx::query_scalar( + "SELECT error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + // Error should mention the script that couldn't be found + assert!( + error.contains("nonexistent"), + "error message should describe the failure, got: {error}" + ); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: disabled schedule leaves no side effects + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_disabled_schedule_no_side_effects(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/disabled_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', false, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/disabled_schedule".to_string(); + s.enabled = false; + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + + // Schedule should remain disabled (not re-enabled) and no error set + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/disabled_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(!enabled); + assert!(error.is_none(), "disabled schedule should not get an error set"); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: path mismatch leaves schedule unchanged + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_path_mismatch_no_side_effects(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + "f/system/different_script", + ) + .await; + tx.commit().await?; + assert!(err.is_none()); + + // Schedule should remain enabled, no error, no jobs pushed + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "schedule must stay enabled on path mismatch"); + assert!(error.is_none(), "no error should be set on path mismatch"); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // try_schedule_next_job: push failure with schedule not in DB + // When the schedule row doesn't exist, the UPDATE affects 0 rows but + // doesn't error. The function should return (tx, None). + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_push_failure_schedule_not_in_db(db: Pool) -> anyhow::Result<()> { + // Do NOT insert a schedule row — the disable UPDATE will match 0 rows + let schedule = make_schedule(|s| { + s.path = "f/system/ghost_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + drop(tx); + // NotFound: disable succeeds (UPDATE 0 rows is not an error), no error returned + assert!(err.is_none()); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // ----------------------------------------------------------------------- + // Critical invariant: after commit, it's impossible to have schedule + // enabled + no next tick + function returned success. + // We verify: if push succeeds, both the job AND the schedule's + // unmodified enabled state are committed together. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_invariant_success_means_tick_committed(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + assert!(err.is_none()); + tx.commit().await?; + + // After commit: schedule enabled AND next tick exists — invariant holds + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "schedule must be enabled after successful push"); + assert_eq!( + count_queued_jobs(&db).await, + 1, + "next tick must exist after successful push + commit" + ); + Ok(()) + } + + // ----------------------------------------------------------------------- + // Critical invariant: after commit with push failure, the schedule is + // disabled. It's never the case that we commit with the schedule still + // enabled and no next tick. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_invariant_failure_means_disabled_after_commit( + db: Pool, + ) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + // NotFound: schedule disabled in tx, no error returned (caller commits) + assert!(err.is_none()); + tx.commit().await?; + + // After commit: no next tick, but schedule is disabled — invariant holds + assert_eq!(count_queued_jobs(&db).await, 0); + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!( + !enabled, + "schedule must be disabled when push fails with NotFound and tx commits" + ); + Ok(()) + } + + // ----------------------------------------------------------------------- + // Critical invariant: if tx is NOT committed (zombie path), neither + // the next tick nor the schedule disable persists. The schedule stays + // enabled so that zombie retry can re-attempt. + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_invariant_rollback_preserves_schedule_for_retry( + db: Pool, + ) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + let tx = db.begin().await?; + let (tx, err) = try_schedule_next_job( + &db, + tx, + &job, + &schedule, + &schedule.script_path, + ) + .await; + // NotFound: schedule disabled in tx, no error returned + assert!(err.is_none()); + + // Simulate zombie path: drop tx without commit + drop(tx); + + // Schedule still enabled (disable was rolled back with tx) — ready for retry + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "schedule must remain enabled after rollback"); + assert!(error.is_none(), "error must not persist after rollback"); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + // =================================================================== + // Failpoint tests — feature-gated, only compiled under `failpoints` + // =================================================================== + + mod failpoint_tests { + use super::*; + use windmill_queue::jobs::schedule_failpoints::{ScheduleFailPoint, ACTIVE}; + + // --------------------------------------------------------------- + // SavepointCreate failpoint → schedule disabled, 0 jobs + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_savepoint_create_disables(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::SavepointCreate, async { + let tx = db.begin().await.unwrap(); + let (tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + // Transient error returned to caller for retry + assert!(err.is_some()); + drop(tx); + + assert_eq!(count_queued_jobs(&db).await, 0); + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await + .unwrap(); + assert!(enabled, "schedule must stay enabled for caller retry"); + }).await; + Ok(()) + } + + // --------------------------------------------------------------- + // Push failpoint → schedule disabled, 0 jobs + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_push_disables(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::Push, async { + let tx = db.begin().await.unwrap(); + let (tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + // Transient error returned to caller for retry + assert!(err.is_some()); + drop(tx); + + assert_eq!(count_queued_jobs(&db).await, 0); + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await + .unwrap(); + assert!(enabled, "schedule must stay enabled for caller retry"); + }).await; + Ok(()) + } + + // --------------------------------------------------------------- + // SavepointCommit failpoint → schedule disabled, 0 jobs + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_savepoint_commit_disables(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::SavepointCommit, async { + let tx = db.begin().await.unwrap(); + let (tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + // Transient error returned to caller for retry + assert!(err.is_some()); + drop(tx); + + assert_eq!(count_queued_jobs(&db).await, 0, "pushed job must be rolled back"); + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await + .unwrap(); + assert!(enabled, "schedule must stay enabled for caller retry"); + }).await; + Ok(()) + } + + // --------------------------------------------------------------- + // ScheduleDisable failpoint → returns Some(err), caller doesn't commit + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_schedule_disable_returns_err(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::ScheduleDisable, async { + let tx = db.begin().await.unwrap(); + let (_tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + assert!(err.is_some(), "must return Some(err) when disable fails"); + }).await; + Ok(()) + } + + // --------------------------------------------------------------- + // ScheduleDisable failpoint + tx drop → schedule stays enabled + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_disable_failure_rollback(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/bad_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/nonexistent', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/system/bad_schedule".to_string(); + s.script_path = "f/system/nonexistent".to_string(); + }); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::ScheduleDisable, async { + let tx = db.begin().await.unwrap(); + let (tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + assert!(err.is_some(), "must return Some(err) when disable fails"); + drop(tx); + + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await + .unwrap(); + assert!(enabled, "schedule must stay enabled when tx is dropped after disable failure"); + assert!(error.is_none(), "error must not persist after rollback"); + }).await; + Ok(()) + } + + // --------------------------------------------------------------- + // PushQuotaExceeded failpoint (script) → schedule disabled, 0 jobs, + // no error handler notification (QuotaExceeded is silenced) + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_push_quota_exceeded_script(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_script', false, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|_| {}); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::PushQuotaExceeded, async { + let tx = db.begin().await.unwrap(); + let (tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + // QuotaExceeded: schedule disabled internally, no error returned + assert!(err.is_none(), "QuotaExceeded should be handled internally (returns None)"); + tx.commit().await.unwrap(); + + assert_eq!(count_queued_jobs(&db).await, 0); + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await + .unwrap(); + assert!(!enabled, "schedule must be disabled after QuotaExceeded"); + assert!(error.is_some(), "error must be set on schedule"); + assert!(error.unwrap().contains("quota"), "error message should mention quota"); + }).await; + Ok(()) + } + + // --------------------------------------------------------------- + // PushQuotaExceeded failpoint (flow) → schedule disabled, 0 jobs + // --------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_failpoint_push_quota_exceeded_flow(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/flow_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_flow', true, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.is_flow = true; + s.script_path = "f/system/test_flow".to_string(); + s.path = "f/system/flow_schedule".to_string(); + }); + let job = make_completed_job(&schedule); + + ACTIVE.scope(ScheduleFailPoint::PushQuotaExceeded, async { + let tx = db.begin().await.unwrap(); + let (tx, err) = try_schedule_next_job( + &db, tx, &job, &schedule, &schedule.script_path, + ).await; + // QuotaExceeded: schedule disabled internally, no error returned + assert!(err.is_none(), "QuotaExceeded should be handled internally (returns None)"); + tx.commit().await.unwrap(); + + assert_eq!(count_queued_jobs(&db).await, 0); + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/flow_schedule'", + ) + .fetch_one(&db) + .await + .unwrap(); + assert!(!enabled, "flow schedule must be disabled after QuotaExceeded"); + assert!(error.is_some(), "error must be set on flow schedule"); + assert!(error.unwrap().contains("quota"), "error message should mention quota"); + }).await; + Ok(()) + } + } + + // =================================================================== + // Zombie detection tests — verify that a flow left in queue after + // SchedulePushZombieError meets the restart criteria in monitor.rs + // =================================================================== + + // ----------------------------------------------------------------------- + // When both schedule push AND post-retry disable fail, the flow job stays + // in v2_job_queue as a zombie. This test simulates that state and verifies: + // + // 1. The zombie detection query (from handle_zombie_flows) finds the flow + // 2. The flow meets restart criteria: first module is WaitingForPriorSteps + // and same_worker is false + // 3. After the restart UPDATE, the flow is re-queued (running=false) + // 4. The schedule remains enabled for retry on next flow execution + // ----------------------------------------------------------------------- + + #[sqlx::test(fixtures("base", "schedule_push"))] + async fn test_zombie_flow_after_schedule_push_failure_meets_restart_criteria( + db: Pool, + ) -> anyhow::Result<()> { + use windmill_common::flow_status::{FlowStatus, FlowStatusModule}; + + let flow_job_id = uuid::Uuid::new_v4(); + let now = Utc::now(); + let stale_ping = now - chrono::Duration::minutes(5); + + // Schedule still enabled — simulates both push and disable failing + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap) + VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_flow', true, 'test@windmill.dev', '{}', false, false)" + ) + .execute(&db) + .await?; + + // Flow job in v2_job (kind=flow, triggered by schedule, same_worker=false) + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_at, created_by, permissioned_as, permissioned_as_email, kind, runnable_path, trigger, trigger_kind, same_worker, visible_to_owner, tag) + VALUES ($1, 'test-workspace', $2, 'test-user', 'u/test-user', 'test@windmill.dev', 'flow', 'f/system/test_flow', 'f/system/test_schedule', 'schedule', false, false, 'flow')" + ) + .bind(flow_job_id) + .bind(now - chrono::Duration::minutes(5)) + .execute(&db) + .await?; + + // Queue entry: running=true (worker caught SchedulePushZombieError and returned Ok) + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, created_at, scheduled_for, running, started_at, tag, suspend) + VALUES ($1, 'test-workspace', $2, $2, true, $3, 'flow', 0)" + ) + .bind(flow_job_id) + .bind(now - chrono::Duration::minutes(5)) + .bind(now - chrono::Duration::minutes(4)) + .execute(&db) + .await?; + + // Stale ping — older than the 60s zombie transition timeout + sqlx::query("INSERT INTO v2_job_runtime (id, ping) VALUES ($1, $2)") + .bind(flow_job_id) + .bind(stale_ping) + .execute(&db) + .await?; + + // Flow status at step 0, first module = WaitingForPriorSteps + // This is the initial state of a flow that hasn't started any steps yet + let flow_status = serde_json::json!({ + "step": 0, + "modules": [{"type": "WaitingForPriorSteps", "id": "a"}], + "failure_module": {"type": "WaitingForPriorSteps", "id": "failure"}, + "retry": {"fail_count": 0, "failed_jobs": []}, + "cleanup_module": {"flow_jobs_to_clean": []} + }); + + sqlx::query("INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2::jsonb)") + .bind(flow_job_id) + .bind(&flow_status) + .execute(&db) + .await?; + + // Run the same zombie detection query from handle_zombie_flows (60s timeout) + let zombie_flows = sqlx::query_as::<_, (uuid::Uuid, String, Option, Option)>( + r#" + SELECT + j.id, j.workspace_id, j.same_worker, + COALESCE(s.flow_status, s.workflow_as_code_status)::text AS flow_status + FROM v2_job_queue q + JOIN v2_job j USING (id) + LEFT JOIN v2_job_runtime r USING (id) + LEFT JOIN v2_job_status s USING (id) + WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null + AND q.scheduled_for <= now() + AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode') + AND r.ping IS NOT NULL + AND r.ping < NOW() - ('60' || ' seconds')::interval + AND q.canceled_by IS NULL + "#, + ) + .fetch_all(&db) + .await?; + + assert_eq!(zombie_flows.len(), 1, "zombie flow must be detected"); + let (id, _ws, same_worker, flow_status_json) = &zombie_flows[0]; + assert_eq!(*id, flow_job_id); + + // Replicate the exact branching logic from handle_zombie_flows (monitor.rs:2711-2754). + // Only flows matching the restart condition get restarted; others are cancelled. + let status = flow_status_json + .as_deref() + .and_then(|x| serde_json::from_str::(x).ok()); + let should_restart = !same_worker.unwrap_or(false) + && status.is_some_and(|s| { + s.modules + .get(0) + .is_some_and(|x| matches!(x, FlowStatusModule::WaitingForPriorSteps { .. })) + }); + + assert!( + should_restart, + "flow must match the restart branch (not the cancel branch) in handle_zombie_flows" + ); + + // Apply the restart action — same UPDATE as handle_zombie_flows + sqlx::query( + "UPDATE v2_job_queue SET running = false, started_at = null + WHERE id = $1 AND canceled_by IS NULL", + ) + .bind(flow_job_id) + .execute(&db) + .await?; + + // Flow is re-queued for processing + let (running, started_at): (bool, Option>) = sqlx::query_as( + "SELECT running, started_at FROM v2_job_queue WHERE id = $1", + ) + .bind(flow_job_id) + .fetch_one(&db) + .await?; + assert!(!running, "flow must not be running after zombie restart"); + assert!(started_at.is_none(), "started_at must be null after zombie restart"); + + // Schedule still enabled — will be retried when flow re-executes + let enabled: bool = sqlx::query_scalar( + "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "schedule must remain enabled for retry after zombie restart"); + + // Flow job is NOT in v2_job_completed (it was never completed with error) + let completed_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM v2_job_completed WHERE id = $1", + ) + .bind(flow_job_id) + .fetch_one(&db) + .await?; + assert_eq!(completed_count, 0, "flow must not be in completed_job — it's a zombie, not an error"); + + Ok(()) + } +} diff --git a/backend/tests/schedules.rs b/backend/tests/schedules.rs new file mode 100644 index 0000000000..ee66264c5f --- /dev/null +++ b/backend/tests/schedules.rs @@ -0,0 +1,237 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn schedule_url(port: u16, endpoint: &str, path: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/schedules/{endpoint}/{path}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn authed_get(port: u16, endpoint: &str, path: &str) -> reqwest::Response { + authed(client().get(schedule_url(port, endpoint, path))) + .send() + .await + .unwrap() +} + +#[sqlx::test(fixtures("base"))] +async fn test_schedule_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/schedules"); + + // create a script for the schedule to reference + let resp = authed(client().post(format!( + "http://localhost:{port}/api/w/test-workspace/scripts/create" + ))) + .json(&json!({ + "path": "u/test-user/scheduled_script", + "summary": "Scheduled script", + "description": "", + "content": "export async function main() { return 1; }", + "language": "deno", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create script: {}", resp.text().await?); + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/test_schedule", + "schedule": "0 0 */6 * * *", + "timezone": "UTC", + "script_path": "u/test-user/scheduled_script", + "is_flow": false, + "enabled": false + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + + // create second schedule + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/another_schedule", + "schedule": "0 0 0 * * *", + "timezone": "America/New_York", + "script_path": "u/test-user/scheduled_script", + "is_flow": false, + "enabled": false + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create another: {}", resp.text().await?); + + // --- exists --- + let resp = authed_get(port, "exists", "u/test-user/test_schedule").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "exists", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get --- + let resp = authed_get(port, "get", "u/test-user/test_schedule").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_schedule"); + assert_eq!(body["schedule"], "0 0 */6 * * *"); + assert_eq!(body["timezone"], "UTC"); + assert_eq!(body["script_path"], "u/test-user/scheduled_script"); + assert_eq!(body["is_flow"], false); + assert_eq!(body["enabled"], false); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!( + list.len() >= 2, + "expected at least 2 schedules, got {}", + list.len() + ); + assert!(list.iter().any(|s| s["path"] == "u/test-user/test_schedule")); + + // --- list_with_jobs --- + let resp = authed(client().get(format!("{base}/list_with_jobs"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + + // --- update --- + let resp = authed(client().post(schedule_url( + port, + "update", + "u/test-user/test_schedule", + ))) + .json(&json!({ + "schedule": "0 0 */12 * * *", + "timezone": "Europe/Paris" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + + // verify update + let resp = authed_get(port, "get", "u/test-user/test_schedule").await; + let body = resp.json::().await?; + assert_eq!(body["schedule"], "0 0 */12 * * *"); + assert_eq!(body["timezone"], "Europe/Paris"); + + // --- setenabled --- + let resp = authed(client().post(schedule_url( + port, + "setenabled", + "u/test-user/test_schedule", + ))) + .json(&json!({"enabled": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get", "u/test-user/test_schedule").await; + let body = resp.json::().await?; + assert_eq!(body["enabled"], true); + + // disable it back + let resp = authed(client().post(schedule_url( + port, + "setenabled", + "u/test-user/test_schedule", + ))) + .json(&json!({"enabled": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- setdefaulthandler --- + let resp = authed(client().post(format!("{base}/setdefaulthandler"))) + .json(&json!({ + "handler_type": "error", + "override_existing": false, + "path": "u/test-user/scheduled_script", + "number_of_occurence": 1, + "number_of_occurence_exact": false + })) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "setdefaulthandler: {}", + resp.text().await? + ); + + // clear default handler + let resp = authed(client().post(format!("{base}/setdefaulthandler"))) + .json(&json!({ + "handler_type": "error", + "override_existing": false + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- delete --- + let resp = authed(client().delete(schedule_url( + port, + "delete", + "u/test-user/another_schedule", + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "exists", "u/test-user/another_schedule").await; + assert_eq!(resp.json::().await?, false); + + // ===== Global endpoints ===== + + // --- preview --- + let resp = authed(client().post(format!( + "http://localhost:{port}/api/schedules/preview" + ))) + .json(&json!({ + "schedule": "0 0 */6 * * *", + "timezone": "UTC" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "preview: {}", resp.text().await?); + + Ok(()) +} diff --git a/backend/tests/scripts.rs b/backend/tests/scripts.rs new file mode 100644 index 0000000000..3f55f36a64 --- /dev/null +++ b/backend/tests/scripts.rs @@ -0,0 +1,389 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn script_url(port: u16, endpoint: &str, path: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/scripts/{endpoint}/{path}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn authed_get(port: u16, endpoint: &str, path: &str) -> reqwest::Response { + authed(client().get(script_url(port, endpoint, path))) + .send() + .await + .unwrap() +} + +fn new_script(path: &str, summary: &str, content: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": summary, + "description": "", + "content": content, + "language": "deno", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {}, + "required": [] + } + }) +} + +#[sqlx::test(fixtures("base"))] +async fn test_script_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/scripts"); + + // --- create --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_script( + "u/test-user/test_script", + "Test script", + "export async function main() { return 42; }", + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create: {}", resp.text().await?); + + // create second script + let resp = authed(client().post(format!("{base}/create"))) + .json(&new_script( + "u/test-user/another_script", + "Another script", + "export async function main() { return 'hello'; }", + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "create another: {}", resp.text().await?); + + // --- exists --- + let resp = authed_get(port, "exists/p", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "exists/p", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get by path --- + let resp = authed_get(port, "get/p", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_script"); + assert_eq!(body["summary"], "Test script"); + assert_eq!(body["language"], "deno"); + assert!(body["hash"].is_string(), "expected hash to be a hex string"); + let hash = body["hash"].as_str().unwrap().to_string(); + + // get not found + let resp = authed_get(port, "get/p", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- get by hash --- + let resp = authed_get(port, "get/h", &hash).await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_script"); + + // --- get draft --- + let resp = authed_get(port, "get/draft", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/test_script"); + + // --- raw by path (requires language extension) --- + let resp = authed_get(port, "raw/p", "u/test-user/test_script.ts").await; + assert_eq!(resp.status(), 200); + let body = resp.text().await?; + assert!(body.contains("return 42"), "expected script content, got: {body}"); + + // --- raw by hash (requires .ts suffix) --- + let resp = authed_get(port, "raw/h", &format!("{hash}.ts")).await; + assert_eq!(resp.status(), 200); + let body = resp.text().await?; + assert!(body.contains("return 42")); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!( + list.len() >= 2, + "expected at least 2 scripts, got {}", + list.len() + ); + assert!(list.iter().any(|s| s["path"] == "u/test-user/test_script")); + + // list with path_start filter + let resp = authed(client().get(format!( + "{base}/list?path_start=u/test-user/another" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert_eq!(list.len(), 1); + assert_eq!(list[0]["path"], "u/test-user/another_script"); + + // --- list_search --- + let resp = authed(client().get(format!("{base}/list_search"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + + // --- list_paths --- + let resp = authed(client().get(format!("{base}/list_paths"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let paths = resp.json::>().await?; + assert!(paths.contains(&"u/test-user/test_script".to_string())); + + // --- history --- + let resp = authed_get(port, "history/p", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + let history = resp.json::>().await?; + assert!(!history.is_empty()); + + // --- get_latest_version --- + let resp = authed_get(port, "get_latest_version", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + + // --- deployment_status --- + let resp = authed_get(port, "deployment_status/h", &hash).await; + assert_eq!(resp.status(), 200); + + // --- raw_unpinned by path --- + let resp = authed_get(port, "raw_unpinned/p", "u/test-user/test_script.ts").await; + assert_eq!(resp.status(), 200); + let body = resp.text().await?; + assert!(body.contains("return 42")); + + // --- list_tokens --- + let resp = authed_get(port, "list_tokens", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- list_paths_from_workspace_runnable --- + let resp = authed_get( + port, + "list_paths_from_workspace_runnable", + "u/test-user/test_script", + ) + .await; + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- update script (create new version) --- + let mut updated = new_script( + "u/test-user/test_script", + "Updated test script", + "export async function main() { return 99; }", + ); + updated["parent_hash"] = json!(&hash); + let resp = authed(client().post(format!("{base}/create"))) + .json(&updated) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "update: {}", resp.text().await?); + + // verify new version + let resp = authed_get(port, "get/p", "u/test-user/test_script").await; + let body = resp.json::().await?; + assert_eq!(body["summary"], "Updated test script"); + let new_hash = body["hash"].as_str().unwrap(); + assert_ne!(new_hash, hash, "hash should change on update"); + + // history should have 2 entries now + let resp = authed_get(port, "history/p", "u/test-user/test_script").await; + let history = resp.json::>().await?; + assert!( + history.len() >= 2, + "expected at least 2 history entries, got {}", + history.len() + ); + + // --- history_update --- + let resp = authed(client().post(format!( + "{base}/history_update/h/{new_hash}/p/u/test-user/test_script" + ))) + .json(&json!({"deployment_msg": "deployed v2"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "history_update: {}", + resp.text().await? + ); + + // --- toggle_workspace_error_handler (EE-gated, expect 400 in OSS) --- + let resp = authed(client().post(script_url( + port, + "toggle_workspace_error_handler/p", + "u/test-user/test_script", + ))) + .json(&json!({})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "toggle_workspace_error_handler: unexpected status {}", + resp.status() + ); + + // --- get_triggers_count --- + let resp = authed_get(port, "get_triggers_count", "u/test-user/test_script").await; + assert_eq!(resp.status(), 200); + + // --- tokened_raw (global unauthed, token in URL) --- + let resp = client() + .get(format!( + "http://localhost:{port}/api/scripts_u/tokened_raw/test-workspace/SECRET_TOKEN/u/test-user/test_script.ts" + )) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "tokened_raw: {}", + resp.text().await? + ); + + // --- archive by path --- + let resp = authed(client().post(script_url( + port, + "archive/p", + "u/test-user/another_script", + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // archived script should still be gettable + let resp = authed_get(port, "get/p", "u/test-user/another_script").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["archived"], true); + let another_hash = body["hash"].as_str().unwrap().to_string(); + + // --- archive by hash --- + let resp = authed(client().post(script_url(port, "archive/h", &another_hash))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- delete by hash --- + let resp = authed(client().post(script_url(port, "delete/h", &another_hash))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- delete_bulk --- + let resp = authed(client().delete(format!("{base}/delete_bulk"))) + .json(&json!({"paths": ["u/test-user/test_script"]})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete_bulk: {}", resp.text().await?); + + let resp = authed_get(port, "exists/p", "u/test-user/test_script").await; + assert_eq!(resp.json::().await?, false); + + // --- empty_ts (global unauthed) --- + let resp = client() + .get(format!( + "http://localhost:{port}/api/scripts_u/empty_ts/u/test-user/any_script" + )) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.text().await?; + assert!(body.is_empty(), "expected empty string, got: {body}"); + + // ===== Hub endpoints (require external network, expect 500 or 200) ===== + + // --- hub/top --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/scripts/hub/top" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/top: unexpected status {}", + resp.status() + ); + + // --- hub/get (raw script by path, needs hub/ prefix in path) --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/scripts/hub/get/hub/1/hello" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/get: unexpected status {}", + resp.status() + ); + + // --- hub/get_full (full script by path, needs hub/ prefix in path) --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/scripts/hub/get_full/hub/1/hello" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "hub/get_full: unexpected status {}", + resp.status() + ); + + // --- integrations hub/list --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/integrations/hub/list" + ))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "integrations hub/list: unexpected status {}", + resp.status() + ); + + Ok(()) +} diff --git a/backend/tests/secret_backend_integration.rs b/backend/tests/secret_backend_integration.rs index e6deb84ff9..f9f0fe9f5f 100644 --- a/backend/tests/secret_backend_integration.rs +++ b/backend/tests/secret_backend_integration.rs @@ -26,28 +26,56 @@ #[cfg(all(feature = "private", feature = "enterprise"))] mod tests { use sqlx::{Pool, Postgres}; - use std::collections::HashMap; use windmill_common::secret_backend::{ migrate_secrets_to_database, migrate_secrets_to_vault, test_vault_connection, SecretBackend, VaultBackend, VaultSettings, }; + use windmill_common::variables::{build_crypt, decrypt, encrypt}; + + /// Plaintext values for test secrets. The fixture inserts PLACEHOLDERs that + /// `encrypt_fixture_secrets` replaces with properly encrypted values. + const TEST_SECRETS: &[(&str, &str, &str)] = &[ + ("test-workspace", "u/test-user/db_password", "db-pass-123"), + ("test-workspace", "u/test-user/api_key", "api-key-abc"), + ( + "test-workspace-2", + "u/test-user/other_secret", + "other-secret-value", + ), + ]; + + /// Encrypt the PLACEHOLDER values inserted by the fixture using the real + /// workspace encryption keys, so migration tests can decrypt them correctly. + async fn encrypt_fixture_secrets(db: &Pool) { + for &(workspace_id, path, plaintext) in TEST_SECRETS { + let mc = build_crypt(db, workspace_id).await.unwrap(); + let encrypted = encrypt(&mc, plaintext); + sqlx::query!( + "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3 AND is_secret = true", + encrypted, + workspace_id, + path, + ) + .execute(db) + .await + .unwrap(); + } + } - /// Check if vault tests should run (requires RUN_VAULT_TESTS=1 env var) fn should_run_vault_tests() -> bool { std::env::var("RUN_VAULT_TESTS") .map(|v| v == "1" || v.to_lowercase() == "true") .unwrap_or(false) } - /// Set up BASE_URL for JWT tests (required for OIDC issuer URL generation) + #[cfg(feature = "openidconnect")] async fn setup_base_url() { - let base_url = std::env::var("BASE_URL") - .unwrap_or_else(|_| "http://localhost:8000".to_string()); + let base_url = + std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost:8000".to_string()); let mut url = windmill_common::BASE_URL.write().await; *url = base_url; } - /// Skip test if RUN_VAULT_TESTS is not set macro_rules! skip_if_no_vault { () => { if !should_run_vault_tests() { @@ -71,6 +99,7 @@ mod tests { } } + #[cfg(feature = "openidconnect")] fn vault_settings_jwt() -> VaultSettings { VaultSettings { address: std::env::var("VAULT_ADDR") @@ -163,6 +192,7 @@ mod tests { // ==================== JWT Auth Tests ==================== /// Test Vault connection with JWT authentication + #[cfg(feature = "openidconnect")] #[sqlx::test(fixtures("base", "secret_backend"))] async fn test_vault_connection_jwt(db: Pool) { skip_if_no_vault!(); @@ -236,70 +266,51 @@ mod tests { let settings = vault_settings_static_token(); - // Verify Vault connection test_vault_connection(&settings, Some(&db)) .await .expect("Failed to connect to Vault"); - // Check initial state - let secrets_before = sqlx::query!( - "SELECT workspace_id, path, value FROM variable WHERE is_secret = true ORDER BY workspace_id, path" - ) - .fetch_all(&db) - .await - .expect("Failed to query secrets"); + // Encrypt fixture placeholders with real workspace keys + encrypt_fixture_secrets(&db).await; - println!( - "Found {} secrets in database before migration:", - secrets_before.len() - ); - for s in &secrets_before { - println!(" - {}/{}: {} chars", s.workspace_id, s.path, s.value.len()); - } + let secret_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM variable WHERE is_secret = true" + ) + .fetch_one(&db) + .await + .expect("Failed to count secrets"); + println!("Found {} secrets in database before migration", secret_count.unwrap_or(0)); // Run migration - println!("\nMigrating secrets to Vault..."); + println!("Migrating secrets to Vault..."); let report = migrate_secrets_to_vault(&db, &settings) .await .expect("Migration to Vault failed"); - println!("Migration report:"); - println!(" Total secrets: {}", report.total_secrets); - println!(" Migrated: {}", report.migrated_count); - println!(" Failed: {}", report.failed_count); + println!("Migration report: total={}, migrated={}, failed={}", + report.total_secrets, report.migrated_count, report.failed_count); if !report.failures.is_empty() { - println!(" Failures:"); for f in &report.failures { - println!(" - {}/{}: {}", f.workspace_id, f.path, f.error); + println!(" FAIL: {}/{}: {}", f.workspace_id, f.path, f.error); } } assert_eq!(report.failed_count, 0, "Migration had failures"); assert!(report.migrated_count > 0, "No secrets were migrated"); - // Verify secrets in Vault - println!("\nVerifying secrets in Vault..."); + // Verify decrypted values in Vault match original plaintexts let vault_backend = VaultBackend::new(settings.clone()); - - for secret in &secrets_before { - let result = vault_backend - .get_secret(&secret.workspace_id, &secret.path) - .await; - assert!( - result.is_ok(), - "Failed to read secret {}/{} from Vault: {:?}", - secret.workspace_id, - secret.path, - result.err() - ); - println!( - " ✓ {}/{} exists in Vault", - secret.workspace_id, secret.path - ); + for &(ws, path, expected_plaintext) in TEST_SECRETS { + let value = vault_backend + .get_secret(ws, path) + .await + .unwrap_or_else(|e| panic!("Failed to read {}/{} from Vault: {:?}", ws, path, e)); + assert_eq!(value, expected_plaintext, "Vault value mismatch for {}/{}", ws, path); + println!(" ✓ {}/{} correct in Vault", ws, path); } - println!("\n✓ Migration to Vault completed successfully"); + println!("✓ Migration to Vault completed successfully"); } /// Test migration from Vault back to database @@ -313,49 +324,50 @@ mod tests { .await .expect("Failed to connect to Vault"); + encrypt_fixture_secrets(&db).await; + // First migrate TO Vault println!("Setting up: migrating secrets to Vault first..."); let to_vault = migrate_secrets_to_vault(&db, &settings) .await .expect("Initial migration to Vault failed"); assert!(to_vault.migrated_count > 0, "No secrets to test with"); - println!(" Migrated {} secrets to Vault", to_vault.migrated_count); // Clear database values - println!("\nClearing database secret values..."); sqlx::query!("UPDATE variable SET value = 'CLEARED' WHERE is_secret = true") .execute(&db) .await .expect("Failed to clear values"); // Migrate back from Vault - println!("\nMigrating secrets from Vault to database..."); + println!("Migrating secrets from Vault to database..."); let report = migrate_secrets_to_database(&db, &settings) .await .expect("Migration to database failed"); - println!("Migration report:"); - println!(" Total secrets: {}", report.total_secrets); - println!(" Migrated: {}", report.migrated_count); - println!(" Failed: {}", report.failed_count); + println!("Migration report: total={}, migrated={}, failed={}", + report.total_secrets, report.migrated_count, report.failed_count); assert_eq!(report.failed_count, 0, "Migration had failures"); assert!(report.migrated_count > 0, "No secrets were migrated"); - // Verify restored - let restored = sqlx::query!( - "SELECT COUNT(*) as count FROM variable WHERE is_secret = true AND value != 'CLEARED'" - ) - .fetch_one(&db) - .await - .expect("Failed to count restored"); + // Verify restored values decrypt to the original plaintexts + for &(ws, path, expected_plaintext) in TEST_SECRETS { + let row = sqlx::query_scalar!( + "SELECT value FROM variable WHERE workspace_id = $1 AND path = $2 AND is_secret = true", + ws, path + ) + .fetch_one(&db) + .await + .unwrap_or_else(|_| panic!("Secret {}/{} not found in DB after migration", ws, path)); - assert!( - restored.count.unwrap_or(0) > 0, - "No secrets were restored in database" - ); + let mc = build_crypt(&db, ws).await.unwrap(); + let decrypted = decrypt(&mc, row).expect("Failed to decrypt restored value"); + assert_eq!(decrypted, expected_plaintext, "Restored value mismatch for {}/{}", ws, path); + println!(" ✓ {}/{} correctly restored in DB", ws, path); + } - println!("\n✓ Migration to database completed successfully"); + println!("✓ Migration to database completed successfully"); } // ==================== Variable Rename Tests ==================== @@ -443,68 +455,47 @@ mod tests { .await .expect("Failed to connect to Vault"); - // Get original secrets - let original: HashMap<(String, String), String> = sqlx::query!( - "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" - ) - .fetch_all(&db) - .await - .expect("Failed to query") - .into_iter() - .map(|r| ((r.workspace_id, r.path), r.value)) - .collect(); + encrypt_fixture_secrets(&db).await; - println!("Original secrets: {} entries", original.len()); - - // Step 1: DB -> Vault - println!("\n=== Step 1: Migrate DB -> Vault ==="); + println!("=== Step 1: Migrate DB -> Vault ==="); let to_vault = migrate_secrets_to_vault(&db, &settings) .await .expect("Migration to Vault failed"); - println!("Migrated {} secrets to Vault", to_vault.migrated_count); assert_eq!(to_vault.failed_count, 0); + println!("Migrated {} secrets to Vault", to_vault.migrated_count); - // Step 2: Clear DB - println!("\n=== Step 2: Clear database values ==="); + println!("=== Step 2: Clear database values ==="); sqlx::query!("UPDATE variable SET value = 'ROUND_TRIP_CLEARED' WHERE is_secret = true") .execute(&db) .await .expect("Failed to clear"); - // Step 3: Vault -> DB - println!("\n=== Step 3: Migrate Vault -> DB ==="); + println!("=== Step 3: Migrate Vault -> DB ==="); let to_db = migrate_secrets_to_database(&db, &settings) .await .expect("Migration to database failed"); - println!("Migrated {} secrets to database", to_db.migrated_count); assert_eq!(to_db.failed_count, 0); + println!("Migrated {} secrets to database", to_db.migrated_count); - // Step 4: Verify - println!("\n=== Step 4: Verify round-trip integrity ==="); - let restored: HashMap<(String, String), String> = sqlx::query!( - "SELECT workspace_id, path, value FROM variable WHERE is_secret = true" - ) - .fetch_all(&db) - .await - .expect("Failed to query") - .into_iter() - .map(|r| ((r.workspace_id, r.path), r.value)) - .collect(); - - for ((ws, path), _) in &original { - let restored_value = restored - .get(&(ws.clone(), path.clone())) - .expect(&format!("Secret {}/{} not found after round-trip", ws, path)); - - assert_ne!( - restored_value, "ROUND_TRIP_CLEARED", - "Secret {}/{} was not restored", + println!("=== Step 4: Verify round-trip integrity ==="); + for &(ws, path, expected_plaintext) in TEST_SECRETS { + let encrypted = sqlx::query_scalar!( + "SELECT value FROM variable WHERE workspace_id = $1 AND path = $2 AND is_secret = true", ws, path - ); - println!(" ✓ {}/{}: restored", ws, path); + ) + .fetch_one(&db) + .await + .unwrap_or_else(|_| panic!("Secret {}/{} not found after round-trip", ws, path)); + + assert_ne!(encrypted, "ROUND_TRIP_CLEARED", "Secret {}/{} was not restored", ws, path); + + let mc = build_crypt(&db, ws).await.unwrap(); + let decrypted = decrypt(&mc, encrypted).expect("Failed to decrypt"); + assert_eq!(decrypted, expected_plaintext, "Round-trip value mismatch for {}/{}", ws, path); + println!(" ✓ {}/{}: round-trip OK", ws, path); } - println!("\n✓ Full round-trip completed successfully!"); + println!("✓ Full round-trip completed successfully!"); } // ==================== Workspace Isolation Test ==================== @@ -517,7 +508,8 @@ mod tests { let settings = vault_settings_static_token(); let backend = VaultBackend::new(settings.clone()); - // First migrate secrets to Vault + encrypt_fixture_secrets(&db).await; + migrate_secrets_to_vault(&db, &settings) .await .expect("Migration failed"); @@ -542,7 +534,7 @@ mod tests { assert!(ws1.is_ok(), "Same-workspace access should work"); println!("✓ Same-workspace access works"); - println!("\n✓ Workspace isolation verified!"); + println!("✓ Workspace isolation verified!"); } } diff --git a/backend/tests/trigger_e2e.rs b/backend/tests/trigger_e2e.rs new file mode 100644 index 0000000000..7bfb735a5c --- /dev/null +++ b/backend/tests/trigger_e2e.rs @@ -0,0 +1,726 @@ +/*! + * End-to-end integration tests for Windmill trigger listeners. + * + * Each test is `#[ignore]` because it requires a running external service + * (MQTT broker, NATS server, Kafka broker, etc.). See individual test doc + * comments for setup instructions. + * + * Quick start — use the helper scripts in `tests/fixtures/`: + * ```bash + * ./tests/fixtures/start_all_triggers.sh # start all services + * ./tests/fixtures/start_all_triggers.sh oss # start OSS services only + * ./tests/fixtures/start_all_triggers.sh stop # tear down everything + * ``` + * + * The general pattern: + * 1. Insert a test script + trigger row + resource into the DB + * 2. Start the API server with listeners enabled (server_mode=true) + * 3. Connect to the external service and send a test message + * 4. Poll `v2_job` for a job matching the trigger path + trigger_kind + * 5. Verify the args shape/content + */ + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::time::Duration; + +mod common; +use common::*; + +// ============================================================================ +// Helpers +// ============================================================================ + +/// Row shape for polling v2_job. +#[derive(Debug)] +#[allow(dead_code)] +struct TriggerJobRow { + id: uuid::Uuid, + runnable_path: Option, + trigger_kind: Option, + args: Option>, +} + +/// Poll `v2_job` every 500ms for up to `timeout` for a job whose +/// `runnable_path` and `trigger_kind` match the expected values. +async fn poll_for_trigger_job( + db: &Pool, + script_path: &str, + trigger_kind: &str, + timeout: Duration, +) -> anyhow::Result { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let row = sqlx::query_as!( + TriggerJobRow, + r#" + SELECT id, runnable_path, trigger_kind AS "trigger_kind: String", + args AS "args: sqlx::types::Json" + FROM v2_job + WHERE runnable_path = $1 + AND trigger_kind = $2::job_trigger_kind + ORDER BY created_at DESC + LIMIT 1 + "#, + script_path, + trigger_kind as _, + ) + .fetch_optional(db) + .await?; + + if let Some(job) = row { + return Ok(job); + } + + if tokio::time::Instant::now() >= deadline { + anyhow::bail!( + "timed out waiting for trigger job (script_path={}, trigger_kind={})", + script_path, + trigger_kind + ); + } + + tokio::time::sleep(Duration::from_millis(500)).await; + } +} + +/// Insert a minimal test script row that trigger listeners can reference. +async fn insert_test_script(db: &Pool, path: &str) -> anyhow::Result { + let hash: i64 = rand::random::().unsigned_abs() as i64; + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, kind, lock) + VALUES ('test-workspace', $1, $2, '', '', 'def main(): pass', + 'test-user', 'python3', 'script', '')", + ) + .bind(hash) + .bind(path) + .execute(db) + .await?; + Ok(hash) +} + +/// Insert a resource row for triggers that resolve connection details from the +/// `resource` table. +async fn insert_resource( + db: &Pool, + path: &str, + resource_type: &str, + value: serde_json::Value, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', $1, $2::jsonb, $3, '{}'::jsonb, 'test-user')", + ) + .bind(path) + .bind(value) + .bind(resource_type) + .execute(db) + .await?; + Ok(()) +} + +// ============================================================================ +// MQTT Trigger E2E +// ============================================================================ + +/// End-to-end test for MQTT trigger. +/// +/// Requires a running MQTT broker. Setup: +/// ```bash +/// ./tests/fixtures/start_mqtt.sh +/// ``` +/// +/// Run: +/// ```bash +/// cargo test --test trigger_e2e test_mqtt_e2e --features mqtt_trigger \ +/// -- --ignored --nocapture +/// ``` +#[ignore = "requires running MQTT broker on localhost:1883"] +#[sqlx::test(fixtures("base"))] +async fn test_mqtt_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let script_path = "f/test/mqtt_e2e_handler"; + insert_test_script(&db, script_path).await?; + + insert_resource( + &db, + "u/test-user/mqtt_res", + "mqtt", + json!({ + "broker": "localhost", + "port": 1883 + }), + ) + .await?; + + sqlx::query( + r#" + INSERT INTO mqtt_trigger ( + path, mqtt_resource_path, subscribe_topics, client_version, + script_path, is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, ARRAY[$3::jsonb], $4::mqtt_client_version, $5, $6, $7, $8, $9) + "#, + ) + .bind("f/test/mqtt_e2e_trigger") + .bind("u/test-user/mqtt_res") + .bind(json!({"topic": "windmill/test/e2e", "qos": "qos0"})) + .bind("v5") + .bind(script_path) + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + tokio::time::sleep(Duration::from_secs(3)).await; + + // Publish a message using rumqttc + let mut mqtt_opts = rumqttc::MqttOptions::new("windmill-e2e-test", "localhost", 1883); + mqtt_opts.set_keep_alive(Duration::from_secs(5)); + let (client, mut eventloop) = rumqttc::AsyncClient::new(mqtt_opts, 10); + + // Drive the event loop in the background + let el_handle = tokio::spawn(async move { + loop { + match eventloop.poll().await { + Ok(_) => {} + Err(_) => break, + } + } + }); + + tokio::time::sleep(Duration::from_millis(500)).await; + client + .publish( + "windmill/test/e2e", + rumqttc::QoS::AtLeastOnce, + false, + b"hello from e2e test".to_vec(), + ) + .await?; + + let job = poll_for_trigger_job(&db, script_path, "mqtt", Duration::from_secs(30)).await?; + assert!(job.args.is_some(), "job should have args"); + + client.disconnect().await.ok(); + el_handle.abort(); + + Ok(()) +} + +// ============================================================================ +// WebSocket Trigger E2E +// ============================================================================ + +/// End-to-end test for WebSocket trigger. +/// +/// Requires a WebSocket echo server. Setup: +/// ```bash +/// ./tests/fixtures/start_websocket.sh +/// ``` +/// +/// Run: +/// ```bash +/// cargo test --test trigger_e2e test_websocket_e2e --features websocket \ +/// -- --ignored --nocapture +/// ``` +#[ignore = "requires running WebSocket echo server on localhost:8765"] +#[sqlx::test(fixtures("base"))] +async fn test_websocket_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let script_path = "f/test/ws_e2e_handler"; + insert_test_script(&db, script_path).await?; + + sqlx::query!( + r#" + INSERT INTO websocket_trigger ( + path, url, script_path, is_flow, workspace_id, + edited_by, email, initial_messages + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + "f/test/ws_e2e_trigger", + "ws://localhost:8765", + script_path, + false, + "test-workspace", + "test-user", + "test@windmill.dev", + &[json!({"type": "RawMessage", "content": "hello from e2e test"})] as &[serde_json::Value], + ) + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + + // The WebSocket trigger connects to the server and sends initial_messages, + // and each received message triggers a job. + let job = poll_for_trigger_job(&db, script_path, "websocket", Duration::from_secs(30)).await?; + assert!(job.args.is_some(), "job should have args"); + + Ok(()) +} + +// ============================================================================ +// Postgres Trigger E2E +// ============================================================================ + +/// End-to-end test for Postgres trigger (logical replication). +/// +/// Requires PostgreSQL with `wal_level=logical`. Setup: +/// ```bash +/// ./tests/fixtures/start_postgres_replication.sh +/// ``` +/// (The script checks wal_level and creates the table/publication/slot in the +/// main DB. This test re-creates them in its isolated sqlx::test database.) +/// +/// Run: +/// ```bash +/// cargo test --test trigger_e2e test_postgres_e2e --features postgres_trigger \ +/// -- --ignored --nocapture +/// ``` +#[ignore = "requires PostgreSQL with wal_level=logical"] +#[sqlx::test(fixtures("base"))] +async fn test_postgres_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let script_path = "f/test/pg_e2e_handler"; + insert_test_script(&db, script_path).await?; + + // Create the tracked table + publication + replication slot inside the + // isolated test database (sqlx::test gives us a fresh DB each run). + // Replication slots are server-wide so we use a random suffix. + let suffix: u32 = rand::random(); + let slot_name = format!("test_e2e_slot_{suffix}"); + let pub_name = format!("test_e2e_pub_{suffix}"); + + sqlx::query("CREATE TABLE test_trigger_table (id serial PRIMARY KEY, data text)") + .execute(&db) + .await?; + sqlx::query(&format!("CREATE PUBLICATION {pub_name} FOR TABLE test_trigger_table")) + .execute(&db) + .await?; + sqlx::query(&format!( + "SELECT pg_create_logical_replication_slot('{slot_name}', 'pgoutput')" + )) + .execute(&db) + .await?; + + // Extract the test DB name from the pool so the resource points here, + // not at the main windmill database. + let test_db_name: String = + sqlx::query_scalar("SELECT current_database()") + .fetch_one(&db) + .await?; + + insert_resource( + &db, + "u/test-user/pg_res", + "postgresql", + json!({ + "user": "postgres", + "password": "changeme", + "host": "localhost", + "port": 5432, + "dbname": test_db_name, + "sslmode": "disable" + }), + ) + .await?; + + sqlx::query( + r#" + INSERT INTO postgres_trigger ( + path, script_path, is_flow, workspace_id, edited_by, email, + postgres_resource_path, replication_slot_name, publication_name + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + ) + .bind("f/test/pg_e2e_trigger") + .bind(script_path) + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .bind("u/test-user/pg_res") + .bind(&slot_name) + .bind(&pub_name) + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + tokio::time::sleep(Duration::from_secs(3)).await; + + // Insert a row into the tracked table to trigger a change event + sqlx::query("INSERT INTO test_trigger_table (data) VALUES ('e2e test data')") + .execute(&db) + .await?; + + let job = poll_for_trigger_job(&db, script_path, "postgres", Duration::from_secs(30)).await?; + assert!(job.args.is_some(), "job should have args"); + + Ok(()) +} + +// ============================================================================ +// Kafka Trigger E2E (Enterprise) +// ============================================================================ + +/// End-to-end test for Kafka trigger (Enterprise only). +/// +/// Requires a running Kafka broker with the test topic. Setup: +/// ```bash +/// ./tests/fixtures/start_kafka.sh +/// ``` +/// +/// Run: +/// ```bash +/// cargo test --test trigger_e2e test_kafka_e2e \ +/// --features kafka,enterprise,private -- --ignored --nocapture +/// ``` +#[cfg(all(feature = "enterprise", feature = "private"))] +#[ignore = "requires running Kafka broker on localhost:9092"] +#[sqlx::test(fixtures("base"))] +async fn test_kafka_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let script_path = "f/test/kafka_e2e_handler"; + insert_test_script(&db, script_path).await?; + + insert_resource( + &db, + "u/test-user/kafka_res", + "kafka", + json!({ + "brokers": ["localhost:9092"], + "security": { "label": "PLAINTEXT" } + }), + ) + .await?; + + sqlx::query!( + r#" + INSERT INTO kafka_trigger ( + path, kafka_resource_path, topics, group_id, + script_path, is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "f/test/kafka_e2e_trigger", + "u/test-user/kafka_res", + &["windmill-e2e-test"] as &[&str], + "windmill-e2e-test-group", + script_path, + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + tokio::time::sleep(Duration::from_secs(5)).await; + + // Produce messages using rdkafka. The consumer starts with auto.offset.reset=latest + // and needs time for group rebalance, so we send repeatedly until a job appears. + use rdkafka::config::ClientConfig; + use rdkafka::producer::{FutureProducer, FutureRecord}; + + let producer: FutureProducer = ClientConfig::new() + .set("bootstrap.servers", "localhost:9092") + .create()?; + + let db2 = db.clone(); + let produce_handle = tokio::spawn(async move { + for _ in 0..30 { + let _ = producer + .send( + FutureRecord::to("windmill-e2e-test") + .payload("hello from kafka e2e test") + .key("test-key"), + Duration::from_secs(5), + ) + .await; + tokio::time::sleep(Duration::from_secs(1)).await; + } + }); + + let job = poll_for_trigger_job(&db2, script_path, "kafka", Duration::from_secs(30)).await?; + produce_handle.abort(); + assert!(job.args.is_some(), "job should have args"); + + Ok(()) +} + +// ============================================================================ +// NATS Trigger E2E (Enterprise) +// ============================================================================ + +/// End-to-end test for NATS trigger (Enterprise only). +/// +/// Requires a running NATS server. Setup: +/// ```bash +/// ./tests/fixtures/start_nats.sh +/// ``` +/// +/// Run: +/// ```bash +/// cargo test --test trigger_e2e test_nats_e2e \ +/// --features nats,enterprise,private -- --ignored --nocapture +/// ``` +#[cfg(all(feature = "enterprise", feature = "private"))] +#[ignore = "requires running NATS server on localhost:4222"] +#[sqlx::test(fixtures("base"))] +async fn test_nats_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let script_path = "f/test/nats_e2e_handler"; + insert_test_script(&db, script_path).await?; + + insert_resource( + &db, + "u/test-user/nats_res", + "nats", + json!({ + "servers": ["nats://localhost:4222"], + "auth": { "label": "NO_AUTH" }, + "require_tls": false + }), + ) + .await?; + + sqlx::query!( + r#" + INSERT INTO nats_trigger ( + path, nats_resource_path, subjects, script_path, + is_flow, workspace_id, edited_by, email, use_jetstream + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "f/test/nats_e2e_trigger", + "u/test-user/nats_res", + &["windmill.e2e.test"] as &[&str], + script_path, + false, + "test-workspace", + "test-user", + "test@windmill.dev", + false, + ) + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + tokio::time::sleep(Duration::from_secs(3)).await; + + // Publish a message using async-nats + let nats_client = async_nats::connect("localhost:4222").await?; + nats_client + .publish("windmill.e2e.test", "hello from nats e2e test".into()) + .await?; + nats_client.flush().await?; + + let job = poll_for_trigger_job(&db, script_path, "nats", Duration::from_secs(30)).await?; + assert!(job.args.is_some(), "job should have args"); + + Ok(()) +} + +// ============================================================================ +// SQS Trigger E2E (Enterprise) +// ============================================================================ + +/// End-to-end test for SQS trigger (Enterprise only). +/// +/// Requires LocalStack with the test queue. Setup: +/// ```bash +/// ./tests/fixtures/start_sqs.sh +/// ``` +/// +/// Run: +/// ```bash +/// AWS_ENDPOINT_URL=http://localhost:4566 \ +/// cargo test --test trigger_e2e test_sqs_e2e \ +/// --features sqs_trigger,enterprise,private -- --ignored --nocapture +/// ``` +#[cfg(all(feature = "enterprise", feature = "private"))] +#[ignore = "requires LocalStack SQS on localhost:4566"] +#[sqlx::test(fixtures("base"))] +async fn test_sqs_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // The SQS listener uses aws_config which respects AWS_ENDPOINT_URL for LocalStack. + std::env::set_var("AWS_ENDPOINT_URL", "http://localhost:4566"); + + let script_path = "f/test/sqs_e2e_handler"; + insert_test_script(&db, script_path).await?; + + insert_resource( + &db, + "u/test-user/aws_res", + "aws", + json!({ + "awsAccessKeyId": "test", + "awsSecretAccessKey": "test", + "region": "us-east-1" + }), + ) + .await?; + + sqlx::query!( + r#" + INSERT INTO sqs_trigger ( + path, queue_url, aws_resource_path, script_path, + is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + "f/test/sqs_e2e_trigger", + "http://localhost:4566/000000000000/windmill-e2e-test", + "u/test-user/aws_res", + script_path, + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + tokio::time::sleep(Duration::from_secs(3)).await; + + // Send a message using aws-sdk-sqs + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url("http://localhost:4566") + .region(aws_config::Region::new("us-east-1")) + .credentials_provider(aws_credential_types::Credentials::new( + "test", "test", None, None, "test", + )) + .load() + .await; + let sqs_client = aws_sdk_sqs::Client::new(&config); + + sqs_client + .send_message() + .queue_url("http://localhost:4566/000000000000/windmill-e2e-test") + .message_body("hello from sqs e2e test") + .send() + .await?; + + let job = poll_for_trigger_job(&db, script_path, "sqs", Duration::from_secs(30)).await?; + assert!(job.args.is_some(), "job should have args"); + + Ok(()) +} + +// ============================================================================ +// GCP Pub/Sub Trigger E2E (Enterprise) +// ============================================================================ + +/// End-to-end test for GCP Pub/Sub trigger (Enterprise only). +/// +/// Requires the GCP Pub/Sub emulator with test topic/subscription. Setup: +/// ```bash +/// ./tests/fixtures/start_gcp_pubsub.sh +/// ``` +/// +/// Run: +/// ```bash +/// PUBSUB_EMULATOR_HOST=localhost:8085 \ +/// cargo test --test trigger_e2e test_gcp_e2e \ +/// --features gcp_trigger,enterprise,private -- --ignored --nocapture +/// ``` +#[cfg(all(feature = "enterprise", feature = "private"))] +#[ignore = "requires GCP Pub/Sub emulator on localhost:8085"] +#[sqlx::test(fixtures("base"))] +async fn test_gcp_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let script_path = "f/test/gcp_e2e_handler"; + insert_test_script(&db, script_path).await?; + + // The GCP emulator doesn't require real credentials, but the resource + // row must still exist for the listener to resolve it. + // The private_key must use literal \n (backslash-n) as in real GCP service + // account JSON files. The trigger code re-parses it through serde_json to + // convert those escape sequences to actual newlines. + insert_resource( + &db, + "u/test-user/gcp_res", + "google", + json!({ + "project_id": "test-project", + "private_key_id": "test", + "private_key": "-----BEGIN RSA PRIVATE KEY-----\\nMIIBogIBAAJBALRiMLAH\\n-----END RSA PRIVATE KEY-----\\n", + "client_email": "test@test-project.iam.gserviceaccount.com", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs" + }), + ) + .await?; + + sqlx::query( + r#" + INSERT INTO gcp_trigger ( + path, gcp_resource_path, topic_id, subscription_id, + delivery_type, subscription_mode, script_path, is_flow, + workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5::delivery_mode, $6::gcp_subscription_mode, $7, $8, $9, $10, $11) + "#, + ) + .bind("f/test/gcp_e2e_trigger") + .bind("u/test-user/gcp_res") + .bind("windmill-e2e-test") + .bind("windmill-e2e-sub") + .bind("pull") + .bind("existing") + .bind(script_path) + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + let _server = ApiServer::start_with_listeners(db.clone()).await?; + tokio::time::sleep(Duration::from_secs(3)).await; + + // Publish a message to the emulator via HTTP + let client = reqwest::Client::new(); + let emulator_host = + std::env::var("PUBSUB_EMULATOR_HOST").unwrap_or_else(|_| "localhost:8085".to_string()); + // The google-cloud-pubsub crate uses "local-project" as the default project ID + // when PUBSUB_EMULATOR_HOST is set, so we must publish to that project's topic. + let publish_url = format!( + "http://{}/v1/projects/local-project/topics/windmill-e2e-test:publish", + emulator_host + ); + + let message_data = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + "hello from gcp e2e test", + ); + client + .post(&publish_url) + .json(&json!({ + "messages": [{ "data": message_data }] + })) + .send() + .await?; + + let job = poll_for_trigger_job(&db, script_path, "gcp", Duration::from_secs(30)).await?; + assert!(job.args.is_some(), "job should have args"); + + Ok(()) +} diff --git a/backend/tests/triggers.rs b/backend/tests/triggers.rs new file mode 100644 index 0000000000..ff7d34decd --- /dev/null +++ b/backend/tests/triggers.rs @@ -0,0 +1,1702 @@ +/*! + * Integration tests for the trigger system (captures, HTTP triggers, trigger configs). + * + * These tests verify: + * 1. Capture config CRUD (create/ping/list/delete via API) + * 2. Capture payload insertion and retrieval + * 3. HTTP trigger CRUD and route matching + * 4. All trigger types DB schema validation + */ + +use serde::Deserialize; +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +// ============================================================================ +// Capture Config Tests (direct DB) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_capture_config_insert_and_query(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email) + VALUES ($1, $2, $3, $4::trigger_kind, $5, $6) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let config = sqlx::query!( + r#" + SELECT path, owner, email, trigger_kind AS "trigger_kind: String" + FROM capture_config + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/script", + ) + .fetch_one(&db) + .await?; + + assert_eq!(config.path, "f/test/script"); + assert_eq!(config.owner, "test-user"); + assert_eq!(config.email, "test@windmill.dev"); + assert_eq!(config.trigger_kind, "webhook"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_config_upsert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email) + VALUES ($1, $2, $3, $4::trigger_kind, $5, $6) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + sqlx::query!( + r#" + INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email) + VALUES ($1, $2, $3, $4::trigger_kind, $5, $6) + ON CONFLICT (workspace_id, path, is_flow, trigger_kind) + DO UPDATE SET owner = $5, email = $6, server_id = NULL, error = NULL + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + "new-owner", + "new@windmill.dev", + ) + .execute(&db) + .await?; + + let config = sqlx::query!( + "SELECT owner, email FROM capture_config WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/script", + ) + .fetch_one(&db) + .await?; + + assert_eq!(config.owner, "new-owner"); + assert_eq!(config.email, "new@windmill.dev"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_config_ping_updates_timestamp(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email) + VALUES ($1, $2, $3, $4::trigger_kind, $5, $6) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let before = sqlx::query!( + "SELECT last_client_ping FROM capture_config WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/script", + ) + .fetch_one(&db) + .await?; + assert!(before.last_client_ping.is_none()); + + sqlx::query!( + r#" + UPDATE capture_config SET last_client_ping = NOW() + WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4::trigger_kind + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + ) + .execute(&db) + .await?; + + let after = sqlx::query!( + "SELECT last_client_ping FROM capture_config WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/script", + ) + .fetch_one(&db) + .await?; + assert!(after.last_client_ping.is_some()); + + Ok(()) +} + +// ============================================================================ +// Capture Payload Tests (direct DB) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_capture_insert_and_list(db: Pool) -> anyhow::Result<()> { + for i in 0..2 { + sqlx::query!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"key": format!("value{}", i)}), + json!({"pre": format!("args{}", i)}), + "test-user", + ) + .execute(&db) + .await?; + } + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM capture WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/script", + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(2)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_delete(db: Pool) -> anyhow::Result<()> { + let id = sqlx::query_scalar!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + RETURNING id + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"key": "value"}), + json!({"pre": "args"}), + "test-user", + ) + .fetch_one(&db) + .await?; + + sqlx::query!("DELETE FROM capture WHERE id = $1", id) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM capture WHERE id = $1", + id, + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_filter_by_trigger_kind(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"source": "webhook"}), + json!({}), + "test-user", + ) + .execute(&db) + .await?; + + sqlx::query!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + "#, + "test-workspace", + "f/test/script", + false, + "email" as _, + json!({"source": "email"}), + json!({}), + "test-user", + ) + .execute(&db) + .await?; + + let webhook_count = sqlx::query_scalar!( + r#" + SELECT COUNT(*) FROM capture + WHERE workspace_id = $1 AND path = $2 AND trigger_kind = $3::trigger_kind + "#, + "test-workspace", + "f/test/script", + "webhook" as _, + ) + .fetch_one(&db) + .await?; + + assert_eq!(webhook_count, Some(1)); + + let email_count = sqlx::query_scalar!( + r#" + SELECT COUNT(*) FROM capture + WHERE workspace_id = $1 AND path = $2 AND trigger_kind = $3::trigger_kind + "#, + "test-workspace", + "f/test/script", + "email" as _, + ) + .fetch_one(&db) + .await?; + + assert_eq!(email_count, Some(1)); + + Ok(()) +} + +// ============================================================================ +// Capture API Tests (via HTTP) +// ============================================================================ + +#[derive(Debug, Deserialize)] +struct CaptureResponse { + id: i64, + #[allow(dead_code)] + trigger_kind: String, + main_args: serde_json::Value, +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_api_set_config_and_list(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + let response = client + .client() + .post(format!( + "{}/w/test-workspace/capture/set_config", + client.baseurl() + )) + .json(&json!({ + "path": "f/test/my_script", + "is_flow": false, + "trigger_kind": "webhook", + })) + .send() + .await?; + + assert!( + response.status().is_success(), + "set_config should succeed, got: {}", + response.status() + ); + + let config = sqlx::query!( + "SELECT owner FROM capture_config WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/my_script", + ) + .fetch_one(&db) + .await?; + + assert_eq!(config.owner, "test-user"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_api_list_captures(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + for i in 0..3 { + sqlx::query!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"index": i}), + json!({}), + "test-user", + ) + .execute(&db) + .await?; + } + + let response = client + .client() + .get(format!( + "{}/w/test-workspace/capture/list/script/f/test/script", + client.baseurl() + )) + .send() + .await?; + + assert!(response.status().is_success(), "list captures should succeed"); + + let captures: Vec = response.json().await?; + assert_eq!(captures.len(), 3); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_api_get_single(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + let id = sqlx::query_scalar!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + RETURNING id + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"hello": "world"}), + json!({}), + "test-user", + ) + .fetch_one(&db) + .await?; + + let response = client + .client() + .get(format!( + "{}/w/test-workspace/capture/{}", + client.baseurl(), + id + )) + .send() + .await?; + + assert!(response.status().is_success(), "get capture should succeed"); + + let capture: CaptureResponse = response.json().await?; + assert_eq!(capture.id, id); + assert_eq!(capture.main_args["hello"], "world"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_api_delete(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + let id = sqlx::query_scalar!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + RETURNING id + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"data": "to_delete"}), + json!({}), + "test-user", + ) + .fetch_one(&db) + .await?; + + let response = client + .client() + .delete(format!( + "{}/w/test-workspace/capture/{}", + client.baseurl(), + id + )) + .send() + .await?; + + assert!(response.status().is_success(), "delete should succeed"); + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM capture WHERE id = $1", + id, + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_capture_api_pagination(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + for i in 0..5 { + sqlx::query!( + r#" + INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by) + VALUES ($1, $2, $3, $4::trigger_kind, $5::jsonb, $6::jsonb, $7) + "#, + "test-workspace", + "f/test/script", + false, + "webhook" as _, + json!({"index": i}), + json!({}), + "test-user", + ) + .execute(&db) + .await?; + } + + let response = client + .client() + .get(format!( + "{}/w/test-workspace/capture/list/script/f/test/script?per_page=2", + client.baseurl() + )) + .send() + .await?; + + assert!(response.status().is_success()); + let limited: Vec = response.json().await?; + assert_eq!(limited.len(), 2, "per_page=2 should limit to 2 results"); + + let response = client + .client() + .get(format!( + "{}/w/test-workspace/capture/list/script/f/test/script", + client.baseurl() + )) + .send() + .await?; + + assert!(response.status().is_success()); + let all: Vec = response.json().await?; + assert_eq!(all.len(), 5, "without limit should return all 5"); + + Ok(()) +} + +// ============================================================================ +// HTTP Trigger Tests (direct DB) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_http_trigger_insert_and_query(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, + workspace_id, edited_by, email, http_method, + authentication_method, is_static_website, workspaced_route, + wrap_body, raw_string + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method, + $10::authentication_method, $11, $12, $13, $14) + "#, + "f/test/http_trigger", + "api/v1/users/:id", + "api/v1/users", + "f/test/handler_script", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "post" as _, + "none" as _, + false, + false, + false, + false, + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT + path, route_path, script_path, + http_method AS "http_method: String", + authentication_method AS "authentication_method: String" + FROM http_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/http_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.path, "f/test/http_trigger"); + assert_eq!(trigger.route_path, "api/v1/users/:id"); + assert_eq!(trigger.script_path, "f/test/handler_script"); + assert_eq!(trigger.http_method, "post"); + assert_eq!(trigger.authentication_method, "none"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_http_trigger_multiple_methods(db: Pool) -> anyhow::Result<()> { + let methods = ["get", "post", "put", "delete", "patch"]; + + for method in &methods { + sqlx::query!( + r#" + INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, + workspace_id, edited_by, email, http_method, + authentication_method, is_static_website, workspaced_route, + wrap_body, raw_string + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method, + $10::authentication_method, $11, $12, $13, $14) + "#, + format!("f/test/trigger_{}", method), + format!("api/{}", method), + format!("api/{}", method), + "f/test/handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + *method as _, + "none" as _, + false, + false, + false, + false, + ) + .execute(&db) + .await?; + } + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM http_trigger WHERE workspace_id = $1", + "test-workspace", + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(methods.len() as i64)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_http_trigger_authentication_methods(db: Pool) -> anyhow::Result<()> { + let auth_methods = ["none", "windmill", "api_key", "basic_http", "signature"]; + + for (i, auth) in auth_methods.iter().enumerate() { + sqlx::query!( + r#" + INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, + workspace_id, edited_by, email, http_method, + authentication_method, is_static_website, workspaced_route, + wrap_body, raw_string + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method, + $10::authentication_method, $11, $12, $13, $14) + "#, + format!("f/test/trigger_{}", i), + format!("api/{}", i), + format!("api/{}", i), + "f/test/handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "get" as _, + *auth as _, + false, + false, + false, + auth == &"signature", + ) + .execute(&db) + .await?; + } + + let sig_count = sqlx::query_scalar!( + r#" + SELECT COUNT(*) FROM http_trigger + WHERE workspace_id = $1 AND authentication_method = $2::authentication_method + "#, + "test-workspace", + "signature" as _, + ) + .fetch_one(&db) + .await?; + + assert_eq!(sig_count, Some(1)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_http_trigger_update(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, + workspace_id, edited_by, email, http_method, + authentication_method, is_static_website, workspaced_route, + wrap_body, raw_string + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method, + $10::authentication_method, $11, $12, $13, $14) + "#, + "f/test/trigger", + "api/v1/old", + "api/v1/old", + "f/test/old_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "get" as _, + "none" as _, + false, + false, + false, + false, + ) + .execute(&db) + .await?; + + sqlx::query!( + "UPDATE http_trigger SET script_path = $1 WHERE workspace_id = $2 AND path = $3", + "f/test/new_handler", + "test-workspace", + "f/test/trigger", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + "SELECT script_path FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.script_path, "f/test/new_handler"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_http_trigger_delete(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, + workspace_id, edited_by, email, http_method, + authentication_method, is_static_website, workspaced_route, + wrap_body, raw_string + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method, + $10::authentication_method, $11, $12, $13, $14) + "#, + "f/test/to_delete", + "api/delete_me", + "api/delete_me", + "f/test/handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "get" as _, + "none" as _, + false, + false, + false, + false, + ) + .execute(&db) + .await?; + + sqlx::query!( + "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/to_delete", + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/to_delete", + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_http_trigger_mode_filtering(db: Pool) -> anyhow::Result<()> { + let modes = ["enabled", "disabled", "suspended"]; + + for (i, mode) in modes.iter().enumerate() { + sqlx::query!( + r#" + INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, + workspace_id, edited_by, email, http_method, + authentication_method, is_static_website, workspaced_route, + wrap_body, raw_string, mode + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::http_method, + $10::authentication_method, $11, $12, $13, $14, $15::trigger_mode) + "#, + format!("f/test/trigger_{}", i), + format!("api/{}", i), + format!("api/{}", i), + "f/test/handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "get" as _, + "none" as _, + false, + false, + false, + false, + *mode as _, + ) + .execute(&db) + .await?; + } + + // Query for active triggers (enabled or suspended, matching refresh_routers logic) + let active_count = sqlx::query_scalar!( + r#" + SELECT COUNT(*) FROM http_trigger + WHERE workspace_id = $1 + AND (mode = 'enabled'::trigger_mode OR mode = 'suspended'::trigger_mode) + "#, + "test-workspace", + ) + .fetch_one(&db) + .await?; + + assert_eq!(active_count, Some(2)); + + Ok(()) +} + +// ============================================================================ +// Other Trigger Types Tests (DB schema validation) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_websocket_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO websocket_trigger ( + path, url, script_path, is_flow, workspace_id, + edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) + "#, + "f/test/ws_trigger", + "wss://example.com/feed", + "f/test/ws_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT url, script_path, mode AS "mode: String" + FROM websocket_trigger WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/ws_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.url, "wss://example.com/feed"); + assert_eq!(trigger.script_path, "f/test/ws_handler"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_kafka_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO kafka_trigger ( + path, kafka_resource_path, topics, group_id, script_path, + is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "f/test/kafka_trigger", + "u/admin/kafka_resource", + &["topic-a", "topic-b"] as &[&str], + "my-consumer-group", + "f/test/kafka_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT kafka_resource_path, topics, group_id, mode AS "mode: String" + FROM kafka_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/kafka_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.kafka_resource_path, "u/admin/kafka_resource"); + assert_eq!(trigger.topics, vec!["topic-a", "topic-b"]); + assert_eq!(trigger.group_id, "my-consumer-group"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_postgres_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO postgres_trigger ( + path, script_path, is_flow, workspace_id, edited_by, email, + postgres_resource_path, replication_slot_name, publication_name + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "f/test/pg_trigger", + "f/test/pg_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "u/admin/pg_resource", + "test_slot", + "test_publication", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT postgres_resource_path, replication_slot_name, publication_name, mode AS "mode: String" + FROM postgres_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/pg_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.postgres_resource_path, "u/admin/pg_resource"); + assert_eq!(trigger.replication_slot_name, "test_slot"); + assert_eq!(trigger.publication_name, "test_publication"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_nats_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO nats_trigger ( + path, nats_resource_path, subjects, script_path, + is_flow, workspace_id, edited_by, email, use_jetstream + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "f/test/nats_trigger", + "u/admin/nats_resource", + &["orders.>", "payments.*"] as &[&str], + "f/test/nats_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + false, + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT nats_resource_path, subjects, use_jetstream, mode AS "mode: String" + FROM nats_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/nats_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.nats_resource_path, "u/admin/nats_resource"); + assert_eq!(trigger.subjects, vec!["orders.>", "payments.*"]); + assert_eq!(trigger.use_jetstream, false); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_sqs_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO sqs_trigger ( + path, queue_url, aws_resource_path, script_path, + is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + "f/test/sqs_trigger", + "https://sqs.us-east-1.amazonaws.com/123456789/my-queue", + "u/admin/aws_resource", + "f/test/sqs_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT queue_url, aws_resource_path, + aws_auth_resource_type AS "aws_auth_resource_type: String", + mode AS "mode: String" + FROM sqs_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/sqs_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!( + trigger.queue_url, + "https://sqs.us-east-1.amazonaws.com/123456789/my-queue" + ); + assert_eq!(trigger.aws_resource_path, "u/admin/aws_resource"); + assert_eq!(trigger.aws_auth_resource_type, "credentials"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +// ============================================================================ +// Cross-trigger tests +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_server_state_tracking(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO websocket_trigger ( + path, url, script_path, is_flow, workspace_id, + edited_by, email, server_id, error + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "f/test/ws_with_error", + "wss://example.com/feed", + "f/test/handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + "server-abc-123", + "connection refused", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + "SELECT server_id, error FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/ws_with_error", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.server_id, Some("server-abc-123".to_string())); + assert_eq!(trigger.error, Some("connection refused".to_string())); + + sqlx::query!( + "UPDATE websocket_trigger SET error = NULL WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/ws_with_error", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + "SELECT error FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/ws_with_error", + ) + .fetch_one(&db) + .await?; + + assert!(trigger.error.is_none()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_trigger_mode_filtering(db: Pool) -> anyhow::Result<()> { + let modes = ["enabled", "disabled", "enabled"]; + + for (i, mode) in modes.iter().enumerate() { + sqlx::query!( + r#" + INSERT INTO websocket_trigger ( + path, url, script_path, is_flow, workspace_id, + edited_by, email, mode + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8::trigger_mode) + "#, + format!("f/test/ws_trigger_{}", i), + "wss://example.com", + "f/test/handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + *mode as _, + ) + .execute(&db) + .await?; + } + + let enabled_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM websocket_trigger WHERE workspace_id = $1 AND mode = 'enabled'::trigger_mode", + "test-workspace", + ) + .fetch_one(&db) + .await?; + + assert_eq!(enabled_count, Some(2)); + + let disabled_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM websocket_trigger WHERE workspace_id = $1 AND mode = 'disabled'::trigger_mode", + "test-workspace", + ) + .fetch_one(&db) + .await?; + + assert_eq!(disabled_count, Some(1)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_multiple_capture_configs_per_path(db: Pool) -> anyhow::Result<()> { + let trigger_kinds = ["webhook", "email", "kafka"]; + + for kind in &trigger_kinds { + sqlx::query!( + r#" + INSERT INTO capture_config (workspace_id, path, is_flow, trigger_kind, owner, email) + VALUES ($1, $2, $3, $4::trigger_kind, $5, $6) + "#, + "test-workspace", + "f/test/multi_trigger_script", + false, + *kind as _, + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + } + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM capture_config WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/multi_trigger_script", + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(3)); + + Ok(()) +} + +// ============================================================================ +// Schedule Tests (DB-level) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_schedule_insert_and_query(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO schedule ( + workspace_id, path, edited_by, schedule, enabled, + script_path, is_flow, email, timezone + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + "#, + "test-workspace", + "f/test/my_schedule", + "test-user", + "0 */5 * * *", + true, + "f/test/scheduled_script", + false, + "test@windmill.dev", + "UTC", + ) + .execute(&db) + .await?; + + let schedule = sqlx::query!( + r#" + SELECT path, schedule, enabled, script_path, timezone + FROM schedule + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/my_schedule", + ) + .fetch_one(&db) + .await?; + + assert_eq!(schedule.path, "f/test/my_schedule"); + assert_eq!(schedule.schedule, "0 */5 * * *"); + assert_eq!(schedule.enabled, true); + assert_eq!(schedule.script_path, "f/test/scheduled_script"); + assert_eq!(schedule.timezone, "UTC"); + + Ok(()) +} + +// ============================================================================ +// MQTT Trigger Tests (DB-level) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_mqtt_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO mqtt_trigger ( + path, mqtt_resource_path, subscribe_topics, client_version, + script_path, is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, ARRAY[$3::jsonb], $4::mqtt_client_version, $5, $6, $7, $8, $9) + "#, + ) + .bind("f/test/mqtt_trigger") + .bind("u/admin/mqtt_resource") + .bind(json!({"topic": "test/+", "qos": "qos1"})) + .bind("v5") + .bind("f/test/mqtt_handler") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT mqtt_resource_path, client_version AS "client_version: String", + script_path, mode AS "mode: String" + FROM mqtt_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/mqtt_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.mqtt_resource_path, "u/admin/mqtt_resource"); + assert_eq!(trigger.client_version, "v5"); + assert_eq!(trigger.script_path, "f/test/mqtt_handler"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_mqtt_trigger_update(db: Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO mqtt_trigger ( + path, mqtt_resource_path, subscribe_topics, client_version, + script_path, is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, ARRAY[$3::jsonb], $4::mqtt_client_version, $5, $6, $7, $8, $9) + "#, + ) + .bind("f/test/mqtt_trigger") + .bind("u/admin/mqtt_resource") + .bind(json!({"topic": "test/+", "qos": "qos1"})) + .bind("v5") + .bind("f/test/old_handler") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + sqlx::query!( + "UPDATE mqtt_trigger SET script_path = $1 WHERE workspace_id = $2 AND path = $3", + "f/test/new_handler", + "test-workspace", + "f/test/mqtt_trigger", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + "SELECT script_path FROM mqtt_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/mqtt_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.script_path, "f/test/new_handler"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_mqtt_trigger_delete(db: Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO mqtt_trigger ( + path, mqtt_resource_path, subscribe_topics, client_version, + script_path, is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, ARRAY[$3::jsonb], $4::mqtt_client_version, $5, $6, $7, $8, $9) + "#, + ) + .bind("f/test/mqtt_trigger") + .bind("u/admin/mqtt_resource") + .bind(json!({"topic": "test/+", "qos": "qos1"})) + .bind("v5") + .bind("f/test/mqtt_handler") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + sqlx::query!( + "DELETE FROM mqtt_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/mqtt_trigger", + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM mqtt_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/mqtt_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(0)); + + Ok(()) +} + +// ============================================================================ +// GCP Trigger Tests (DB-level) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_gcp_trigger_insert_pull(db: Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO gcp_trigger ( + path, gcp_resource_path, topic_id, subscription_id, + delivery_type, subscription_mode, script_path, is_flow, + workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5::delivery_mode, $6::gcp_subscription_mode, $7, $8, $9, $10, $11) + "#, + ) + .bind("f/test/gcp_trigger_pull") + .bind("u/admin/gcp_resource") + .bind("my-topic") + .bind("my-subscription") + .bind("pull") + .bind("create_update") + .bind("f/test/gcp_handler") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT gcp_resource_path, topic_id, subscription_id, + delivery_type AS "delivery_type: String", + subscription_mode AS "subscription_mode: String", + mode AS "mode: String" + FROM gcp_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/gcp_trigger_pull", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.gcp_resource_path, "u/admin/gcp_resource"); + assert_eq!(trigger.topic_id, "my-topic"); + assert_eq!(trigger.subscription_id, "my-subscription"); + assert_eq!(trigger.delivery_type, "pull"); + assert_eq!(trigger.subscription_mode, "create_update"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_gcp_trigger_insert_push(db: Pool) -> anyhow::Result<()> { + sqlx::query( + r#" + INSERT INTO gcp_trigger ( + path, gcp_resource_path, topic_id, subscription_id, + delivery_type, delivery_config, subscription_mode, + script_path, is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5::delivery_mode, $6::jsonb, $7::gcp_subscription_mode, $8, $9, $10, $11, $12) + "#, + ) + .bind("f/test/gcp_trigger_push") + .bind("u/admin/gcp_resource") + .bind("my-topic") + .bind("my-push-subscription") + .bind("push") + .bind(json!({"endpoint": "https://example.com/push"})) + .bind("create_update") + .bind("f/test/gcp_handler") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT delivery_type AS "delivery_type: String", + delivery_config + FROM gcp_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/gcp_trigger_push", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.delivery_type, "push"); + assert!(trigger.delivery_config.is_some()); + assert_eq!( + trigger.delivery_config.unwrap()["endpoint"], + "https://example.com/push" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_gcp_trigger_unique_constraint(db: Pool) -> anyhow::Result<()> { + let insert_query = r#" + INSERT INTO gcp_trigger ( + path, gcp_resource_path, topic_id, subscription_id, + delivery_type, subscription_mode, script_path, is_flow, + workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5::delivery_mode, $6::gcp_subscription_mode, $7, $8, $9, $10, $11) + "#; + + sqlx::query(insert_query) + .bind("f/test/gcp_trigger_1") + .bind("u/admin/gcp_resource") + .bind("my-topic") + .bind("shared-subscription") + .bind("pull") + .bind("create_update") + .bind("f/test/gcp_handler") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await?; + + // Inserting a second trigger with same (subscription_id, gcp_resource_path, workspace_id) should fail + let result = sqlx::query(insert_query) + .bind("f/test/gcp_trigger_2") + .bind("u/admin/gcp_resource") + .bind("my-topic") + .bind("shared-subscription") + .bind("pull") + .bind("create_update") + .bind("f/test/gcp_handler_2") + .bind(false) + .bind("test-workspace") + .bind("test-user") + .bind("test@windmill.dev") + .execute(&db) + .await; + + assert!( + result.is_err(), + "should fail due to unique constraint on (subscription_id, gcp_resource_path, workspace_id)" + ); + + Ok(()) +} + +// ============================================================================ +// Email Trigger Tests (DB-level) +// ============================================================================ + +#[sqlx::test(fixtures("base"))] +async fn test_email_trigger_insert(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO email_trigger ( + path, local_part, workspaced_local_part, script_path, + is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + "f/test/email_trigger", + "support", + true, + "f/test/email_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + r#" + SELECT local_part, workspaced_local_part, script_path, + mode AS "mode: String" + FROM email_trigger + WHERE workspace_id = $1 AND path = $2 + "#, + "test-workspace", + "f/test/email_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.local_part, "support"); + assert_eq!(trigger.workspaced_local_part, true); + assert_eq!(trigger.script_path, "f/test/email_handler"); + assert_eq!(trigger.mode, "enabled"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_email_trigger_update(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO email_trigger ( + path, local_part, workspaced_local_part, script_path, + is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + "f/test/email_trigger", + "support", + true, + "f/test/old_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + sqlx::query!( + "UPDATE email_trigger SET script_path = $1, local_part = $2 WHERE workspace_id = $3 AND path = $4", + "f/test/new_handler", + "billing", + "test-workspace", + "f/test/email_trigger", + ) + .execute(&db) + .await?; + + let trigger = sqlx::query!( + "SELECT script_path, local_part FROM email_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/email_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(trigger.script_path, "f/test/new_handler"); + assert_eq!(trigger.local_part, "billing"); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_email_trigger_delete(db: Pool) -> anyhow::Result<()> { + sqlx::query!( + r#" + INSERT INTO email_trigger ( + path, local_part, workspaced_local_part, script_path, + is_flow, workspace_id, edited_by, email + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + "#, + "f/test/email_trigger", + "support", + true, + "f/test/email_handler", + false, + "test-workspace", + "test-user", + "test@windmill.dev", + ) + .execute(&db) + .await?; + + sqlx::query!( + "DELETE FROM email_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/email_trigger", + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM email_trigger WHERE workspace_id = $1 AND path = $2", + "test-workspace", + "f/test/email_trigger", + ) + .fetch_one(&db) + .await?; + + assert_eq!(count, Some(0)); + + Ok(()) +} diff --git a/backend/tests/users.rs b/backend/tests/users.rs new file mode 100644 index 0000000000..352b3cc5c6 --- /dev/null +++ b/backend/tests/users.rs @@ -0,0 +1,499 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn user_url(port: u16, endpoint: &str, name: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/users/{endpoint}/{name}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +#[sqlx::test(fixtures("base"))] +async fn test_user_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/users"); + + // ===== Global (non-workspace) endpoints ===== + let global_base = format!("http://localhost:{port}/api/users"); + + // --- global whoami --- + let resp = authed(client().get(format!("{global_base}/whoami"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["email"], "test@windmill.dev"); + + // --- get_email --- + let resp = authed(client().get(format!("{global_base}/email"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let email = resp.text().await?; + assert_eq!(email, "test@windmill.dev"); + + // --- exists_email --- + let resp = authed(client().get(format!( + "{global_base}/exists/test@windmill.dev" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed(client().get(format!( + "{global_base}/exists/nonexistent@windmill.dev" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- list_as_super_admin --- + let resp = authed(client().get(format!("{global_base}/list_as_super_admin"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + + // --- tokens/list --- + let resp = authed(client().get(format!("{global_base}/tokens/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- tokens/create --- + let resp = authed(client().post(format!("{global_base}/tokens/create"))) + .json(&json!({"label": "ephemeral-test-token"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + let new_token = resp.text().await?; + assert!(!new_token.is_empty()); + + // --- tokens/delete --- + let token_prefix = &new_token[..std::cmp::min(new_token.len(), 10)]; + let resp = authed(client().delete(format!( + "{global_base}/tokens/delete/{token_prefix}" + ))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "delete token: {}", + resp.text().await? + ); + + // --- list_invites --- + let resp = authed(client().get(format!("{global_base}/list_invites"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- username_info --- + let resp = authed(client().get(format!( + "{global_base}/username_info/test@windmill.dev" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["username"], "test-user"); + + // --- global usage --- + let resp = authed(client().get(format!("{global_base}/usage"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- tutorial_progress (get, then set, then get again) --- + let resp = authed(client().get(format!("{global_base}/tutorial_progress"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await?; + + let resp = authed(client().post(format!("{global_base}/tutorial_progress"))) + .json(&json!({"progress": 42, "skipped_all": false})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "set tutorial_progress: {}", + resp.text().await? + ); + + let resp = authed(client().get(format!("{global_base}/tutorial_progress"))) + .send() + .await + .unwrap(); + let body = resp.json::().await?; + assert_eq!(body["progress"], 42); + + // --- global update user --- + let resp = authed(client().post(format!( + "{global_base}/update/test2@windmill.dev" + ))) + .json(&json!({"name": "Updated Test User 2"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "global update user: {}", + resp.text().await? + ); + + // --- setpassword (EE-gated in OSS) --- + let resp = authed(client().post(format!("{global_base}/setpassword"))) + .json(&json!({"password": "new-test-password-123"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "setpassword: unexpected status {}", + resp.status() + ); + + // --- all_runnables --- + let resp = authed(client().get(format!("{global_base}/all_runnables"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- onboarding (EE-gated in OSS) --- + let resp = authed(client().post(format!("{global_base}/onboarding"))) + .json(&json!({"touch_point": "test", "use_case": "testing"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "onboarding: unexpected status {}", + resp.status() + ); + + // --- decline_invite (no pending invite, but endpoint should handle gracefully) --- + let resp = authed(client().post(format!("{global_base}/decline_invite"))) + .json(&json!({"workspace_id": "nonexistent-ws"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 404, + "decline_invite: unexpected status {}", + resp.status() + ); + + // --- auth: is_first_time_setup (unauthed) --- + let resp = client() + .get(format!("http://localhost:{port}/api/auth/is_first_time_setup")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let is_first = resp.json::().await?; + assert_eq!(is_first, false); + + // --- auth: is_smtp_configured (unauthed) --- + let resp = client() + .get(format!("http://localhost:{port}/api/auth/is_smtp_configured")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await?; + + // --- create user (global, EE-gated in OSS) --- + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ + "email": "newglobaluser@windmill.dev", + "password": "test-password-123", + "super_admin": false, + "name": "New Global User" + })) + .send() + .await + .unwrap(); + let create_status = resp.status(); + assert!( + create_status == 201 || create_status == 500, + "create user: unexpected status {}", + create_status + ); + + if create_status == 201 { + // --- rename user (only if create succeeded / EE) --- + let resp = authed(client().post(format!( + "{global_base}/rename/newglobaluser@windmill.dev" + ))) + .json(&json!({"new_username": "renamed_user"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "rename user: {}", + resp.text().await? + ); + + // --- global delete user --- + let resp = authed(client().delete(format!( + "{global_base}/delete/newglobaluser@windmill.dev" + ))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "global delete user: {}", + resp.text().await? + ); + } + + // ===== Auth (unauthed) endpoints ===== + let auth_base = format!("http://localhost:{port}/api/auth"); + + // --- login (will fail: password hash in fixture is fake) --- + let resp = client() + .post(format!("{auth_base}/login")) + .json(&json!({"email": "test@windmill.dev", "password": "wrong-password"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 400 || resp.status() == 401 || resp.status() == 500, + "login: unexpected status {}", + resp.status() + ); + + // --- logout (POST, with auth token) --- + let resp = authed(client().post(format!("{auth_base}/logout"))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 303, + "logout POST: unexpected status {}", + resp.status() + ); + + // --- logout (GET, with auth token) --- + let resp = authed(client().get(format!("{auth_base}/logout"))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 303, + "logout GET: unexpected status {}", + resp.status() + ); + + // --- request_password_reset (returns 400 if SMTP not configured) --- + let resp = client() + .post(format!("{auth_base}/request_password_reset")) + .json(&json!({"email": "test@windmill.dev"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "request_password_reset: unexpected status {}", + resp.status() + ); + + // --- reset_password (EE-gated, invalid token) --- + let resp = client() + .post(format!("{auth_base}/reset_password")) + .json(&json!({"token": "invalid-token", "new_password": "new-pass"})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 400 || resp.status() == 500, + "reset_password: unexpected status {}", + resp.status() + ); + + // ===== Workspace-scoped endpoints ===== + + // --- whoami --- + let resp = authed(client().get(format!("{base}/whoami"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["username"], "test-user"); + assert_eq!(body["email"], "test@windmill.dev"); + assert_eq!(body["is_admin"], true); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + assert!(list.iter().any(|u| u["username"] == "test-user")); + + // --- list_usernames --- + let resp = authed(client().get(format!("{base}/list_usernames"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let usernames = resp.json::>().await?; + assert!(usernames.contains(&"test-user".to_string())); + + // --- get --- + let resp = authed(client().get(user_url(port, "get", "test-user"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["username"], "test-user"); + + // --- whois --- + let resp = authed(client().get(user_url(port, "whois", "test-user"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["username"], "test-user"); + + // --- username_to_email --- + let resp = authed(client().get(user_url(port, "username_to_email", "test-user"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let email = resp.text().await?; + assert_eq!(email, "test@windmill.dev"); + + // --- exists --- + let resp = authed(client().post(format!("{base}/exists"))) + .json(&json!({"username": "test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed(client().post(format!("{base}/exists"))) + .json(&json!({"username": "nonexistent"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- list_usage --- + let resp = authed(client().get(format!("{base}/list_usage"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- is_owner --- + let resp = authed(client().get(format!( + "http://localhost:{port}/api/w/test-workspace/users/is_owner/u/test-user/test" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- update (make non-admin, then revert) --- + let resp = authed(client().post(user_url(port, "update", "test-user"))) + .json(&json!({"is_admin": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "update user: {}", resp.text().await?); + + // revert back to admin + let resp = authed(client().post(user_url(port, "update", "test-user"))) + .json(&json!({"is_admin": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- delete workspace user (admin deletes test-user-2) --- + let resp = authed(client().delete(user_url(port, "delete", "test-user-2"))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "delete user: {}", + resp.text().await? + ); + + // verify deleted + let resp = authed(client().get(format!("{base}/list_usernames"))) + .send() + .await + .unwrap(); + let usernames = resp.json::>().await?; + assert!(!usernames.contains(&"test-user-2".to_string())); + + // --- leave workspace (test-user-3 leaves voluntarily) --- + let resp = client() + .post(format!("{base}/leave")) + .header("Authorization", "Bearer SECRET_TOKEN_3") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "leave: {}", resp.text().await?); + + // verify left + let resp = authed(client().get(format!("{base}/list_usernames"))) + .send() + .await + .unwrap(); + let usernames = resp.json::>().await?; + assert!(!usernames.contains(&"test-user-3".to_string())); + + Ok(()) +} diff --git a/backend/tests/variables.rs b/backend/tests/variables.rs new file mode 100644 index 0000000000..22892e522a --- /dev/null +++ b/backend/tests/variables.rs @@ -0,0 +1,255 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn variable_url(port: u16, endpoint: &str, path: &str) -> String { + format!("http://localhost:{port}/api/w/test-workspace/variables/{endpoint}/{path}") +} + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +async fn authed_get(port: u16, endpoint: &str, path: &str) -> reqwest::Response { + authed(client().get(variable_url(port, endpoint, path))) + .send() + .await + .unwrap() +} + +#[sqlx::test(fixtures("base", "variables_test"))] +async fn test_variable_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/variables"); + + // --- exists --- + let resp = authed_get(port, "exists", "u/test-user/plain_var").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "exists", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- get (plain) --- + let resp = authed_get(port, "get", "u/test-user/plain_var").await; + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/plain_var"); + assert_eq!(body["value"], "hello world"); + assert_eq!(body["is_secret"], false); + assert_eq!(body["description"], "A plain variable"); + + // --- get (secret, decrypt_secret=false) --- + // fixture secrets are stored as plaintext, so skip decryption + let resp = authed(client().get(format!( + "{}/get/u/test-user/secret_var?decrypt_secret=false", + base + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["path"], "u/test-user/secret_var"); + assert_eq!(body["is_secret"], true); + assert_eq!(body["value"], serde_json::Value::Null); + + // get (secret, include_encrypted=true returns raw stored value) + let resp = authed(client().get(format!( + "{}/get/u/test-user/secret_var?decrypt_secret=false&include_encrypted=true", + base + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["is_secret"], true); + assert!(body["value"].is_string(), "expected encrypted value string"); + + // --- get not found --- + let resp = authed_get(port, "get", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- get_value (plain) --- + let resp = authed_get(port, "get_value", "u/test-user/plain_var").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, "hello world"); + + let resp = authed_get(port, "get_value", "u/test-user/nonexistent").await; + assert_eq!(resp.status(), 404); + + // --- list --- + let resp = authed(client().get(format!("{base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!( + list.len() >= 3, + "expected at least 3 variables from fixture, got {}", + list.len() + ); + assert!(list.iter().any(|v| v["path"] == "u/test-user/plain_var")); + // secrets should have null value in list + let secret = list + .iter() + .find(|v| v["path"] == "u/test-user/secret_var") + .expect("secret_var missing from list"); + assert_eq!(secret["value"], serde_json::Value::Null); + + // list with path_start filter + let resp = authed(client().get(format!( + "{base}/list?path_start=u/test-user/plain" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert_eq!(list.len(), 1); + assert_eq!(list[0]["path"], "u/test-user/plain_var"); + + // --- list_contextual --- + let resp = authed(client().get(format!("{base}/list_contextual"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(!list.is_empty()); + // should contain reserved variable names like WM_TOKEN, WM_WORKSPACE + assert!( + list.iter().any(|v| v["name"] == "WM_WORKSPACE"), + "expected WM_WORKSPACE in contextual variables" + ); + + // --- create (plain) --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/new_var", + "value": "new_value", + "is_secret": false, + "description": "Created in test" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + let resp = authed_get(port, "exists", "u/test-user/new_var").await; + assert_eq!(resp.json::().await?, true); + + let resp = authed_get(port, "get_value", "u/test-user/new_var").await; + assert_eq!(resp.json::().await?, "new_value"); + + // create duplicate -> 400 + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/new_var", + "value": "dup", + "is_secret": false, + "description": "Duplicate" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + // --- create (secret) --- + let resp = authed(client().post(format!("{base}/create"))) + .json(&json!({ + "path": "u/test-user/new_secret", + "value": "my_secret_val", + "is_secret": true, + "description": "A new secret" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + // secret is stored encrypted, get_value should decrypt it + let resp = authed_get(port, "get_value", "u/test-user/new_secret").await; + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, "my_secret_val"); + + // --- update (value) --- + let resp = authed(client().post(variable_url(port, "update", "u/test-user/new_var"))) + .json(&json!({"value": "updated_value"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get_value", "u/test-user/new_var").await; + assert_eq!(resp.json::().await?, "updated_value"); + + // --- update (description) --- + let resp = authed(client().post(variable_url(port, "update", "u/test-user/new_var"))) + .json(&json!({"description": "Updated desc"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get", "u/test-user/new_var").await; + let body = resp.json::().await?; + assert_eq!(body["description"], "Updated desc"); + + // --- delete --- + let resp = authed(client().delete(variable_url(port, "delete", "u/test-user/new_var"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "exists", "u/test-user/new_var").await; + assert_eq!(resp.json::().await?, false); + + // delete nonexistent -> 200 (no-op, doesn't error) + let resp = authed(client().delete(variable_url(port, "delete", "u/test-user/new_var"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- delete_bulk --- + let resp = authed(client().delete(format!("{base}/delete_bulk"))) + .json(&json!({"paths": ["u/test-user/new_secret", "u/test-user/another_var"]})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let deleted = resp.json::>().await?; + assert_eq!(deleted.len(), 2); + + let resp = authed_get(port, "exists", "u/test-user/new_secret").await; + assert_eq!(resp.json::().await?, false); + + let resp = authed_get(port, "exists", "u/test-user/another_var").await; + assert_eq!(resp.json::().await?, false); + + // --- encrypt --- + let resp = authed(client().post(format!("{base}/encrypt"))) + .json(&"test_plaintext".to_string()) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let encrypted = resp.text().await?; + assert!(!encrypted.is_empty()); + assert_ne!(encrypted, "test_plaintext"); + + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 381fe21db2..f98e1e3ad7 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -342,7 +342,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { let server = ApiServer::start(db.clone()).await?; - let write_file = r#"export async function main(loop: boolean, i: number, path: string) { + let write_file = r#"export async function main(loop: boolean, i: number, path: string) { await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`); }"# .to_string(); @@ -458,7 +458,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { )] .into(), language: ScriptLang::Deno, - content: r#"export async function main(path: string, path2: string) { + content: r#"export async function main(path: string, path2: string) { return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`); }"# .to_string(), @@ -917,47 +917,50 @@ fn main(world: String) -> Result { Ok(()) } -// #[sqlx::test(fixtures("base"))] -// async fn test_csharp_job(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// let content = r#" -// using System; -// -// class Script -// { -// public static string Main(string world, int b = 2) -// { -// Console.WriteLine($"Hello {world} - {b}. This is a log line"); -// return $"Hello {world} - {b}"; -// } -// } -// "# -// .to_owned(); -// -// let result = RunJob::from(JobPayload::Code(RawCode { -// hash: None, -// content, -// path: None, -// lock: None, -// language: ScriptLang::CSharp, -// custom_concurrency_key: None, -// concurrent_limit: None, -// concurrency_time_window_s: None, -// cache_ttl: None, -// dedicated_worker: None, -// })) -// .arg("world", json!("Arakis")) -// .arg("b", json!(3)) -// .run_until_complete(&db, false, port) -// .await -// .json_result() -// .unwrap(); -// -// assert_eq!(result, serde_json::json!("Hello Arakis - 3")); -// } +#[cfg(feature = "csharp")] +#[sqlx::test(fixtures("base"))] +async fn test_csharp_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +using System; + +class Script +{ + public static string Main(string world, int b = 2) + { + Console.WriteLine($"Hello {world} - {b}. This is a log line"); + return $"Hello {world} - {b}"; + } +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::CSharp, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("world", json!("Arakis")) + .arg("b", json!(3)) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("Hello Arakis - 3")); + Ok(()) +} #[sqlx::test(fixtures("base"))] async fn test_bash_job(db: Pool) -> anyhow::Result<()> { @@ -1033,7 +1036,7 @@ async fn test_nu_job_full(db: Pool) -> anyhow::Result<()> { let port = server.addr.port(); let content = r#" -def main [ +def main [ # Required ## Primitive a @@ -1145,6 +1148,379 @@ public class Main { Ok(()) } +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_nativets_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export async function main(name: string): Promise { + return `hello ${name}`; +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Nativets, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("name", json!("world")) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("hello world")); + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_nativets_job_with_args(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export async function main(a: number, b: number): Promise { + return a + b; +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Nativets, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("a", json!(3)) + .arg("b", json!(7)) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(10)); + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_nativets_job_object_return(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export async function main(items: string[]): Promise<{ count: number; items: string[] }> { + return { count: items.length, items }; +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Nativets, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("items", json!(["a", "b", "c"])) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!({"count": 3, "items": ["a", "b", "c"]})); + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_nativets_job_datetime(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // nativets passes Date-typed args as strings (no auto-conversion unlike Bun/Deno) + let content = r#" +export async function main(a: Date) { + return typeof a; +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Nativets, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("a", json!("2024-09-24T10:00:00.000Z")) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("string")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_postgresql_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +-- $1 name +SELECT 'hello ' || $1::text AS result; +"# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Postgresql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("name", json!("world")) + .arg( + "database", + json!({"host": "localhost", "port": 5432, "dbname": "windmill", "user": "postgres", "password": "changeme"}), + ) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!([{"result": "hello world"}])); + Ok(()) +} + +#[cfg(feature = "mysql")] +#[sqlx::test(fixtures("base"))] +async fn test_mysql_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +-- ? name (varchar) +SELECT ? AS result; +"# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Mysql, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("name", json!("world")) + .arg( + "database", + json!({"host": "localhost", "port": 3306, "user": "root", "password": "changeme", "database": "windmill_test"}), + ) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, json!([{"result": "world"}])); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_bunnative_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +export async function main(name: string): Promise { + return `hello ${name}`; +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Bunnative, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("name", json!("world")) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("hello world")); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_powershell_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +param($msg) +Write-Output "hello $msg" +"# + .to_owned(); + + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Powershell, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("msg", json!("world")) + .run_until_complete(&db, false, port) + .await; + assert_eq!(job.json_result(), Some(json!("hello world"))); + Ok(()) +} + +#[cfg(feature = "php")] +#[sqlx::test(fixtures("base"))] +async fn test_php_job(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +def main(name) + "hello #{name}" +end +"# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + lock: None, + language: ScriptLang::Ruby, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + })) + .arg("name", json!("world")) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!("hello world")); + Ok(()) +} + #[sqlx::test(fixtures("base"))] async fn test_bun_job_datetime(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -2856,7 +3232,7 @@ def heavy_compute(n: int): def send_result(res: int, email: str): print(f"Sending result {res} to {email}") return "OK" - + def main(n: int): l = [] for i in range(n): @@ -2967,10 +3343,11 @@ async fn test_duckdb_ffi(db: Pool) -> anyhow::Result<()> { /// Test that flow substeps with tags that are not available for the workspace fail. /// This validates that `check_tag_available_for_workspace_internal` is properly called /// when pushing jobs from worker_flow. -#[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow::Result<()> { - use windmill_common::worker::{CustomTags, SpecificTagData, SpecificTagType, CUSTOM_TAGS_PER_WORKSPACE}; + use windmill_common::worker::{ + CustomTags, SpecificTagData, SpecificTagType, CUSTOM_TAGS_PER_WORKSPACE, + }; initialize_tracing().await; @@ -3007,19 +3384,21 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: let result = RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) + .email("test2@windmill.dev") .run_until_complete(&db, false, server.addr.port()) .await; // The flow should fail because the tag is not available for test-workspace - assert!(!result.success, "Flow should have failed due to unavailable tag"); + assert!( + !result.success, + "Flow should have failed due to unavailable tag" + ); let result_json = result.json_result(); assert!(result_json.is_some(), "Result should have error details"); let error_result = result_json.unwrap(); - let error_message = error_result["error"]["message"] - .as_str() - .unwrap_or(""); + let error_message = error_result["error"]["message"].as_str().unwrap_or(""); // Verify the error is about tag availability assert!( diff --git a/backend/tests/workspace_dependencies.rs b/backend/tests/workspace_dependencies.rs index b41eb7c550..124e727e91 100644 --- a/backend/tests/workspace_dependencies.rs +++ b/backend/tests/workspace_dependencies.rs @@ -7,7 +7,7 @@ mod workspace_dependencies { use sqlx::{Pool, Postgres}; use tokio_stream::StreamExt; use windmill_common::scripts::ScriptLang; - use windmill_worker::workspace_dependencies::NewWorkspaceDependencies; + use windmill_dep_map::workspace_dependencies::NewWorkspaceDependencies; mod deps { pub const REQUIREMENTS_IN: &'static str = "tiny==0.1.3"; // pub const GO_MOD: &'static str = r##" diff --git a/backend/tests/workspaces.rs b/backend/tests/workspaces.rs new file mode 100644 index 0000000000..4a7507b155 --- /dev/null +++ b/backend/tests/workspaces.rs @@ -0,0 +1,715 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +mod common; +use common::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +#[sqlx::test(fixtures("base"))] +async fn test_workspace_endpoints(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + let global_base = format!("http://localhost:{port}/api/workspaces"); + + // ===== Global endpoints ===== + + // --- list --- + let resp = authed(client().get(format!("{global_base}/list"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().any(|w| w["id"] == "test-workspace")); + + // --- list_as_superadmin --- + let resp = authed(client().get(format!("{global_base}/list_as_superadmin"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let list = resp.json::>().await?; + assert!(list.iter().any(|w| w["id"] == "test-workspace")); + + // --- users (user's workspaces) --- + let resp = authed(client().get(format!("{global_base}/users"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + let workspaces = body["workspaces"].as_array().unwrap(); + assert!(workspaces.iter().any(|w| w["id"] == "test-workspace")); + + // --- exists --- + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "test-workspace"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, true); + + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "nonexistent-workspace"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + assert_eq!(resp.json::().await?, false); + + // --- exists_username (validates username is available) --- + let resp = authed(client().post(format!("{global_base}/exists_username"))) + .json(&json!({"id": "test-workspace", "username": "test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + + let resp = authed(client().post(format!("{global_base}/exists_username"))) + .json(&json!({"id": "test-workspace", "username": "available-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- allowed_domain_auto_invite --- + let resp = authed(client().get(format!( + "{global_base}/allowed_domain_auto_invite" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await?; + + // --- create workspace --- + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ + "id": "new-test-ws", + "name": "New Test Workspace" + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + + // verify it exists + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "new-test-ws"})) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, true); + + // ===== Workspace-scoped endpoints (read) ===== + + // --- get_settings --- + let resp = authed(client().get(format!("{base}/get_settings"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let settings = resp.json::().await?; + assert!(settings.is_object()); + + // --- get_deploy_to --- + let resp = authed(client().get(format!("{base}/get_deploy_to"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- is_premium --- + let resp = authed(client().get(format!("{base}/is_premium"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- default_app --- + let resp = authed(client().get(format!("{base}/default_app"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- default_scripts --- + let resp = authed(client().get(format!("{base}/default_scripts"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- list_pending_invites --- + let resp = authed(client().get(format!("{base}/list_pending_invites"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- encryption_key --- + let resp = authed(client().get(format!("{base}/encryption_key"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- get_dependency_map --- + let resp = authed(client().get(format!("{base}/get_dependency_map"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- get_as_superadmin --- + let resp = authed(client().get(format!("{base}/get_as_superadmin"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let body = resp.json::().await?; + assert_eq!(body["id"], "test-workspace"); + + // --- get_workspace_name --- + let resp = authed(client().get(format!("{base}/get_workspace_name"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let name = resp.text().await?; + assert_eq!(name, "test-workspace"); + + // --- get_usage --- + let resp = authed(client().get(format!("{base}/usage"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- get_used_triggers --- + let resp = authed(client().get(format!("{base}/used_triggers"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await?; + + // --- get_secondary_storage_names --- + let resp = authed(client().get(format!("{base}/get_secondary_storage_names"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- get_dependents (empty, no dependencies exist) --- + let resp = authed(client().get(format!( + "{base}/get_dependents/u/test-user/nonexistent" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + let dependents = resp.json::>().await?; + assert!(dependents.is_empty()); + + // --- get_dependents_amounts --- + let resp = authed(client().post(format!("{base}/get_dependents_amounts"))) + .json(&json!(["u/test-user/some_script"])) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- list_ducklakes --- + let resp = authed(client().get(format!("{base}/list_ducklakes"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- list_datatables --- + let resp = authed(client().get(format!("{base}/list_datatables"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // --- list_datatable_schemas --- + let resp = authed(client().get(format!("{base}/list_datatable_schemas"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::>().await?; + + // ===== Workspace-scoped endpoints (mutations) ===== + + // --- update (edit_workspace) --- + let resp = authed(client().post(format!("{base}/update"))) + .json(&json!({"name": "renamed-workspace", "owner": "test-user"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + + // --- change_workspace_name --- + let resp = authed(client().post(format!("{base}/change_workspace_name"))) + .json(&json!({"new_name": "Test Workspace Renamed"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "change_workspace_name: {}", + resp.text().await? + ); + + // verify name changed + let resp = authed(client().get(format!("{base}/get_workspace_name"))) + .send() + .await + .unwrap(); + assert_eq!(resp.text().await?, "Test Workspace Renamed"); + + // --- change_workspace_color --- + let resp = authed(client().post(format!("{base}/change_workspace_color"))) + .json(&json!({"color": "#FF5733"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "change_workspace_color: {}", + resp.text().await? + ); + + // --- edit_webhook --- + let resp = authed(client().post(format!("{base}/edit_webhook"))) + .json(&json!({"webhook": "https://example.com/hook"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "edit_webhook: {}", resp.text().await?); + + // verify in settings + let resp = authed(client().get(format!("{base}/get_settings"))) + .send() + .await + .unwrap(); + let settings = resp.json::().await?; + assert_eq!(settings["webhook"], "https://example.com/hook"); + + // clear webhook + let resp = authed(client().post(format!("{base}/edit_webhook"))) + .json(&json!({"webhook": null})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- edit_auto_invite (EE-gated) --- + let resp = authed(client().post(format!("{base}/edit_auto_invite"))) + .json(&json!({"operator": false, "invite_all": false, "auto_add": false})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 500, + "edit_auto_invite: unexpected status {}", + resp.status() + ); + + // --- edit_slack_command --- + let resp = authed(client().post(format!("{base}/edit_slack_command"))) + .json(&json!({"slack_command_script": null})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "edit_slack_command: {}", + resp.text().await? + ); + + // --- edit_error_handler (new format) --- + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "extra_args": null})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "edit_error_handler: {}", + resp.text().await? + ); + + // --- edit_success_handler (new format) --- + let resp = authed(client().post(format!("{base}/edit_success_handler"))) + .json(&json!({"path": null, "extra_args": null})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "edit_success_handler: {}", + resp.text().await? + ); + + // --- edit_default_scripts --- + let resp = authed(client().post(format!("{base}/default_scripts"))) + .json(&json!(null)) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "edit_default_scripts: {}", + resp.text().await? + ); + + // --- edit_default_app (EE-gated, may return 200 or error) --- + let resp = authed(client().post(format!("{base}/edit_default_app"))) + .json(&json!({})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "edit_default_app: unexpected status {}", + resp.status() + ); + + // --- set_environment_variable --- + let resp = authed(client().post(format!("{base}/set_environment_variable"))) + .json(&json!({"name": "TEST_ENV_VAR", "value": "test_value"})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "set_environment_variable: {}", + resp.text().await? + ); + + // --- edit_deploy_to (EE-gated) --- + let resp = authed(client().post(format!("{base}/edit_deploy_to"))) + .json(&json!({"deploy_to": null})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "edit_deploy_to: unexpected status {}", + resp.status() + ); + + // --- edit_large_file_storage_config --- + let resp = authed(client().post(format!( + "{base}/edit_large_file_storage_config" + ))) + .json(&json!({"large_file_storage": null})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "edit_large_file_storage_config: {}", + resp.text().await? + ); + + // --- edit_deploy_ui_config (EE-gated) --- + let resp = authed(client().post(format!("{base}/edit_deploy_ui_config"))) + .json(&json!({"deploy_ui": null})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "edit_deploy_ui_config: unexpected status {}", + resp.status() + ); + + // --- edit_git_sync_config (EE-gated) --- + let resp = authed(client().post(format!("{base}/edit_git_sync_config"))) + .json(&json!({"git_sync_settings": null})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 400, + "edit_git_sync_config: unexpected status {}", + resp.status() + ); + + // --- update_operator_settings --- + let resp = authed(client().post(format!("{base}/operator_settings"))) + .json(&json!({})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "update_operator_settings: {}", + resp.text().await? + ); + + // --- edit_public_app_rate_limit --- + let resp = authed(client().post(format!("{base}/public_app_rate_limit"))) + .json(&json!({"public_app_execution_limit_per_minute": null})) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "edit_public_app_rate_limit: {}", + resp.text().await? + ); + + // --- rebuild_dependency_map --- + let resp = authed(client().post(format!("{base}/rebuild_dependency_map"))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "rebuild_dependency_map: {}", + resp.text().await? + ); + + // --- add_user --- + let resp = authed(client().post(format!("{base}/add_user"))) + .json(&json!({ + "email": "newuser@windmill.dev", + "is_admin": false, + "operator": false + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "add_user: {}", resp.text().await?); + + // --- invite_user + list_pending_invites + delete_invite --- + let resp = authed(client().post(format!("{base}/invite_user"))) + .json(&json!({ + "email": "invited@example.com", + "is_admin": false, + "operator": false + })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "invite_user: {}", resp.text().await?); + + // verify invite shows in pending + let resp = authed(client().get(format!("{base}/list_pending_invites"))) + .send() + .await + .unwrap(); + let invites = resp.json::>().await?; + assert!( + invites + .iter() + .any(|i| i["email"] == "invited@example.com"), + "invite not found: {:?}", + invites + ); + + // delete invite + let resp = authed(client().post(format!("{base}/delete_invite"))) + .json(&json!({ + "email": "invited@example.com", + "is_admin": false, + "operator": false + })) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 201, + "delete_invite: {}", + resp.text().await? + ); + + // ===== Critical alerts (EE-gated, returns 404 in OSS) ===== + + // --- get critical_alerts --- + let resp = authed(client().get(format!("{base}/critical_alerts"))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 404, + "critical_alerts: unexpected status {}", + resp.status() + ); + + // --- acknowledge critical alert (nonexistent id) --- + let resp = authed(client().post(format!("{base}/critical_alerts/1/acknowledge"))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 404, + "acknowledge_critical_alert: unexpected status {}", + resp.status() + ); + + // --- acknowledge_all critical alerts --- + let resp = authed(client().post(format!("{base}/critical_alerts/acknowledge_all"))) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 404, + "acknowledge_all_critical_alerts: unexpected status {}", + resp.status() + ); + + // --- mute critical alerts --- + let resp = authed(client().post(format!("{base}/critical_alerts/mute"))) + .json(&json!({"mute_critical_alerts": false})) + .send() + .await + .unwrap(); + assert!( + resp.status() == 200 || resp.status() == 404, + "mute_critical_alerts: unexpected status {}", + resp.status() + ); + + // ===== Tarball export ===== + + // --- tarball (download workspace as tar archive) --- + let resp = authed(client().get(format!("{base}/tarball"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "tarball: {}", resp.status()); + + // ===== Fork operations (on the newly created workspace) ===== + + // --- create_fork (workspace-scoped, from new-test-ws) --- + let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces"); + let resp = authed(client().post(format!("{new_ws_base}/create_fork"))) + .json(&json!({ + "id": "wm-fork-test-ws", + "name": "Forked Test Workspace" + })) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "create_fork: {}", + resp.text().await? + ); + + // verify fork exists + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "wm-fork-test-ws"})) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, true); + + // --- change_workspace_id --- + let fork_ws_base = format!("http://localhost:{port}/api/w/wm-fork-test-ws/workspaces"); + let resp = authed(client().post(format!("{fork_ws_base}/change_workspace_id"))) + .json(&json!({ + "new_id": "wm-fork-renamed", + "new_name": "Renamed Fork" + })) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "change_workspace_id: {}", + resp.text().await? + ); + + // verify renamed workspace exists + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "wm-fork-renamed"})) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, true); + + // clean up renamed fork + let resp = authed(client().delete(format!("{global_base}/delete/wm-fork-renamed"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + // --- archive workspace (on the newly created one, not our main test workspace) --- + let new_ws_base = format!("http://localhost:{port}/api/w/new-test-ws/workspaces"); + let resp = authed(client().post(format!("{new_ws_base}/archive"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "archive: {}", resp.text().await?); + + // --- unarchive workspace (global) --- + let resp = authed(client().post(format!("{global_base}/unarchive/new-test-ws"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "unarchive: {}", resp.text().await?); + + // --- delete workspace (global) --- + let resp = authed(client().delete(format!("{global_base}/delete/new-test-ws"))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + + // verify deleted + let resp = authed(client().post(format!("{global_base}/exists"))) + .json(&json!({"id": "new-test-ws"})) + .send() + .await + .unwrap(); + assert_eq!(resp.json::().await?, false); + + // --- create_workspace_require_superadmin --- + let resp = authed(client().get(format!( + "{global_base}/create_workspace_require_superadmin" + ))) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + Ok(()) +} diff --git a/backend/windmill-api-auth/Cargo.toml b/backend/windmill-api-auth/Cargo.toml new file mode 100644 index 0000000000..515a51cab7 --- /dev/null +++ b/backend/windmill-api-auth/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "windmill-api-auth" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_auth" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "dep:jsonwebtoken", "dep:reqwest"] +cloud = ["windmill-common/cloud"] +private = ["windmill-common/private"] +no_auth = [] +mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true +windmill-mcp = { workspace = true, optional = true } +axum.workspace = true +http.workspace = true +serde.workspace = true +serde_json.workspace = true +uuid.workspace = true +itertools.workspace = true +tower-cookies.workspace = true +tracing.workspace = true +chrono.workspace = true +quick_cache.workspace = true +sqlx.workspace = true +lazy_static.workspace = true +tokio.workspace = true +time.workspace = true +anyhow.workspace = true +jsonwebtoken = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs new file mode 100644 index 0000000000..186f7af257 --- /dev/null +++ b/backend/windmill-api-auth/src/auth.rs @@ -0,0 +1,745 @@ +#[cfg(feature = "enterprise")] +use crate::ee_oss::ExternalJwks; +use axum::{ + async_trait, + extract::{FromRequestParts, OriginalUri, Query}, + Extension, Json, +}; +use chrono::TimeZone; +use http::{request::Parts, StatusCode}; +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use tower_cookies::Cookies; +use tracing::Span; + +use crate::{ApiAuthed, OptJobAuthed}; +use std::{ + str::FromStr, + sync::{ + atomic::{AtomicI64, AtomicU64, Ordering}, + Arc, + }, +}; +#[cfg(feature = "enterprise")] +use tokio::sync::RwLock; +use windmill_common::DB; + +use windmill_common::{ + auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, + error::{Error, JsonResult}, + jwt, + users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, +}; + +lazy_static::lazy_static! { + // Global auth cache accessible from main.rs for direct invalidation + pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); + +} +// Global function to invalidate a specific token from cache +pub fn invalidate_token_from_cache(token: &str) { + // Remove all cache entries for this token (across all workspaces) + AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); + tracing::info!( + "Invalidated token from auth cache: {}...", + &token[..token.len().min(8)] + ); +} + +#[derive(Clone)] +pub struct ExpiringAuthCache { + pub authed: ApiAuthed, + pub expiry: chrono::DateTime, + pub job_id: Option, +} + +pub struct AuthCache { + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] + ext_jwks: Option>>, +} + +impl AuthCache { + pub fn new( + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] ext_jwks: Option>>, + ) -> Self { + AuthCache { + db, + superadmin_secret, + #[cfg(feature = "enterprise")] + ext_jwks, + } + } + + pub async fn invalidate(&self, w_id: &str, token: String) { + AUTH_CACHE.remove(&(w_id.to_string(), token)); + } + + pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { + Some(self.get_opt_job_authed(w_id, token).await?.authed) + } + + pub async fn get_opt_job_authed( + &self, + w_id: Option, + token: &str, + ) -> Option { + let key = ( + w_id.as_ref().unwrap_or(&"".to_string()).to_string(), + token.to_string(), + ); + let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); + match s { + Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => { + Some(OptJobAuthed { authed, job_id }) + } + #[cfg(feature = "enterprise")] + _ if token.starts_with("jwt_ext_") => { + let authed_and_exp = match crate::ee_oss::jwt_ext_auth( + w_id.as_ref(), + token.trim_start_matches("jwt_ext_"), + self.ext_jwks.clone(), + &self.db, + ) + .await + { + Ok(r) => Some(r), + Err(e) => { + tracing::error!("JWT_EXT auth error: {:?}", e); + None + } + }; + + if let Some((authed, exp, job_id)) = authed_and_exp.clone() { + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), + job_id, + }, + ); + + Some(OptJobAuthed { authed, job_id }) + } else { + None + } + } + _ if token.starts_with("jwt_") => { + let jwt_token = token.trim_start_matches("jwt_"); + + let jwt_result = jwt::decode_with_internal_secret::(jwt_token).await; + + match jwt_result { + Ok(claims) => { + if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { + tracing::error!("JWT auth error: workspace_id mismatch"); + return None; + } + let username_override = username_override_from_label(claims.label); + + let authed = ApiAuthed { + email: claims.email, + username: claims.username, + is_admin: claims.is_admin, + is_operator: claims.is_operator, + groups: claims.groups, + folders: claims.folders, + scopes: None, + username_override, + token_prefix: claims.audit_span, + }; + let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok()); + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc + .timestamp_nanos(claims.exp as i64 * 1_000_000_000), + job_id, + }, + ); + + Some(OptJobAuthed { authed, job_id }) + } + Err(err) => { + tracing::error!("JWT auth error: {:?}", err); + None + } + } + } + _ => { + let user_o = sqlx::query!( + "UPDATE token SET last_used_at = now() WHERE + token = $1 + AND (expiration > NOW() OR expiration IS NULL) + AND (workspace_id IS NULL OR workspace_id = $2) + RETURNING owner, email, super_admin, scopes, label", + token, + w_id.as_ref(), + ) + .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) + .fetch_optional(&self.db) + .await + .ok() + .flatten(); + + if let Some(user) = user_o { + let authed_o = { + match user { + (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { + let username_override = username_override_from_label(label); + if let Some((prefix, name)) = owner.split_once('/') { + if prefix == "u" { + let (is_admin, is_operator) = if super_admin { + (true, false) + } else { + let r = sqlx::query!( + "SELECT is_admin, operator FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id.as_ref().unwrap() + ) + .fetch_one(&self.db) + .await + .ok(); + if let Some(r) = r { + (r.is_admin, r.operator) + } else { + (false, true) + } + }; + + let w_id = &w_id.unwrap(); + let groups = + get_groups_for_user(w_id, &name, &email, &self.db) + .await + .ok() + .unwrap_or_default(); + + let folders = + get_folders_for_user(w_id, &name, &groups, &self.db) + .await + .ok() + .unwrap_or_default(); + + Some(ApiAuthed { + email: email, + username: name.to_string(), + is_admin, + is_operator, + groups, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } else { + let groups = vec![name.to_string()]; + let folders = get_folders_for_user( + &w_id.unwrap(), + "", + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email: email, + username: format!("group-{name}"), + is_admin: false, + groups, + is_operator: false, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + } else { + let groups = vec![]; + let folders = vec![]; + Some(ApiAuthed { + email: email, + username: owner, + is_admin: super_admin, + is_operator: true, + groups, + folders, + scopes: None, + username_override, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }) + } + } + (_, Some(email), super_admin, scopes, label) => { + let username_override = username_override_from_label(label); + if w_id.is_some() { + let row_o = sqlx::query!( + "SELECT username, is_admin, operator FROM usr WHERE + email = $1 AND workspace_id = $2 AND disabled = false", + &email, + w_id.as_ref().unwrap() + ) + .map(|x| (x.username, x.is_admin, x.operator)) + .fetch_optional(&self.db) + .await + .unwrap_or(Some(("error".to_string(), false, false))); + + match row_o { + Some((username, is_admin, is_operator)) => { + let groups = get_groups_for_user( + &w_id.as_ref().unwrap(), + &username, + &email, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user( + &w_id.unwrap(), + &username, + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email, + username, + is_admin: is_admin || super_admin, + is_operator, + groups, + folders, + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + None if super_admin => Some(ApiAuthed { + email: email.clone(), + username: email, + is_admin: super_admin, + is_operator: false, + groups: vec![], + folders: vec![], + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }), + None => None, + } + } else { + Some(ApiAuthed { + email: email.to_string(), + username: email, + is_admin: super_admin, + is_operator: true, + groups: Vec::new(), + folders: Vec::new(), + scopes, + username_override, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }) + } + } + _ => None, + } + }; + if let Some(authed) = authed_o.as_ref() { + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc::now() + + chrono::Duration::try_seconds(120).unwrap(), + job_id: None, + }, + ); + } + authed_o.map(|authed| OptJobAuthed { authed, job_id: None }) + } else if self + .superadmin_secret + .as_ref() + .map(|x| x == token) + .unwrap_or(false) + { + let authed = ApiAuthed { + email: SUPERADMIN_SECRET_EMAIL.to_string(), + username: "superadmin_secret".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }; + Some(OptJobAuthed { authed, job_id: None }) + } else { + None + } + } + } + } +} + +pub(crate) async fn extract_token(parts: &mut Parts, state: &S) -> Option { + let auth_header = parts + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + + let from_cookie = match auth_header { + Some(x) => Some(x.to_owned()), + None => Extension::::from_request_parts(parts, state) + .await + .ok() + .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), + }; + + #[derive(Deserialize)] + struct Token { + token: Option, + } + match from_cookie { + Some(token) => Some(token), + None => Query::::from_request_parts(parts, state) + .await + .ok() + .and_then(|token| token.token.clone()), + } +} + +#[derive(Clone, Debug)] +pub struct Tokened { + pub token: String, +} + +#[derive(Clone, Debug)] +pub struct OptTokened { + #[allow(dead_code)] + pub token: Option, +} + +struct BruteForceCounter { + counter: AtomicU64, + last_reset: AtomicI64, +} + +lazy_static::lazy_static! { + static ref BRUTE_FORCE_COUNTER: BruteForceCounter = + BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; +} + +impl BruteForceCounter { + async fn increment(&self) { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { + tracing::error!( + "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if now - self.last_reset.load(Ordering::Relaxed) > 60 { + self.counter.store(0, Ordering::Relaxed); + self.last_reset.store(now, Ordering::Relaxed); + } + } +} + +#[async_trait] +impl FromRequestParts for Tokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(Tokened { token: "".to_string() }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(tokened.clone()) + } else { + let token_o = extract_token(parts, state).await; + if let Some(token) = token_o { + let tokened = Self { token }; + parts.extensions.insert(tokened.clone()); + Ok(tokened) + } else { + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } + } +} + +#[async_trait] +impl FromRequestParts for OptTokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(OptTokened { token: None }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(OptTokened { token: Some(tokened.token.clone()) }) + } else { + let token_o = extract_token(parts, state).await; + Ok(OptTokened { token: token_o }) + } + } +} + +pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { + if let Some(scopes) = scopes { + for scope in scopes.iter_mut() { + if scope.starts_with("run:") { + let (_, part_scope) = scope.split_once(":").unwrap(); + + if let Some((kind, path)) = part_scope.split_once("/") { + //appending a 's' as runnable kind is singular while new scope format expect it to be plural + *scope = format!("jobs:run:{}s:{}", kind, path); + } + } else if scope.starts_with("jobs:") { + // Map old jobs scopes to new format + let new_scope = match scope.as_str() { + "jobs:listjobs" => "jobs:read", + "jobs:runscript" => "jobs:run:scripts", + "jobs:runflow" => "jobs:run:flows", + "jobs:resumeflow" => "jobs:run:flows", + "jobs:deletejob" => "jobs:write", + _ => continue, + }; + + *scope = new_scope.to_string(); + } + } + } +} + +fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { + let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { + Some(path_vec[3].to_owned()) + } else if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[1] == "api" + && path_vec[2] == "mcp" + && path_vec[3] == "w" + { + Some(path_vec[4].to_owned()) + } else { + if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" { + Some(path_vec[4].to_owned()) + } else { + None + } + }; + + workspace_id +} + +/// Resolves OptJobAuthed from request parts. +/// Takes ownership of Parts and returns them back. +#[allow(unreachable_code, unused_mut)] +pub async fn resolve_opt_job_authed( + mut parts: Parts, +) -> std::result::Result<(OptJobAuthed, Parts), (Error, Parts)> { + if parts.method == http::Method::OPTIONS { + return Ok((OptJobAuthed::default(), parts)); + }; + + #[cfg(feature = "no_auth")] + { + let authed = ApiAuthed { + email: "admin@windmill.dev".to_string(), + username: "admin".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }; + return Ok((OptJobAuthed { authed, job_id: None }, parts)); + } + + let already_authed = parts.extensions.get::().cloned(); + + if let Some(authed) = already_authed { + return Ok((authed, parts)); + } + + let already_tokened = parts.extensions.get::().cloned(); + let token_o = if let Some(token) = already_tokened { + Some(token.token.clone()) + } else { + extract_token(&mut parts, &()).await + }; + if let Some(token) = token_o { + if let Ok(Extension(cache)) = + Extension::>::from_request_parts(&mut parts, &()).await + { + let original_uri = OriginalUri::from_request_parts(&mut parts, &()) + .await + .ok() + .map(|x| x.0) + .unwrap_or_default(); + let path_vec: Vec<&str> = original_uri.path().split("/").collect(); + let workspace_id = maybe_get_workspace_id_from_path(&path_vec); + + if let Some(mut opt_job_authed) = + cache.get_opt_job_authed(workspace_id.clone(), &token).await + { + let authed = &mut opt_job_authed.authed; + if authed.scopes.is_some() { + transform_old_scope_to_new_scope(authed.scopes.as_mut()); + + let path = original_uri.path(); + let method = parts.method.as_str(); + + if let Err(err) = crate::scopes::check_scopes_for_route( + authed.scopes.as_deref(), + path, + method, + ) { + BRUTE_FORCE_COUNTER.increment().await; + return Err((err, parts)); + } + } + parts.extensions.insert(authed.clone()); + + Span::current().record("username", &authed.username.as_str()); + Span::current().record("email", &authed.email); + + if let Some(workspace_id) = workspace_id { + Span::current().record("workspace_id", &workspace_id); + } + return Ok((opt_job_authed, parts)); + } + } + } + BRUTE_FORCE_COUNTER.increment().await; + Err((Error::NotAuthorized("Unauthorized".to_string()), parts)) +} + +fn username_override_from_label(label: Option) -> Option { + match label { + Some(label) + if label.starts_with("webhook-") + || label.starts_with("http-") + || label.starts_with("email-") + || label.starts_with("ws-") => + { + Some(label) + } + Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( + label + .trim_start_matches("ephemeral-script-end-user-") + .to_string(), + ), + Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), + Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { + Some(format!("label-{label}")) + } + _ => None, + } +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedTokenWithEmail { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, + pub email: Option, +} + +pub async fn list_tokens_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult> { + let tokens = if is_flow { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:flows:' || $2]::text[] + OR scopes @> ARRAY['run:flow/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + } else { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] + OR scopes @> ARRAY['run:script/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + }; + + Ok(Json(tokens)) +} diff --git a/backend/windmill-api-auth/src/ee_oss.rs b/backend/windmill-api-auth/src/ee_oss.rs new file mode 100644 index 0000000000..33b1e9d41a --- /dev/null +++ b/backend/windmill-api-auth/src/ee_oss.rs @@ -0,0 +1,36 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +use {std::sync::Arc, tokio::sync::RwLock}; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub async fn jwt_ext_auth( + _w_id: Option<&String>, + _token: &str, + _external_jwks: Option>>, + _db: &windmill_common::DB, +) -> anyhow::Result<(crate::ApiAuthed, usize, Option)> { + // Implementation is not open source + Err(anyhow::anyhow!("External JWT auth is not open source")) +} + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub struct ExternalJwks; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +impl ExternalJwks { + pub async fn load() -> Option>> { + // Implementation is not open source + None + } +} diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs new file mode 100644 index 0000000000..ba0bf928e7 --- /dev/null +++ b/backend/windmill-api-auth/src/lib.rs @@ -0,0 +1,713 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod auth; +#[cfg(feature = "private")] +pub mod ee; +pub mod ee_oss; +pub mod scopes; + +use axum::async_trait; +use axum::extract::FromRequestParts; +use http::request::Parts; + +use windmill_audit::audit_oss::AuditAuthorable; +use windmill_common::{ + auth::{fetch_authed_from_permissioned_as, is_devops_email, is_super_admin_email}, + db::{Authable, Authed, AuthedRef}, + error::{self, Error, Result}, + users::username_to_permissioned_as, + DB, +}; + +use scopes::ScopeDefinition; + +// Re-export key auth types and functions +pub use auth::{ + invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, Tokened, + TruncatedTokenWithEmail, AUTH_CACHE, +}; + +// ------------ ApiAuthed & OptJobAuthed types ------------ + +#[derive(Default, Clone, Debug)] +pub struct OptJobAuthed { + pub job_id: Option, + pub authed: ApiAuthed, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] +pub struct ApiAuthed { + pub email: String, + pub username: String, + pub is_admin: bool, + pub is_operator: bool, + pub groups: Vec, + // (folder name, can write, is owner) + pub folders: Vec<(String, bool, bool)>, + pub scopes: Option>, + pub username_override: Option, + pub token_prefix: Option, +} + +impl ApiAuthed { + pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> { + AuthedRef { + email: &self.email, + username: &self.username, + is_admin: &self.is_admin, + is_operator: &self.is_operator, + groups: &self.groups, + folders: &self.folders, + scopes: &self.scopes, + token_prefix: &self.token_prefix, + } + } + + pub fn display_username(&self) -> &str { + self.username_override.as_ref().unwrap_or(&self.username) + } +} + +impl From for Authed { + fn from(value: ApiAuthed) -> Self { + Self { + email: value.email, + username: value.username, + is_admin: value.is_admin, + is_operator: value.is_operator, + groups: value.groups, + folders: value.folders, + scopes: value.scopes, + token_prefix: value.token_prefix, + } + } +} + +impl From for ApiAuthed { + fn from(value: Authed) -> Self { + Self { + email: value.email, + username: value.username, + is_admin: value.is_admin, + is_operator: value.is_operator, + groups: value.groups, + folders: value.folders, + scopes: value.scopes, + username_override: None, + token_prefix: value.token_prefix, + } + } +} + +impl AuditAuthorable for ApiAuthed { + fn username(&self) -> &str { + self.username.as_str() + } + fn email(&self) -> &str { + self.email.as_str() + } + fn username_override(&self) -> Option<&str> { + self.username_override.as_deref() + } + fn token_prefix(&self) -> Option<&str> { + self.token_prefix.as_deref() + } +} + +impl Authable for ApiAuthed { + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[std::string::String]> { + self.scopes.as_ref().map(|x| x.as_slice()) + } + + fn email(&self) -> &str { + &self.email + } + + fn username(&self) -> &str { + &self.username + } +} + +// ------------ McpAuth impl (feature-gated) ------------ + +#[cfg(feature = "mcp")] +impl windmill_mcp::server::McpAuth for ApiAuthed { + fn username(&self) -> &str { + &self.username + } + + fn email(&self) -> &str { + &self.email + } + + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[String]> { + self.scopes.as_deref() + } +} + +// ------------ Utility functions ------------ + +pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { + let is_admin = is_super_admin_email(db, email).await?; + + if !is_admin { + Err(Error::NotAuthorized( + "This endpoint requires the caller to be a super admin".to_owned(), + )) + } else { + Ok(()) + } +} + +pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if let Some(scopes) = authed.scopes.as_ref() { + let mut is_scoped_token = false; + let required_scope = ScopeDefinition::from_scope_string(&required())?; + for scope in scopes { + if !scope.starts_with("if_jobs:filter_tags:") { + if !is_scoped_token { + is_scoped_token = true; + } + + match ScopeDefinition::from_scope_string(scope) { + Ok(scope) if scope.includes(&required_scope) => return Ok(()), + _ => {} + } + } + } + + if is_scoped_token { + return Err(Error::NotAuthorized(format!( + "Required scope: {}", + required_scope.as_string() + ))); + } + } + Ok(()) +} + +pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { + let is_devops = is_devops_email(db, email).await?; + + if is_devops { + Ok(()) + } else { + Err(Error::NotAuthorized( + "This endpoint requires the caller to have the `devops` role".to_string(), + )) + } +} + +// ------------ Folder ownership checks ------------ + +pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool { + if *is_admin { + true + } else { + folders.into_iter().any(|x| x.0 == name && x.2) + } +} + +pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> { + if is_owner(authed, name) { + Ok(()) + } else { + Err(Error::NotAuthorized(format!( + "You are not owner of the folder {}", + name + ))) + } +} + +pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + let splitted = path.split("/").collect::>(); + if splitted[0] == "u" { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "only the owner {} is authorized to perform this operation", + splitted[1] + ))) + } + } else if splitted[0] == "f" { + require_is_owner(authed, splitted[1]) + } else { + Err(Error::BadRequest(format!( + "Not recognized path kind: {}", + path + ))) + } + } else { + Err(Error::BadRequest( + "Cannot be owner of an empty path".to_string(), + )) + } +} + +// ------------ Scope tag helpers ------------ + +pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { + authed.scopes.as_ref()?.iter().find_map(|s| { + if s.starts_with("if_jobs:filter_tags:") { + Some( + s.trim_start_matches("if_jobs:filter_tags:") + .split(",") + .collect::>(), + ) + } else { + None + } + }) +} + +// ------------ Maybe refresh folders ------------ + +pub async fn maybe_refresh_folders( + path: &str, + w_id: &str, + authed: ApiAuthed, + db: &DB, +) -> ApiAuthed { + use windmill_common::auth::{get_folders_for_user, get_groups_for_user}; + + if authed.is_admin { + return authed; + } + let splitted = path.split('/').collect::>(); + if splitted.len() >= 2 + && splitted[0] == "f" + && !authed.folders.iter().any(|(f, _, _)| f == splitted[1]) + { + let name = &authed.username; + let groups = get_groups_for_user(w_id, name, &authed.email, db) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user(w_id, name, &groups, db) + .await + .ok() + .unwrap_or_default(); + ApiAuthed { folders, ..authed } + } else { + authed + } +} + +// ------------ FromRequestParts impls (direct call to auth module) ------------ + +#[async_trait] +impl FromRequestParts for ApiAuthed +where + S: Send + Sync, +{ + type Rejection = Error; + + async fn from_request_parts( + parts: &mut Parts, + _state: &S, + ) -> std::result::Result { + let opt_job_authed = OptJobAuthed::from_request_parts(parts, _state).await?; + Ok(opt_job_authed.authed) + } +} + +#[async_trait] +impl FromRequestParts for OptJobAuthed +where + S: Send + Sync, +{ + type Rejection = Error; + + async fn from_request_parts( + parts: &mut Parts, + _state: &S, + ) -> std::result::Result { + // Swap out parts so we can pass ownership to resolve_opt_job_authed + let owned_parts = std::mem::replace(parts, empty_parts()); + match auth::resolve_opt_job_authed(owned_parts).await { + Ok((result, returned_parts)) => { + *parts = returned_parts; + Ok(result) + } + Err((err, returned_parts)) => { + *parts = returned_parts; + Err(err) + } + } + } +} + +fn empty_parts() -> Parts { + let (parts, _body) = http::Request::new(()).into_parts(); + parts +} + +// ------------ OptAuthed (optional auth extractor) ------------ + +#[derive(Clone, Debug)] +pub struct OptAuthed(pub Option); + +#[async_trait] +impl FromRequestParts for OptAuthed +where + S: Send + Sync, +{ + type Rejection = (http::StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + ApiAuthed::from_request_parts(parts, state) + .await + .map(|authed| Self(Some(authed))) + .or_else(|_| Ok(Self(None))) + } +} + +// ------------ fetch_api_authed helpers ------------ + +lazy_static::lazy_static! { + static ref API_AUTHED_CACHE: quick_cache::sync::Cache<(String,String,String), ExpiringAuthCache> = quick_cache::sync::Cache::new(300); +} + +#[allow(unused)] +pub async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: Option, +) -> error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await +} + +#[allow(unused)] +pub async fn fetch_api_authed_from_permissioned_as( + permissioned_as: String, + email: String, + w_id: &str, + db: &DB, + username_override: Option, +) -> error::Result { + let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); + + let mut api_authed = match API_AUTHED_CACHE.get(&key) { + Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => { + tracing::debug!("API authed cache hit for user {}", email); + expiring_authed.authed + } + _ => { + tracing::debug!("API authed cache miss for user {}", email); + + let authed = + fetch_authed_from_permissioned_as(&permissioned_as, &email, w_id, db).await?; + + let api_authed = ApiAuthed { + username: authed.username, + email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: None, + token_prefix: authed.token_prefix, + }; + + API_AUTHED_CACHE.insert( + key, + ExpiringAuthCache { + authed: api_authed.clone(), + expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(), + job_id: None, + }, + ); + + api_authed + } + }; + + api_authed.username_override = username_override; + Ok(api_authed) +} + +// ------------ Token creation ------------ + +#[derive(serde::Deserialize)] +pub struct NewToken { + pub label: Option, + pub expiration: Option>, + pub impersonate_email: Option, + pub scopes: Option>, + pub workspace_id: Option, +} + +impl NewToken { + pub fn new( + label: Option, + expiration: Option>, + impersonate_email: Option, + scopes: Option>, + workspace_id: Option, + ) -> Self { + Self { label, expiration, impersonate_email, scopes, workspace_id } + } +} + +pub async fn create_token_internal( + tx: &mut sqlx::PgConnection, + db: &DB, + authed: &ApiAuthed, + token_config: NewToken, +) -> Result { + use tracing::Instrument; + use windmill_audit::{audit_oss::audit_log, ActionKind}; + use windmill_common::{utils::rd_string, worker::CLOUD_HOSTED}; + + let token = rd_string(32); + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + if *CLOUD_HOSTED { + let nb_tokens = + sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email) + .fetch_one(db) + .await?; + if nb_tokens.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + token, + authed.email, + token_config.label, + token_config.expiration, + is_super_admin, + token_config.scopes.as_ref().map(|x| x.as_slice()), + token_config.workspace_id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + authed, + "users.token.create", + ActionKind::Create, + &"global", + Some(&token[0..10]), + None, + ) + .instrument(tracing::info_span!("token", email = &authed.email)) + .await?; + + Ok(token) +} + +// ------------ Permission helpers ------------ + +pub fn get_perm_in_extra_perms_for_authed( + v: serde_json::Value, + authed: &ApiAuthed, +) -> Option { + match v { + serde_json::Value::Object(obj) => { + let mut keys = vec![format!("u/{}", authed.username)]; + for g in authed.groups.iter() { + keys.push(format!("g/{}", g)); + } + let mut res = None; + for k in keys { + if let Some(v) = obj.get(&k) { + if let Some(v) = v.as_bool() { + if v { + return Some(true); + } + res = Some(v); + } + } + } + res + } + _ => None, + } +} + +pub async fn require_is_writer( + authed: &ApiAuthed, + path: &str, + w_id: &str, + db: DB, + query: &str, + kind: &str, +) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + if require_owner_of_path(authed, path).is_ok() { + return Ok(()); + } + if path.starts_with("f/") && path.split('/').count() >= 2 { + let folder = path.split('/').nth(1).unwrap(); + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let is_folder_writer = + get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); + if is_folder_writer { + return Ok(()); + } + } + } + let extra_perms = sqlx::query_scalar(query) + .bind(path) + .bind(w_id) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let perm = get_perm_in_extra_perms_for_authed(perms, authed); + match perm { + Some(true) => Ok(()), + Some(false) => Err(Error::BadRequest(format!( + "User {} is not a writer of {kind} path {path}", + authed.username + ))), + None => Err(Error::BadRequest(format!( + "User {} has neither read or write permission on {kind} {path}", + authed.username + ))), + } + } else { + Err(Error::BadRequest(format!( + "{path} does not exist yet and user {} is not an owner of the parent folder", + authed.username + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be writer of an empty path" + ))) + } +} + +// ------------ Preview access check ------------ + +pub fn require_path_read_access_for_preview( + authed: &ApiAuthed, + path: &Option, +) -> Result<()> { + let Some(path) = path else { + return Ok(()); + }; + + if authed.is_admin { + return Ok(()); + } + + if path.is_empty() { + return Ok(()); + } + + let splitted: Vec<&str> = path.split('/').collect(); + if splitted.len() < 2 { + return Err(Error::BadRequest(format!( + "Invalid path format for preview job: {}", + path + ))); + } + + match splitted[0] { + "u" => { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "You can only run preview jobs in your own namespace (u/{}) or in folders you have read access to", + authed.username + ))) + } + } + "f" => { + let folder = splitted[1]; + if authed.folders.iter().any(|(f, _, _)| f == folder) { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "You do not have read access to folder '{}'. Preview jobs require at least read access to the target folder.", + folder + ))) + } + } + "hub" => Ok(()), + _ => Err(Error::BadRequest(format!( + "Invalid path format for preview job: {}. Path must start with 'u/' or 'f/'", + path + ))), + } +} diff --git a/backend/windmill-api/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs similarity index 97% rename from backend/windmill-api/src/scopes.rs rename to backend/windmill-api-auth/src/scopes.rs index 1c1bdd4f26..26a2b3c74c 100644 --- a/backend/windmill-api/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -230,6 +230,10 @@ fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bo || matches_wildcard(accepted_resource, scope_resource) } +// ───────────────────────────────────────────────────────────────── +// Route-level scope checking +// ───────────────────────────────────────────────────────────────── + /// Available scope domains (top-level API categories) #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ScopeDomain { @@ -883,8 +887,6 @@ mod tests { #[test] fn test_user_example_case() { - // User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"] - // Should grant access because scope "u/dieri/*" falls under required pattern "u/*" let user_scope = ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()])); let required_mixed = ScopeDefinition::new( @@ -893,9 +895,8 @@ mod tests { None, Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]), ); - assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/* + assert!(user_scope.includes(&required_mixed)); - // Another example: scope covers one but not both paths let scope_specific = ScopeDefinition::new( "scripts", "read", @@ -908,9 +909,8 @@ mod tests { None, Some(vec!["folder/file1".to_string(), "other/file2".to_string()]), ); - assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly + assert!(scope_specific.includes(&required_multi)); - // Test bidirectional pattern matching more explicitly let scope_broad = ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); let required_specific = ScopeDefinition::new( @@ -919,7 +919,7 @@ mod tests { None, Some(vec!["u/dieri/script.py".to_string()]), ); - assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py + assert!(scope_broad.includes(&required_specific)); let scope_specific_path = ScopeDefinition::new( "scripts", @@ -929,6 +929,6 @@ mod tests { ); let required_broad = ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); - assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/* + assert!(scope_specific_path.includes(&required_broad)); } } diff --git a/backend/windmill-api-embeddings/Cargo.toml b/backend/windmill-api-embeddings/Cargo.toml new file mode 100644 index 0000000000..ff7c3435aa --- /dev/null +++ b/backend/windmill-api-embeddings/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "windmill-api-embeddings" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_embeddings" +path = "src/lib.rs" + +[features] +default = [] +embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-store.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +reqwest.workspace = true +tokio.workspace = true +tracing.workspace = true +lazy_static.workspace = true +anyhow.workspace = true +tinyvector = { workspace = true, optional = true } +hf-hub = { workspace = true, optional = true } +tokenizers = { workspace = true, optional = true } +candle-core = { workspace = true, optional = true } +candle-transformers = { workspace = true, optional = true } +candle-nn = { workspace = true, optional = true } diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs new file mode 100644 index 0000000000..1225f789c3 --- /dev/null +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -0,0 +1,637 @@ +#[cfg(feature = "embedding")] +use anyhow::{anyhow, Error, Result}; +#[cfg(feature = "embedding")] +use std::{collections::HashMap, path::PathBuf, sync::Arc}; +#[cfg(feature = "embedding")] +use windmill_common::DEFAULT_HUB_BASE_URL; +#[cfg(feature = "embedding")] +use windmill_common::HUB_BASE_URL; + +use axum::Router; + +#[cfg(feature = "embedding")] +use axum::{ + extract::{Path, Query}, + Json, +}; + +#[cfg(feature = "embedding")] +use axum::routing::get; +#[cfg(feature = "embedding")] +use candle_core::{Device, Tensor}; +#[cfg(feature = "embedding")] +use candle_nn::VarBuilder; +#[cfg(feature = "embedding")] +use candle_transformers::models::bert::{BertModel, Config, DTYPE}; +#[cfg(feature = "embedding")] +use hf_hub::api::tokio::Api; +#[cfg(feature = "embedding")] +use serde::Deserialize; +#[cfg(feature = "embedding")] +use serde::Serialize; +#[cfg(feature = "embedding")] +use sqlx::{Pool, Postgres}; +#[cfg(feature = "embedding")] +use tinyvector::{ + db::{Db, Embedding}, + similarity::Distance, +}; +#[cfg(feature = "embedding")] +use tokenizers::Tokenizer; +#[cfg(feature = "embedding")] +use tokio::sync::RwLock; +#[cfg(feature = "embedding")] +use windmill_common::utils::http_get_from_hub; + +#[cfg(feature = "embedding")] +use windmill_common::error::JsonResult; + +#[cfg(feature = "embedding")] +use windmill_store::resources::ResourceType; + +#[cfg(feature = "embedding")] +lazy_static::lazy_static! { + pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); + pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); + pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); + // Match windmill-api's HTTP_CLIENT config: 30s timeout, ACCEPT_INVALID_CERTS support + static ref HTTP_CLIENT: reqwest::Client = windmill_common::utils::configure_client(reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .connect_timeout(std::time::Duration::from_secs(10)) + .timeout(std::time::Duration::from_secs(30)) + .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())) + .build().unwrap(); +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize)] +struct HubScriptsQuery { + text: String, + limit: Option, + kind: Option, + app: Option, +} + +#[cfg(feature = "embedding")] +#[derive(Serialize)] +pub struct HubScriptResult { + ask_id: i64, + id: i64, + version_id: i64, + summary: String, + app: String, + kind: String, + score: f32, +} + +#[cfg(feature = "embedding")] +async fn query_hub_scripts( + Query(query): Query, +) -> JsonResult> { + let embeddings_db = EMBEDDINGS_DB.read().await; + + if let Some(embeddings_db) = embeddings_db.as_ref() { + let results = embeddings_db + .query_hub_scripts(&query.text, query.limit, query.kind, query.app) + .await?; + + Ok(Json(results)) + } else { + Err(windmill_common::error::Error::internal_err( + "Embeddings db not initialized".to_string(), + )) + } +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize)] +struct ResourceTypesQuery { + text: String, + limit: Option, +} + +#[cfg(feature = "embedding")] +#[derive(Serialize)] +pub struct ResourceTypeResult { + name: String, + score: f32, + schema: Option, +} +#[cfg(feature = "embedding")] +async fn query_resource_types( + Query(query): Query, + Path(w_id): Path, +) -> JsonResult> { + let embeddings_db = EMBEDDINGS_DB.read().await; + + if let Some(embeddings_db) = embeddings_db.as_ref() { + let results = embeddings_db + .query_resource_types(w_id, &query.text, query.limit) + .await?; + + Ok(Json(results)) + } else { + Err(windmill_common::error::Error::internal_err( + "Embeddings db not initialized".to_string(), + )) + } +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize, Debug, Clone)] +struct HubScript { + ask_id: i64, + id: i64, + version_id: i64, + summary: String, + app: String, + kind: String, + embedding: Vec, +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize, Debug)] +struct HubResourceType { + name: String, + embedding: Vec, +} + +#[cfg(feature = "embedding")] +pub struct ModelInstance { + model: BertModel, + tokenizer: Tokenizer, +} + +#[cfg(feature = "embedding")] +impl ModelInstance { + pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> { + let api = Api::new()?; + let repo_api = api.model("thenlper/gte-small".to_string()); + + let (config_filename, tokenizer_filename, weights_filename) = + ( + repo_api + .get("config.json") + .await + .map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?, + repo_api.get("tokenizer.json").await.map_err(|e| { + anyhow!("Failed to get tokenizer.json from hugging face: {}", e) + })?, + repo_api.get("model.safetensors").await.map_err(|e| { + anyhow!("Failed to get model.safetensors from hugging face: {}", e) + })?, + ); + + Ok((config_filename, tokenizer_filename, weights_filename)) + } + + pub async fn new() -> Result { + tracing::info!("Loading embedding model..."); + let device = Device::Cpu; + let (config_filename, tokenizer_filename, weights_filename) = + Self::load_model_files().await?; + let config = std::fs::read_to_string(config_filename)?; + let config: Config = serde_json::from_str(&config)?; + let tokenizer = Tokenizer::from( + Tokenizer::from_file(tokenizer_filename) + .map_err(Error::msg)? + .with_padding(None) + .to_owned(), + ); + + let vb = + unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? }; + let model = BertModel::load(vb, &config)?; + tracing::info!("Loaded embedding model"); + Ok(Self { model, tokenizer }) + } + + pub async fn create_embedding(self: Arc, sentence: &str) -> Result> { + let sentence = sentence.to_owned(); + tokio::task::spawn_blocking(move || { + let tokens = self + .tokenizer + .encode(sentence, true) + .map_err(Error::msg)? + .get_ids() + .to_vec(); + + let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?; + let token_type_ids = token_ids.zeros_like()?; + + let embedding = self.model.forward(&token_ids, &token_type_ids, None)?; + let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?; + let embedding = normalize_l2(&embedding)?; + + let embedding = embedding.get(0)?.to_vec1()?; + + Ok(embedding) + }) + .await? + } +} + +#[cfg(feature = "embedding")] +pub struct EmbeddingsDb { + db: Db, + model_instance: Arc, +} + +#[cfg(feature = "embedding")] +impl EmbeddingsDb { + pub async fn new(pg_db: &Pool, model_instance: Arc) -> Result { + let db = Db::new(); + + let mut embeddings_db = Self { db, model_instance: model_instance.clone() }; + + embeddings_db.fill_db(pg_db).await?; + + Ok(embeddings_db) + } + + async fn fill_db(&mut self, pg_db: &Pool) -> Result<()> { + if self.db.get_collection("scripts").is_some() { + self.db.delete_collection("scripts")?; + } + + self.db + .create_collection("scripts".to_string(), 384, Distance::Cosine)?; + + if self.db.get_collection("resource_types").is_some() { + self.db.delete_collection("resource_types")?; + } + + self.db + .create_collection("resource_types".to_string(), 384, Distance::Cosine)?; + + let hub_base_url = HUB_BASE_URL.read().await.clone(); + + let response = match hub_base_url.as_str() { + DEFAULT_HUB_BASE_URL => { + let response = HTTP_CLIENT + .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json") + .send() + .await; + + if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { + tracing::warn!("Failed to get scripts embeddings from bucket, trying hub..."); + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } else { + response.unwrap() + } + } + _ => { + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } + }; + + if response.error_for_status_ref().is_err() { + return Err(anyhow!( + "Failed to get scripts embeddings from hub with error code: {}", + response.status() + )); + } + + let hub_scripts = response.json::>().await?; + + for script in &hub_scripts { + let mut hm = HashMap::new(); + hm.insert("ask_id".to_string(), script.ask_id.clone().to_string()); + hm.insert("summary".to_string(), script.summary.clone()); + hm.insert("app".to_string(), script.app.clone()); + hm.insert("kind".to_string(), script.kind.clone()); + hm.insert("id".to_string(), script.id.clone().to_string()); + hm.insert( + "version_id".to_string(), + script.version_id.clone().to_string(), + ); + let embedding = Embedding { + id: script.ask_id.clone().to_string(), + vector: script.embedding.clone(), + metadata: Some(hm), + }; + self.db.insert_into_collection("scripts", embedding)?; + } + + let response = match hub_base_url.as_str() { + DEFAULT_HUB_BASE_URL => { + let response = HTTP_CLIENT + .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json") + .send() + .await; + if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { + tracing::warn!( + "Failed to get resource types embeddings from bucket, trying hub..." + ); + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/resource_types/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } else { + response.unwrap() + } + } + _ => { + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/resource_types/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } + }; + + if response.error_for_status_ref().is_err() { + return Err(anyhow!( + "Failed to get resource types embeddings from hub with error code: {}", + response.status() + )); + } + let hub_resource_types = response.json::>().await?; + + let resource_types: Vec = + sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",) + .fetch_all(pg_db) + .await?; + + for rt in resource_types { + let mut hm = HashMap::new(); + hm.insert("name".to_string(), rt.name.clone()); + if let Some(schema) = rt.schema.clone() { + hm.insert("schema".to_string(), serde_json::to_string(&schema)?); + } + hm.insert("workspace".to_string(), rt.workspace_id.clone()); + let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name); + + let vector = if let Some(hub_rt) = hub_rt { + hub_rt.embedding.clone() + } else { + self.model_instance + .clone() + .create_embedding(&format!( + "{};{}", + rt.name, + rt.description.unwrap_or_default() + )) + .await? + }; + + let embedding = Embedding { + id: format!("{}_{}", rt.workspace_id, rt.name), + vector, + metadata: Some(hm), + }; + + self.db + .insert_into_collection("resource_types", embedding)?; + } + + Ok(()) + } + + pub async fn query_hub_scripts( + &self, + query: &str, + limit: Option, + kind: Option, + app: Option, + ) -> Result> { + let model_instance = self.model_instance.clone(); + let query_embedding = model_instance.create_embedding(query).await?; + + let collection = self.db.get_collection("scripts"); + + let collection = collection.ok_or(Error::msg("no collection found"))?; + + let filter = |embedding: &Embedding| { + if let Some(metadata) = embedding.metadata.as_ref() { + match ( + metadata.get("kind"), + kind.clone(), + metadata.get("app"), + app.clone(), + ) { + (Some(script_kind), Some(kind), Some(script_app), Some(app)) => { + &kind == script_kind && &app == script_app + } + (Some(script_kind), Some(kind), _, _) => &kind == script_kind, + (_, _, Some(script_app), Some(app)) => &app == script_app, + (_, None, _, None) => true, + _ => false, + } + } else { + false + } + }; + + let results = collection.get_similarity( + &query_embedding, + limit.unwrap_or(10) as usize, + Some(&filter), + Some(0.8), + ); + + let results: Result> = results + .iter() + .map(|r| { + let metadata = r + .embedding + .metadata + .as_ref() + .ok_or(Error::msg("no metadata"))?; + + Ok(HubScriptResult { + ask_id: metadata + .get("ask_id") + .ok_or(Error::msg("no ask_id"))? + .parse::()?, + summary: metadata + .get("summary") + .ok_or(Error::msg("no summary"))? + .to_owned(), + app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(), + kind: metadata + .get("kind") + .ok_or(Error::msg("no kind"))? + .to_owned(), + id: metadata + .get("id") + .ok_or(Error::msg("no id"))? + .parse::()?, + version_id: metadata + .get("version_id") + .ok_or(Error::msg("no version_id"))? + .parse::()?, + score: r.score, + }) + }) + .collect(); + + let mut results = results?; + + if results.len() > 1 { + let top_score = results[0].score; + results = results + .into_iter() + .take_while(|r| (top_score - r.score) / top_score <= 0.05) + .collect(); + } + + Ok(results) + } + + pub async fn query_resource_types( + &self, + workspace: String, + query: &str, + limit: Option, + ) -> Result> { + let model_instance = self.model_instance.clone(); + let query_embedding = model_instance.create_embedding(query).await?; + + let collection = self.db.get_collection("resource_types"); + + if collection.is_none() { + return Ok(vec![]); + } + + let collection = collection.ok_or(Error::msg("no collection found"))?; + + let filter = |embedding: &Embedding| { + if let Some(metadata) = embedding.metadata.as_ref() { + match metadata.get("workspace").map(|x| x.as_str()) { + Some("admins") => true, + Some(rt_workspace) => &workspace == rt_workspace, + _ => false, + } + } else { + false + } + }; + + let results = collection.get_similarity( + &query_embedding, + limit.unwrap_or(10) as usize, + Some(&filter), + Some(0.75), + ); + + let results: Result<_> = results + .iter() + .map(|r| { + let metadata = r + .embedding + .metadata + .as_ref() + .ok_or(Error::msg("no metadata"))?; + Ok(ResourceTypeResult { + name: metadata + .get("name") + .ok_or(Error::msg("no name"))? + .to_owned(), + schema: match metadata.get("schema") { + Some(schema) => serde_json::from_str(schema)?, + None => None, + }, + score: r.score, + }) + }) + .collect(); + + results + } +} + +#[cfg(feature = "embedding")] +fn normalize_l2(v: &Tensor) -> Result { + Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?) +} + +#[cfg(feature = "embedding")] +pub fn load_embeddings_db(db: &Pool) -> () { + let disable_embedding = std::env::var("DISABLE_EMBEDDING") + .ok() + .map(|x| x.parse::().unwrap_or(false)) + .unwrap_or(false); + + if !disable_embedding { + let db_clone = db.clone(); + tokio::spawn(async move { + let model_instance = ModelInstance::new().await; + if let Ok(model_instance) = model_instance { + let mut model_instance_lock = MODEL_INSTANCE.write().await; + *model_instance_lock = Some(Arc::new(model_instance)); + drop(model_instance_lock); + loop { + update_embeddings_db(&db_clone).await; + tokio::time::sleep(std::time::Duration::from_secs( + *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, + )) + .await; + } + } else { + tracing::error!( + "Failed to initialize model instance: {}", + model_instance.err().unwrap() + ); + } + }); + } +} + +#[cfg(feature = "embedding")] +pub async fn update_embeddings_db(db: &Pool) -> () { + if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { + tracing::info!("Creating embeddings DB..."); + let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; + if let Err(e) = new_embeddings_db.as_ref() { + tracing::error!("Failed to create embeddings db: {}", e); + } else { + let mut embeddings_db = EMBEDDINGS_DB.write().await; + *embeddings_db = new_embeddings_db.ok(); + tracing::info!("Created embeddings DB"); + } + } else { + tracing::error!("Could not update embeddings DB, model instance not initialized"); + } +} + +#[cfg(feature = "embedding")] +pub fn workspaced_service() -> Router { + Router::new().route("/query_resource_types", get(query_resource_types)) +} + +#[cfg(feature = "embedding")] +pub fn global_service() -> Router { + Router::new().route("/query_hub_scripts", get(query_hub_scripts)) +} + +#[cfg(not(feature = "embedding"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "embedding"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api-groups/Cargo.toml b/backend/windmill-api-groups/Cargo.toml new file mode 100644 index 0000000000..f785b9ebed --- /dev/null +++ b/backend/windmill-api-groups/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "windmill-api-groups" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_groups" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +private = ["windmill-common/private", "windmill-api-workspaces/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-api-workspaces.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true + +axum.workspace = true +chrono.workspace = true +lazy_static.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +sql-builder.workspace = true +sqlx.workspace = true +uuid.workspace = true diff --git a/backend/windmill-api-groups/src/folder_history.rs b/backend/windmill-api-groups/src/folder_history.rs new file mode 100644 index 0000000000..b11f328e3d --- /dev/null +++ b/backend/windmill-api-groups/src/folder_history.rs @@ -0,0 +1,68 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Extension, Path, Query}, + routing::get, + Router, +}; +use windmill_api_auth::ApiAuthed; +use windmill_common::{ + db::UserDB, + error::JsonResult, + utils::{paginate, Pagination}, +}; + +use serde::Serialize; +use sqlx::FromRow; + +pub fn workspaced_service() -> Router { + Router::new().route("/get/:name", get(get_folder_permission_history)) +} + +#[derive(Serialize, FromRow)] +pub struct FolderPermissionChange { + pub id: i64, + pub changed_by: String, + pub changed_at: chrono::DateTime, + pub change_type: String, + pub affected: Option, +} + +async fn get_folder_permission_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, + Query(pagination): Query, +) -> JsonResult> { + // Check if user is owner of the folder (before starting transaction for performance) + crate::folders::require_is_owner(&authed, &name)?; + + let mut tx = user_db.begin(&authed).await?; + + let (per_page, offset) = paginate(pagination); + + let history = sqlx::query_as!( + FolderPermissionChange, + "SELECT id, changed_by, changed_at, change_type, affected + FROM folder_permission_history + WHERE workspace_id = $1 AND folder_name = $2 + ORDER BY id DESC + LIMIT $3 OFFSET $4", + w_id, + name, + per_page as i64, + offset as i64 + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(axum::Json(history)) +} diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs new file mode 100644 index 0000000000..1a4d530462 --- /dev/null +++ b/backend/windmill-api-groups/src/folders.rs @@ -0,0 +1,829 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::sync::Arc; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use lazy_static::lazy_static; +use regex::Regex; +use windmill_api_auth::{check_scopes, ApiAuthed, AuthCache, Tokened}; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::webhook::{WebhookMessage, WebhookShared}; +use windmill_common::DB; +use windmill_common::{ + db::UserDB, + error::{self, to_anyhow, JsonResult, Result}, + users::username_to_permissioned_as, + utils::{not_found_if_none, paginate, Pagination}, +}; + +use serde::{Deserialize, Serialize}; +use sqlx::{FromRow, Postgres, Transaction}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_folders)) + .route("/listnames", get(list_foldernames)) + .route("/create", post(create_folder)) + .route("/get/:name", get(get_folder)) + .route("/exists/:name", get(exists_folder)) + .route("/update/:name", post(update_folder)) + .route("/getusage/:name", get(get_folder_usage)) + .route("/delete/:name", delete(delete_folder)) + .route("/addowner/:name", post(add_owner)) + .route("/removeowner/:name", post(remove_owner)) + .route("/is_owner/*path", get(is_owner_api)) +} + +#[derive(FromRow, Serialize, Deserialize, Clone)] +pub struct Folder { + pub workspace_id: String, + pub name: String, + pub display_name: String, + pub owners: Vec, + pub extra_perms: serde_json::Value, + pub summary: Option, + pub created_by: Option, + pub edited_at: Option>, +} + +#[derive(Deserialize)] +pub struct NewFolder { + pub name: String, + pub summary: Option, + pub display_name: Option, + pub owners: Option>, + pub extra_perms: Option, +} + +#[derive(Deserialize)] +pub struct UpdateFolder { + pub summary: Option, + pub display_name: Option, + pub owners: Option>, + pub extra_perms: Option, +} + +#[derive(Deserialize)] +pub struct Owner { + pub owner: String, + pub write: Option, +} + +async fn list_folders( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let mut tx = user_db.begin(&authed).await?; + + let rows = sqlx::query_as!( + Folder, + "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", + w_id, + per_page as i64, + offset as i64 + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(rows)) +} +async fn list_foldernames( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let mut tx = user_db.begin(&authed).await?; + + let rows = sqlx::query_scalar!( + "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", + w_id, + per_page as i64, + offset as i64 + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(rows)) +} + +fn validate_owner(owner: &str) -> Result<()> { + if !owner + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-') + { + return Err(error::Error::BadRequest( + "Invalid owner: must contain only alphanumeric characters, underscores, hyphens, or slashes".to_string(), + )); + } + Ok(()) +} + +async fn check_name_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + name: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", + name, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Folder {} already exists", + name + ))); + } + return Ok(()); +} + +lazy_static! { + static ref VALID_FOLDER_NAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap(); +} + +async fn create_folder( + authed: ApiAuthed, + Tokened { token }: Tokened, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Extension(cache): Extension>, + Path(w_id): Path, + Json(ng): Json, +) -> Result { + let mut tx = user_db.clone().begin(&authed).await?; + + if !VALID_FOLDER_NAME.is_match(&ng.name) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Folder name can only contain alphanumeric characters, underscores" + ))); + } + check_name_conflict(&mut tx, &w_id, &ng.name).await?; + cache.invalidate(&w_id, token).await; + let owner = username_to_permissioned_as(&authed.username); + let owners = ng.owners.unwrap_or_else(|| vec![owner.clone()]); + let owners = if owners.contains(&owner) { + owners.clone() + } else { + owners + .iter() + .cloned() + .chain(std::iter::once(owner)) + .collect() + }; + + let mut extra_perms = ng + .extra_perms + .unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new())); + + if extra_perms.is_object() { + let extra_mut = extra_perms.as_object_mut().unwrap(); + for o in &owners { + extra_mut.insert(o.clone(), serde_json::json!(true)); + } + } else { + return Err(error::Error::BadRequest( + "extra_perms must be an object".to_string(), + )); + } + + if let Err(e) = + sqlx::query_as!( + Folder, + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + w_id, + ng.name, + ng.display_name.unwrap_or(ng.name.clone()), + &owners, + extra_perms, + ng.summary, + authed.username + ) + .execute(&mut *tx) + .await { + drop(tx); + let mut tx = user_db.begin(&authed).await?; + + let exists_for_user = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2 AND $3 = ANY(owners))", + ng.name, + w_id, + authed.username + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", + ng.name, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !exists_for_user && exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Folder '{}' already exists in workspace '{}' but you do not have permission to read to it", ng.name, w_id + ))); + } else if exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Folder '{}' already exists in workspace '{}'", ng.name, w_id + ))); + } else { + return Err(windmill_common::error::Error::InternalErr(format!( + "Failed to create folder: {}", e + ))); + } + } + + audit_log( + &mut *tx, + &authed, + "folder.create", + ActionKind::Create, + &w_id, + Some(&ng.name.to_string()), + None, + ) + .await?; + + log_folder_permission_change(&mut *tx, &w_id, &ng.name, &authed.username, "create", None) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Folder { path: format!("f/{}", ng.name) }, + Some(format!("Folder '{}' created", ng.name)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateFolder { workspace: w_id, name: ng.name.clone() }, + ); + + Ok(format!("Created folder {}", ng.name)) +} + +pub async fn is_owner_api( + authed: ApiAuthed, + Path((_w_id, name)): Path<(String, String)>, +) -> JsonResult { + Ok(Json(is_owner(&authed, &name))) +} + +use windmill_api_auth::is_owner; +pub use windmill_api_auth::require_is_owner; + +async fn update_folder( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(mut ng): Json, +) -> Result { + use sql_builder::prelude::*; + + let mut sqlb = SqlBuilder::update_table("folder"); + sqlb.and_where_eq("name", "?".bind(&name)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + + if let Some(display_name) = ng.display_name { + sqlb.set("display_name", "?".bind(&display_name)); + } + + if let Some(summary) = ng.summary { + sqlb.set("summary", "?".bind(&summary)); + } + + sqlb.set("edited_at", "now()"); + + // Track whether permission-related fields are being updated + let owners_changed = ng.owners.is_some(); + let extra_perms_changed = ng.extra_perms.is_some(); + + if !authed.is_admin { + let prefixed_username = format!("u/{}", authed.username); + if ng.owners.as_ref().is_some_and(|x| { + !x.contains(&prefixed_username) + && !authed.groups.iter().any(|g| x.contains(&format!("g/{g}"))) + }) { + ng.owners.as_mut().unwrap().push(prefixed_username.clone()); + if ng.extra_perms.is_none() { + ng.extra_perms = Some(serde_json::Value::Object(serde_json::Map::new())); + } + ng.extra_perms + .as_mut() + .unwrap() + .as_object_mut() + .unwrap() + .insert(prefixed_username, serde_json::json!(true)); + } + } + if let Some(owners) = ng.owners { + sqlb.set( + "owners", + "?".bind(&format!( + "{{{}}}", + owners + .iter() + .map(|x| format!("\"{x}\"")) + .collect::>() + .join(","), + )), + ); + } + if let Some(extra_perms) = ng.extra_perms { + if !extra_perms.is_object() { + return Err(windmill_common::error::Error::BadRequest(format!( + "extra_perms must be an object, received {}", + extra_perms.to_string() + ))); + } + sqlb.set( + "extra_perms", + "?".bind(&serde_json::to_string(&extra_perms).map_err(to_anyhow)?), + ); + } + + sqlb.returning("*"); + + let mut tx = user_db.begin(&authed).await?; + + let sql = sqlb + .sql() + .map_err(|e| error::Error::internal_err(e.to_string()))?; + let nfolder = sqlx::query_as::<_, Folder>(&sql) + .fetch_optional(&mut *tx) + .await?; + + let nfolder = nfolder.ok_or_else(|| { + windmill_common::error::Error::NotAuthorized(format!( + "You are not an owner of {} and hence cannot modify it", + name + )) + })?; + + if let Some(extra_perms) = nfolder.extra_perms.as_object() { + for o in nfolder.owners { + if !extra_perms + .get(&o) + .and_then(|x| x.as_bool()) + .unwrap_or(false) + { + return Err(windmill_common::error::Error::BadRequest(format!( + "Owner {} would not have permission to write to folder and that is an invalid state", + o + ))); + } + } + } + + audit_log( + &mut *tx, + &authed, + "folder.update", + ActionKind::Update, + &w_id, + Some(&name.to_string()), + None, + ) + .await?; + + // Log permission changes if owners or extra_perms were updated + if owners_changed { + log_folder_permission_change( + &mut *tx, + &w_id, + &name, + &authed.username, + "update_owners", + None, + ) + .await?; + } + if extra_perms_changed { + log_folder_permission_change(&mut *tx, &w_id, &name, &authed.username, "update_acl", None) + .await?; + } + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Folder { path: format!("f/{}", name) }, + Some(format!("Folder '{}' updated", name)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone().clone(), + WebhookMessage::UpdateFolder { workspace: w_id, name: name.to_owned() }, + ); + + Ok(format!("Updated folder {}", name)) +} + +pub async fn get_folderopt<'c>( + db: &mut Transaction<'c, Postgres>, + w_id: &str, + name: &str, +) -> Result> { + let folderopt = sqlx::query_as!( + Folder, + "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE name = $1 AND workspace_id = $2", + name, + w_id + ) + .fetch_optional(&mut **db) + .await?; + Ok(folderopt) +} + +async fn get_folder( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || format!("folders:read:f/{}", name))?; + let mut tx = user_db.begin(&authed).await?; + + let folder = not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; + + tx.commit().await?; + Ok(Json(folder)) +} + +async fn exists_folder( + Extension(db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", + name, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +#[derive(Serialize)] +struct FolderUsage { + pub scripts: i64, + pub schedules: i64, + pub flows: i64, + pub apps: i64, + pub resources: i64, + pub variables: i64, +} +async fn get_folder_usage( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + check_scopes(&authed, || format!("folders:read:f/{}", name))?; + let mut tx = user_db.begin(&authed).await?; + + let scripts = sqlx::query_scalar!( + "SELECT count(path) FROM script WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + let schedules = sqlx::query_scalar!( + "SELECT count(path) FROM schedule WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + let flows = sqlx::query_scalar!( + "SELECT count(path) FROM flow WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + let apps = sqlx::query_scalar!( + "SELECT count(path) FROM app WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + let raw_apps = sqlx::query_scalar!( + "SELECT count(path) FROM raw_app WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + let resources = sqlx::query_scalar!( + "SELECT count(path) FROM resource WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + let variables = sqlx::query_scalar!( + "SELECT count(path) FROM variable WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", + name, + w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(0); + + tx.commit().await?; + Ok(Json(FolderUsage { + scripts, + flows, + schedules, + apps: apps + raw_apps, + resources, + variables, + })) +} + +async fn delete_folder( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; + + let del = sqlx::query_scalar!( + "DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1", + name, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + if del.is_none() { + return Err(windmill_common::error::Error::NotAuthorized(format!( + "Not authorized to delete folder {}", + name + ))); + } + + audit_log( + &mut *tx, + &authed, + "folder.delete", + ActionKind::Delete, + &w_id, + Some(&name.to_string()), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Folder { path: format!("f/{}", name) }, + Some(format!("Folder '{}' deleted", name)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteFolder { workspace: w_id, name: name.clone() }, + ); + + Ok(format!("delete folder at name {}", name)) +} + +async fn add_owner( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(Owner { owner, .. }): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; + require_is_owner(&authed, &name)?; + + sqlx::query!( + "UPDATE folder SET owners = array_append(owners::text[], $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name", + owner, + &name, + &w_id, + ) + .fetch_optional(&mut *tx) + .await?; + + validate_owner(&owner)?; + sqlx::query(&format!( + "UPDATE folder SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \ + true) WHERE name = $2 AND workspace_id = $3 RETURNING extra_perms" + )) + .bind(true) + .bind(&name) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "folder.add_owner", + ActionKind::Update, + &w_id, + Some(&name.to_string()), + Some([("owner", owner.as_str())].into()), + ) + .await?; + + log_folder_permission_change( + &mut *tx, + &w_id, + &name, + &authed.username, + "grant_admin", + Some(&owner), + ) + .await?; + + tx.commit().await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateFolder { workspace: w_id, name: name.clone() }, + ); + + Ok(format!("Added {} to folder {}", owner, name)) +} + +async fn remove_owner( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(Owner { owner, write }): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; + require_is_owner(&authed, &name)?; + validate_owner(&owner)?; + + let folder = sqlx::query!( + "UPDATE folder SET owners = array_remove(owners, $1::varchar) WHERE name = $2 AND workspace_id = $3 AND $1 = ANY(owners) RETURNING name", + owner, + &name, + &w_id, + ) + .fetch_optional(&mut *tx) + .await?; + + if folder.is_none() && write.is_none() { + return Ok(format!( + "Owner {} is already not a member of folder {}", + owner, name + )); + } + + if let Some(write) = write { + let old_write = sqlx::query_scalar::<_, Option>(&format!( + "UPDATE folder SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \ + true) FROM (SELECT (extra_perms->>'{owner}')::boolean as old_val FROM folder WHERE name = $2 AND workspace_id = $3) old \ + WHERE name = $2 AND workspace_id = $3 RETURNING old.old_val" + )) + .bind(write) + .bind(&name) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + if folder.is_none() && old_write.is_none_or(|ow| ow == write) { + return Ok(format!( + "Owner {} is already not a member of folder {} and write permission was already {}", + owner, name, write + )); + } + } + + audit_log( + &mut *tx, + &authed, + "folder.remove_owner", + ActionKind::Update, + &w_id, + Some(&name.to_string()), + Some([("owner", owner.as_str())].into()), + ) + .await?; + + let change_type = match write { + Some(true) => "grant_writer_only", + Some(false) => "grant_viewer_only", + None => "revoke_all", + }; + log_folder_permission_change( + &mut *tx, + &w_id, + &name, + &authed.username, + change_type, + Some(&owner), + ) + .await?; + + tx.commit().await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateFolder { workspace: w_id, name: name.clone() }, + ); + + Ok(format!("Removed {} to folder {}", owner, name)) +} + +pub async fn log_folder_permission_change<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + folder_name: &str, + changed_by: &str, + change_type: &str, + affected: Option<&str>, +) -> Result<()> { + sqlx::query!( + "INSERT INTO folder_permission_history + (workspace_id, folder_name, changed_by, change_type, affected) + VALUES ($1, $2, $3, $4, $5)", + workspace_id, + folder_name, + changed_by, + change_type, + affected + ) + .execute(db) + .await?; + Ok(()) +} diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs new file mode 100644 index 0000000000..048a4120f5 --- /dev/null +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -0,0 +1,396 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Extension, Path}, + routing::{get, post}, + Json, Router, +}; +use windmill_api_auth::require_owner_of_path; +use windmill_common::DB; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; + +use windmill_api_auth::ApiAuthed; + +use serde::{Deserialize, Serialize}; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{not_found_if_none, StripPath}, +}; + +const KINDS: [&str; 18] = [ + "script", + "group_", + "resource", + "schedule", + "variable", + "flow", + "folder", + "app", + "raw_app", + "http_trigger", + "websocket_trigger", + "kafka_trigger", + "nats_trigger", + "postgres_trigger", + "mqtt_trigger", + "gcp_trigger", + "sqs_trigger", + "email_trigger", +]; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/get/*path", get(get_granular_acls)) + .route("/add/*path", post(add_granular_acl)) + .route("/remove/*path", post(remove_granular_acl)) +} + +#[derive(Serialize, Deserialize)] +pub struct GranularAcl { + pub owner: String, + pub write: Option, +} + +async fn add_granular_acl( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(GranularAcl { owner, write }): Json, +) -> Result { + let path = path.to_path(); + + let (kind, path) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?; + + if !KINDS.contains(&kind) { + return Err(Error::BadRequest("Invalid kind".to_string())); + } + + let mut tx = user_db.begin(&authed).await?; + + let identifier = if kind == "group_" || kind == "folder" { + "name" + } else { + "path" + }; + + if !authed.is_admin { + if kind == "folder" { + crate::folders::require_is_owner(&authed, path)?; + } else if kind == "group_" { + crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) + .await?; + } else { + require_owner_of_path(&authed, path)?; + } + } + + if kind == "folder" { + if let Some(obj) = sqlx::query_scalar!( + "SELECT owners FROM folder WHERE name = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + if obj.contains(&owner) { + if write != Some(true) { + return Err(Error::BadRequest( + "Cannot remove write permission for folder owner".to_string(), + )); + } + } + } + } + + let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( + "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \ + true) WHERE {identifier} = $3 AND workspace_id = $4 RETURNING extra_perms" + )) + .bind(vec![owner.clone()]) + .bind(write.unwrap_or(false)) + .bind(path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + let _ = not_found_if_none(obj_o, &kind, &path)?; + + if kind == "folder" { + let change_type = if write.unwrap_or(false) { + "grant_read" + } else { + "grant_write" + }; + crate::folders::log_folder_permission_change( + &mut *tx, + &w_id, + path, + &authed.username, + change_type, + Some(&owner), + ) + .await?; + } else if kind == "group_" { + let change_type = if write.unwrap_or(false) { + "grant_admin" + } else { + "grant_member_only" + }; + crate::groups::log_group_permission_change( + &mut *tx, + &w_id, + path, + &authed.username, + change_type, + Some(&owner), + ) + .await?; + } + + tx.commit().await?; + + match kind { + "folder" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Folder { path: format!("f/{}", path) }, + Some(format!("Folder '{}' changed permissions", path)), + true, + None, + ) + .await? + } + // "app" => { + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 }, + // Some(format!("App '{}' changed permissions", path)), + // // true, + // ) + // .await? + // } + // "script" => { + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::Script { + // path: path.to_string(), + // parent_path: None, + // hash: ScriptHash(0), + // }, + // Some(format!("Script '{}' changed permissions", path)), + // // true, + // ) + // .await? + // } + // "flow" => { + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::Flow { path: path.to_string(), parent_path: None }, + // Some(format!("Flow '{}' changed permissions", path)), + // // true, + // ) + // .await? + // } + _ => (), + } + + Ok("Successfully modified granular acl".to_string()) +} + +async fn remove_granular_acl( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(GranularAcl { owner, .. }): Json, +) -> Result { + let path = path.to_path(); + + let (kind, path) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?; + + if !KINDS.contains(&kind) { + return Err(Error::BadRequest("Invalid kind".to_string())); + } + + if !authed.is_admin { + if kind == "folder" { + crate::folders::require_is_owner(&authed, path)?; + } else if kind == "group_" { + crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) + .await?; + } else { + require_owner_of_path(&authed, path)?; + } + } + + let mut tx = user_db.begin(&authed).await?; + + let identifier = if kind == "group_" || kind == "folder" { + "name" + } else { + "path" + }; + + if identifier == "path" { + require_owner_of_path(&authed, path)?; + } + + let obj_o = sqlx::query_scalar::<_, bool>(&format!( + "WITH old AS ( + SELECT extra_perms->$1 as old_write FROM {kind} + WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1 + ) + UPDATE {kind} SET extra_perms = extra_perms - $1 + WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1 + RETURNING (SELECT old_write FROM old)::bool" + )) + .bind(&owner) + .bind(path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + // Only log if something was actually removed (obj_o is Some) + if let Some(write) = obj_o { + // Log permission changes for folders and groups + if kind == "folder" { + let change_type = if write { "revoke_write" } else { "revoke_read" }; + crate::folders::log_folder_permission_change( + &mut *tx, + &w_id, + path, + &authed.username, + change_type, + Some(&owner), + ) + .await?; + } else if kind == "group_" { + crate::groups::log_group_permission_change( + &mut *tx, + &w_id, + path, + &authed.username, + "revoke_admin", + Some(&owner), + ) + .await?; + } + + tx.commit().await?; + + match kind { + "folder" => { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Folder { path: format!("f/{}", path) }, + Some(format!("Folder '{}' changed permissions", path)), + true, + None, + ) + .await? + } + // "app" => { + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 }, + // Some(format!("App '{}' changed permissions", path)), + // // true, + // ) + // .await? + // } + // "script" => { + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::Script { + // path: path.to_string(), + // parent_path: None, + // hash: ScriptHash(0), + // }, + // Some(format!("Script '{}' changed permissions", path)), + // // true, + // ) + // .await? + // } + // "flow" => { + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::Flow { path: path.to_string(), parent_path: None }, + // Some(format!("Flow '{}' changed permissions", path)), + // // true, + // ) + // .await? + // } + _ => (), + } + } + + Ok("Successfully removed granular acl".to_string()) +} + +async fn get_granular_acls( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let (kind, path) = path + .split_once('/') + .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?; + + if !KINDS.contains(&kind) { + return Err(Error::BadRequest("Invalid kind".to_string())); + } + + let mut tx = user_db.begin(&authed).await?; + + let identifier = if kind == "group_" { "name" } else { "path" }; + let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( + "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" + )) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + + let obj = not_found_if_none(obj_o, &kind, &path)?; + tx.commit().await?; + + Ok(Json(obj)) +} diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs new file mode 100644 index 0000000000..a68b72072d --- /dev/null +++ b/backend/windmill-api-groups/src/groups.rs @@ -0,0 +1,1092 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_api_auth::{require_super_admin, ApiAuthed}; +use windmill_common::DB; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::{ + auth::get_groups_for_user, + error::{Error, JsonResult, Result}, + utils::{not_found_if_none, paginate, Pagination}, +}; +use windmill_common::{db::UserDB, users::username_to_permissioned_as}; + +use serde::{Deserialize, Serialize}; +use sqlx::{query_scalar, FromRow, Postgres, Transaction}; +use windmill_git_sync::handle_deployment_metadata; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_groups)) + .route("/listnames", get(list_group_names)) + .route("/create", post(create_group)) + .route("/get/:name", get(get_group)) + .route("/update/:name", post(update_group)) + .route("/delete/:name", delete(delete_group)) + .route("/adduser/:name", post(add_user)) + .route("/removeuser/:name", post(remove_user)) + .route("/is_owner/:name", get(is_owner)) +} + +pub fn global_service() -> Router { + Router::new() + .route("/list", get(list_igroups)) + .route("/list_with_workspaces", get(list_igroups_with_workspaces)) + .route("/get/:name", get(get_igroup)) + .route("/create", post(create_igroup)) + .route("/update/:name", post(update_igroup)) + .route("/delete/:name", delete(delete_igroup)) + .route("/adduser/:name", post(add_user_igroup)) + .route("/removeuser/:name", post(remove_user_igroup)) + .route("/export", get(export_igroups)) + .route("/overwrite", post(overwrite_igroups)) +} + +/// Normalize group names: replace spaces with underscores and convert to lowercase +/// Used when manually creating groups and SCIM-managed groups +pub fn convert_name(name: &str) -> String { + name.replace(" ", "_").to_lowercase() +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct Group { + pub workspace_id: String, + pub name: String, + pub summary: Option, + pub extra_perms: serde_json::Value, +} + +#[derive(Deserialize)] +pub struct NewGroup { + pub name: String, + pub summary: Option, +} + +#[derive(Serialize)] +pub struct GroupInfo { + pub workspace_id: String, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub members: Vec, + pub extra_perms: serde_json::Value, +} + +#[derive(Deserialize)] +pub struct EditGroup { + pub summary: Option, +} + +#[derive(Deserialize)] +pub struct Username { + pub username: String, +} + +#[derive(Deserialize)] +pub struct Email { + pub email: String, +} + +async fn list_groups( + Extension(db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + + let rows = sqlx::query_as!( + Group, + "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", + w_id, + per_page as i64, + offset as i64 + ) + .fetch_all(&db) + .await?; + + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct QueryListGroup { + pub only_member_of: Option, +} +async fn list_group_names( + ApiAuthed { username, email, .. }: ApiAuthed, + Extension(db): Extension, + Query(QueryListGroup { only_member_of }): Query, + Path(w_id): Path, +) -> JsonResult> { + let rows = if !only_member_of.unwrap_or(false) { + sqlx::query_scalar!( + "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name asc", + w_id + ) + .fetch_all(&db) + .await? + .into_iter() + .filter_map(|x| x) + .collect() + } else { + get_groups_for_user(&w_id, &username, &email, &db).await? + }; + + Ok(Json(rows)) +} + +async fn check_name_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + name: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM group_ WHERE name = $1 AND workspace_id = $2)", + name, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Group {} already exists", + name + ))); + } + return Ok(()); +} + +pub async fn is_owner( + ApiAuthed { username, is_admin, groups, .. }: ApiAuthed, + Extension(db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + if is_admin { + Ok(Json(true)) + } else { + Ok(Json( + require_is_owner(&name, &username, &groups, &w_id, &db) + .await + .is_ok(), + )) + } +} + +pub async fn require_is_owner( + group_name: &str, + username: &str, + groups: &Vec, + w_id: &str, + db: &DB, +) -> Result<()> { + let is_owner = query_scalar!( + "SELECT EXISTS(SELECT 1 FROM group_ WHERE (group_.extra_perms ->> CONCAT('u/', $1::text))::boolean AND name = $2 AND workspace_id = $4) OR exists( + SELECT 1 FROM group_ g, jsonb_each_text(g.extra_perms) f + WHERE $2 = g.name AND $4 = g.workspace_id AND SPLIT_PART(key, '/', 1) = 'g' AND key = ANY($3::text[]) + AND value::boolean)", + username, + group_name, + groups, + w_id, + ).fetch_one(db) + .await? + .unwrap_or(false); + if !is_owner { + Err(Error::BadRequest(format!( + "{} is not an owner of {} and hence is not authorized to perform this operation", + username, group_name + ))) + } else { + Ok(()) + } +} + +async fn _check_nb_of_groups(db: &DB) -> Result<()> { + let nb_groups = sqlx::query_scalar!("SELECT COUNT(*) FROM group_ WHERE name != 'all' AND name != 'error_handler' AND name != 'slack'",) + .fetch_one(db) + .await?; + if nb_groups.unwrap_or(0) >= 3 { + return Err(Error::BadRequest( + "You have reached the maximum number of groups (3 outside of native groups 'all', 'slack' and 'error_handler') without an enterprise license" + .to_string(), + )); + } + return Ok(()); +} + +async fn create_group( + authed: ApiAuthed, + Extension(_db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(ng): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + check_name_conflict(&mut tx, &w_id, &ng.name).await?; + + #[cfg(not(feature = "enterprise"))] + _check_nb_of_groups(&_db).await?; + + sqlx::query!( + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4)", + w_id, + ng.name, + ng.summary, + serde_json::json!({username_to_permissioned_as(&authed.username): true}) + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + &authed.username, + ng.name, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "group.create", + ActionKind::Create, + &w_id, + Some(&ng.name.to_string()), + None, + ) + .await?; + + log_group_permission_change(&mut *tx, &w_id, &ng.name, &authed.username, "create", None) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &_db, + &w_id, + windmill_git_sync::DeployedObject::Group { name: ng.name.clone() }, + Some(format!("Created group '{}'", &ng.name)), + true, + None, + ) + .await?; + + Ok(format!("Created group {}", ng.name)) +} + +async fn create_igroup( + authed: ApiAuthed, + Extension(db): Extension, + Json(ng): Json, +) -> Result { + use uuid::Uuid; + + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + let normalized_name = convert_name(&ng.name); + + let id = Uuid::new_v4().to_string(); + sqlx::query!( + "INSERT INTO instance_group (name, summary, id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + normalized_name, + ng.summary, + id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "igroup.create", + ActionKind::Create, + "global", + Some(&normalized_name), + None, + ) + .await?; + + tx.commit().await?; + Ok(format!("Created group {}", normalized_name)) +} + +#[derive(Deserialize)] +struct IGroupUpdate { + new_summary: String, +} + +async fn update_igroup( + authed: ApiAuthed, + Extension(db): Extension, + Path(name): Path, + Json(igroup_update): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + + let exists_opt = sqlx::query("SELECT 1 FROM instance_group WHERE name = $1") + .bind(name.clone()) + .fetch_optional(&mut *tx) + .await?; + not_found_if_none(exists_opt, "instance_group", name.clone())?; + + sqlx::query("UPDATE instance_group SET summary = $1 WHERE name = $2") + .bind(igroup_update.new_summary) + .bind(&name) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "igroup.updated", + ActionKind::Delete, + "global", + Some(&name.to_string()), + None, + ) + .await?; + + tx.commit().await?; + Ok(format!("Deleted group {}", name)) +} + +async fn delete_igroup( + authed: ApiAuthed, + Extension(db): Extension, + Path(name): Path, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + sqlx::query!("DELETE FROM instance_group WHERE name = $1", name) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM email_to_igroup WHERE igroup = $1", name) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "igroup.delete", + ActionKind::Delete, + "global", + Some(&name.to_string()), + None, + ) + .await?; + + tx.commit().await?; + Ok(format!("Deleted group {}", name)) +} + +pub async fn get_group_opt<'c>( + db: &mut Transaction<'c, Postgres>, + w_id: &str, + name: &str, +) -> Result> { + let group_opt = sqlx::query_as!( + Group, + "SELECT * FROM group_ WHERE name = $1 AND workspace_id = $2", + name, + w_id + ) + .fetch_optional(&mut **db) + .await?; + Ok(group_opt) +} + +async fn get_group( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + if *CLOUD_HOSTED && w_id == "demo" && name == "all" && !authed.is_admin { + return Ok(Json(GroupInfo { + workspace_id: w_id, + name: name, + summary: Some("The group that contains all users".to_string()), + members: vec!["redacted_in_demo_workspace".to_string()], + extra_perms: serde_json::json!({}), + })); + } + + let mut tx = user_db.begin(&authed).await?; + + let group = not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + + let members = sqlx::query_scalar!( + "SELECT usr.username + FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2 + WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2", + name, + w_id + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(GroupInfo { + workspace_id: group.workspace_id, + name: group.name, + summary: group.summary, + members, + extra_perms: group.extra_perms, + })) +} + +async fn delete_group( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + if name == "all" { + return Err(Error::BadRequest( + "The group 'all' is a special group that contains all users and cannot be deleted" + .to_string(), + )); + } + + if !authed.is_admin { + require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; + } + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + + sqlx::query!( + "DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2", + name, + w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM group_ WHERE name = $1 AND workspace_id = $2", + name, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "group.delete", + ActionKind::Delete, + &w_id, + Some(&name.to_string()), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Group { name: name.clone() }, + Some(format!("Deleted group '{}'", &name)), + true, + None, + ) + .await?; + + Ok(format!("delete group at name {}", name)) +} + +async fn update_group( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(eg): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + if !authed.is_admin { + require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; + } + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + + sqlx::query!( + "UPDATE group_ SET summary = $1 WHERE name = $2 AND workspace_id = $3", + eg.summary, + &name, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "group.edit", + ActionKind::Update, + &w_id, + Some(&name.to_string()), + None, + ) + .await?; + + log_group_permission_change( + &mut *tx, + &w_id, + &name, + &authed.username, + "update_summary", + None, + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Group { name: name.clone() }, + Some(format!("Updated group '{}'", &name)), + true, + None, + ) + .await?; + + Ok(format!("Edited group {}", name)) +} + +async fn add_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(Username { username: user_username }): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + if !authed.is_admin { + require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; + } + + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + + let result = sqlx::query!( + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + user_username, + name, + ) + .execute(&mut *tx) + .await?; + + if result.rows_affected() == 0 { + return Ok(format!( + "{} is already a member of group {}", + user_username, name + )); + } + + audit_log( + &mut *tx, + &authed, + "group.adduser", + ActionKind::Update, + &w_id, + Some(&name.to_string()), + Some([("user", user_username.as_str())].into()), + ) + .await?; + + log_group_permission_change( + &mut *tx, + &w_id, + &name, + &authed.username, + "add_member", + Some(&user_username), + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Group { name: name.clone() }, + Some(format!("Added user to group '{}'", &name)), + true, + None, + ) + .await?; + + Ok(format!("Added {} to group {}", user_username, name)) +} + +async fn add_user_igroup( + authed: ApiAuthed, + Extension(db): Extension, + Path(name): Path, + Json(Email { email }): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + + let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(group_opt, "IGroup", &name)?; + + sqlx::query!( + "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2) ON CONFLICT DO NOTHING", + email, + name, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "igroup.adduser", + ActionKind::Update, + "global", + Some(&name.to_string()), + Some([("email", email.as_str())].into()), + ) + .await?; + + // Sync user to workspaces configured with this instance group + #[cfg(all(feature = "private", feature = "enterprise"))] + { + use windmill_api_workspaces::workspaces_ee::auto_add_user; + let workspaces = sqlx::query!( + r#" + SELECT workspace_id, auto_invite->'instance_groups_roles' as instance_groups_roles + FROM workspace_settings + WHERE auto_invite->'instance_groups' ? $1 + "#, + &name + ) + .fetch_all(&mut *tx) + .await?; + for ws in workspaces { + let role = ws + .instance_groups_roles + .and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))) + .unwrap_or_else(|| "developer".to_string()); + let (is_admin, is_operator) = match role.as_str() { + "admin" => (true, false), + "operator" => (false, true), + _ => (false, false), + }; + auto_add_user( + &email, + &ws.workspace_id, + &is_operator, + &mut tx, + &authed, + Some(serde_json::json!({"source": "instance_group", "group": &name})), + ) + .await?; + if is_admin { + sqlx::query!( + "UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", + &ws.workspace_id, + &email + ) + .execute(&mut *tx) + .await?; + } + } + } + + tx.commit().await?; + Ok(format!("Added {} to igroup {}", email, name)) +} + +#[derive(Serialize)] +struct IGroup { + name: String, + summary: Option, + emails: Option>, +} + +#[derive(Serialize)] +struct IGroupWithWorkspaces { + name: String, + summary: Option, + emails: Option>, + workspaces: Vec, +} + +#[derive(Serialize, Clone)] +struct WorkspaceInfo { + workspace_id: String, + workspace_name: String, + role: String, +} +async fn list_igroups(Extension(db): Extension) -> JsonResult> { + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + + let groups = sqlx::query_as!( + IGroup, + "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name" + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + return Ok(Json(groups)); +} + +async fn list_igroups_with_workspaces( + Extension(db): Extension, +) -> JsonResult> { + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + + // Get all instance groups with their emails first + let groups = sqlx::query_as!( + IGroup, + "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name, summary" + ) + .fetch_all(&mut *tx) + .await?; + + // Get all workspace mappings for instance groups in a single query + let workspace_mappings = sqlx::query!( + r#" + SELECT + ig.name as group_name, + ws.workspace_id, + w.name as workspace_name, + ws.auto_invite->'instance_groups_roles'->ig.name as role + FROM instance_group ig + INNER JOIN workspace_settings ws ON ws.auto_invite->'instance_groups' IS NOT NULL + AND ws.auto_invite->'instance_groups' ? ig.name + INNER JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false + ORDER BY ig.name, ws.workspace_id + "# + ) + .fetch_all(&mut *tx) + .await?; + + // Create a map of group_name -> Vec + let mut workspaces_by_group: std::collections::HashMap> = + std::collections::HashMap::new(); + for mapping in workspace_mappings { + let role = mapping + .role + .and_then(|r| r.as_str().map(|s| s.to_string())) + .unwrap_or_else(|| "developer".to_string()); + + let workspace_info = WorkspaceInfo { + workspace_id: mapping.workspace_id.clone(), + workspace_name: mapping.workspace_name, + role, + }; + + workspaces_by_group + .entry(mapping.group_name) + .or_insert_with(Vec::new) + .push(workspace_info); + } + + let mut result = Vec::new(); + for group in groups { + let workspaces = workspaces_by_group + .get(&group.name) + .cloned() + .unwrap_or_default(); + + result.push(IGroupWithWorkspaces { + name: group.name, + summary: group.summary, + emails: group.emails, + workspaces, + }); + } + + tx.commit().await?; + return Ok(Json(result)); +} + +async fn get_igroup(Path(name): Path, Extension(db): Extension) -> JsonResult { + let group = sqlx::query_as!( + IGroup, + "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup WHERE name = $1 GROUP BY name", + name + ) + .fetch_optional(&db) + .await?; + let group = not_found_if_none(group, "IGroup", &name)?; + return Ok(Json(group)); +} + +async fn remove_user_igroup( + authed: ApiAuthed, + Extension(db): Extension, + Path(name): Path, + Json(Email { email }): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name,) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(group_opt, "IGroup", &name)?; + + sqlx::query!( + "DELETE FROM email_to_igroup WHERE email = $1 AND igroup = $2", + email, + name, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "igroup.removeuser", + ActionKind::Update, + "global", + Some(&name.to_string()), + Some([("email", email.as_str())].into()), + ) + .await?; + + // Remove user from workspaces where they were added via this instance group + #[cfg(all(feature = "private", feature = "enterprise"))] + { + use windmill_api_workspaces::workspaces_ee::remove_users_from_instance_group_workspaces; + remove_users_from_instance_group_workspaces(&email, &name, &mut tx).await?; + } + + tx.commit().await?; + Ok(format!("Removed {} from igroup {}", email, name)) +} + +async fn remove_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(Username { username: user_username }): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + if !authed.is_admin { + require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; + } + + not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; + if &name == "all" { + return Err(Error::BadRequest(format!("Cannot delete users from all"))); + } + sqlx::query!( + "DELETE FROM usr_to_group WHERE usr = $1 AND group_ = $2 AND workspace_id = $3", + user_username, + name, + &w_id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "group.removeuser", + ActionKind::Update, + &w_id, + Some(&name.to_string()), + Some([("user", user_username.as_str())].into()), + ) + .await?; + + log_group_permission_change( + &mut *tx, + &w_id, + &name, + &authed.username, + "remove_member", + Some(&user_username), + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Group { name: name.clone() }, + Some(format!("Removed user from group '{}'", &name)), + true, + None, + ) + .await?; + + Ok(format!("Removed {} to group {}", user_username, name)) +} + +#[cfg(feature = "enterprise")] +#[derive(Serialize, Deserialize)] +struct ExportedIGroup { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + scim_display_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + external_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + emails: Option>, +} + +#[cfg(feature = "enterprise")] +async fn export_igroups( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + let igroups = sqlx::query_as!( + ExportedIGroup, + "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails, id, scim_display_name, external_id FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name", + ).fetch_all(&mut *tx).await?; + + audit_log( + &mut *tx, + &authed, + "igroups.export", + ActionKind::Execute, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(igroups)) +} + +#[cfg(not(feature = "enterprise"))] +async fn export_igroups() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[cfg(feature = "enterprise")] +async fn overwrite_igroups( + authed: ApiAuthed, + Extension(db): Extension, + Json(igroups): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!("DELETE FROM email_to_igroup") + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM instance_group") + .execute(&mut *tx) + .await?; + + for igroup in igroups.iter() { + sqlx::query!( + "INSERT INTO instance_group (name, summary, id, scim_display_name, external_id) VALUES ($1, $2, $3, $4, $5)", + igroup.name, + igroup.summary, + igroup.id, + igroup.scim_display_name, + igroup.external_id, + ) + .execute(&mut *tx) + .await?; + + if let Some(emails) = &igroup.emails { + for email in emails.iter() { + sqlx::query!( + "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", + email, + igroup.name, + ) + .execute(&mut *tx) + .await?; + } + } + } + + audit_log( + &mut *tx, + &authed, + "igroups.import", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + Ok("Imported igroups".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +async fn overwrite_igroups() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +pub async fn log_group_permission_change<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + group_name: &str, + changed_by: &str, + change_type: &str, + member_affected: Option<&str>, +) -> Result<()> { + sqlx::query!( + "INSERT INTO group_permission_history + (workspace_id, group_name, changed_by, change_type, member_affected) + VALUES ($1, $2, $3, $4, $5)", + workspace_id, + group_name, + changed_by, + change_type, + member_affected + ) + .execute(db) + .await?; + Ok(()) +} diff --git a/backend/windmill-api-groups/src/lib.rs b/backend/windmill-api-groups/src/lib.rs new file mode 100644 index 0000000000..75b9349105 --- /dev/null +++ b/backend/windmill-api-groups/src/lib.rs @@ -0,0 +1,4 @@ +pub mod folder_history; +pub mod folders; +pub mod granular_acls; +pub mod groups; diff --git a/backend/windmill-api-jobs/Cargo.toml b/backend/windmill-api-jobs/Cargo.toml new file mode 100644 index 0000000000..d3e10205ac --- /dev/null +++ b/backend/windmill-api-jobs/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-api-jobs" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_jobs" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-api-sse.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +uuid.workspace = true +chrono.workspace = true +http.workspace = true +tower-http.workspace = true +base64.workspace = true +anyhow.workspace = true +tracing.workspace = true +sql-builder.workspace = true +tokio.workspace = true +hyper.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api-jobs/src/concurrency_groups.rs b/backend/windmill-api-jobs/src/concurrency_groups.rs new file mode 100644 index 0000000000..a727b7b02c --- /dev/null +++ b/backend/windmill-api-jobs/src/concurrency_groups.rs @@ -0,0 +1,364 @@ +use windmill_api_auth::{check_scopes, ApiAuthed}; +use windmill_common::{ + db::{UserDB, DB}, + error::Error::PermissionDenied, + error::{self, JsonResult}, + utils::require_admin, +}; + +use crate::query::{filter_list_completed_query, filter_list_queue_query}; +use crate::types::{ListCompletedQuery, ListQueueQuery, UnifiedJob}; +use windmill_api_sse::Job; + +use axum::extract::Path; +use axum::routing::{delete, get}; +use axum::{extract::Query, Extension, Json}; +use serde::Deserialize; + +use axum::Router; + +use serde::Serialize; +use sql_builder::bind::Bind; +use sql_builder::SqlBuilder; +use uuid::Uuid; + +pub fn global_service() -> Router { + Router::new() + .route("/list", get(list_concurrency_groups)) + .route("/prune/*concurrency_key", delete(prune_concurrency_group)) + .route("/:job_id/key", get(get_concurrency_key)) +} + +pub fn workspaced_service() -> Router { + Router::new().route("/list_jobs", get(get_concurrent_intervals)) +} + +#[derive(Serialize)] +pub struct ConcurrencyGroups { + concurrency_key: String, + total_running: i64, +} + +async fn list_concurrency_groups( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let concurrency_counts = sqlx::query_as::<_, (String, i64)>( + "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter", + ).fetch_all(&db) + .await?; + + let mut concurrency_groups: Vec = vec![]; + for (concurrency_key, count) in concurrency_counts { + concurrency_groups.push(ConcurrencyGroups { + concurrency_key: concurrency_key.clone(), + total_running: count, + }) + } + + return Ok(Json(concurrency_groups)); +} + +async fn prune_concurrency_group( + authed: ApiAuthed, + Extension(db): Extension, + Path(concurrency_key): Path, +) -> JsonResult<()> { + if !authed.is_admin { + return Err(PermissionDenied( + "Only administrators can delete concurrency groups".to_string(), + )); + } + let mut tx = db.begin().await?; + + let concurrency_group = sqlx::query_as::<_, (String, i64)>( + "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter WHERE concurrency_id = $1 FOR UPDATE", + ) + .bind(concurrency_key.clone()) + .fetch_optional(&mut *tx) + .await?; + + let n_job_uuids = concurrency_group.map(|cg| cg.1).unwrap_or_default(); + + if n_job_uuids > 0 { + tx.commit().await?; + return Err(error::Error::internal_err( + "Concurrency group is currently in use, unable to remove it. Retry later.".to_string(), + )); + } + + sqlx::query!( + "DELETE FROM concurrency_counter WHERE concurrency_id = $1", + concurrency_key.clone(), + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM concurrency_key WHERE key = $1", + concurrency_key.clone(), + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(())) +} + +#[derive(Serialize)] +struct ExtendedJobs { + jobs: Vec, + obscured_jobs: Vec, + omitted_obscured_jobs: bool, +} + +#[derive(Serialize)] +struct ObscuredJob { + typ: String, + started_at: Option>, + duration_ms: Option, +} +#[derive(Deserialize)] +struct ExtendedJobsParams { + row_limit: Option, +} + +pub fn join_concurrency_key<'c>( + concurrency_key: Option<&String>, + mut sqlb: SqlBuilder, +) -> SqlBuilder { + if let Some(key) = concurrency_key { + sqlb.join("concurrency_key") + .on_eq("id", "concurrency_key.job_id") + .and_where_eq("key", "?".bind(key)); + } + + sqlb +} + +async fn get_concurrent_intervals( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(iq): Query, + Query(lq): Query, +) -> JsonResult { + check_scopes(&authed, || format!("jobs:read"))?; + + if lq.success.is_some() && lq.running.is_some_and(|x| x) { + return Err(error::Error::BadRequest( + "cannot specify both success and running".to_string(), + )); + } + + let row_limit = iq.row_limit.unwrap_or(1000); + + let lq = ListCompletedQuery { order_desc: Some(true), ..lq }; + let lqc = lq.clone(); + let lqq: ListQueueQuery = lqc.into(); + let mut sqlb_q = SqlBuilder::select_from("v2_job_queue") + .fields(UnifiedJob::queued_job_fields()) + .order_by("created_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + let mut sqlb_c = SqlBuilder::select_from("v2_job_completed") + .fields(UnifiedJob::completed_job_fields()) + .order_by("completed_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue") + .fields(&["id"]) + .order_by("created_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed") + .fields(&["id"]) + .order_by("completed_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + + sqlb_q = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q); + sqlb_c = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c); + sqlb_q_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q_user); + sqlb_c_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c_user); + + let should_fetch_obscured_jobs = match lq { + ListCompletedQuery { + script_path_start: None, + script_path_exact: None, + script_hash: None, + created_by: None, + success: None, + running: None, + parent_job: None, + is_skipped: None | Some(false), + suspended: None, + schedule_path: None, + args: None, + result: None, + tag: None, + has_null_parent: None, + worker: None, + label: None, + trigger_path: None, + scheduled_for_before_now: _, + is_not_schedule: _, + started_before: _, + started_after: _, + created_before: _, + created_after: _, + created_before_queue: _, + created_after_queue: _, + completed_after: _, + completed_before: _, + created_or_started_before: _, + created_or_started_after: _, + created_or_started_after_completed_jobs: _, + order_desc: _, + job_kinds: _, + is_flow_step: _, + all_workspaces: _, + concurrency_key: Some(_), + allow_wildcards: None, + trigger_kind: _, + include_args: _, + } => true, + _ => false, + }; + + // When we have a concurrency key defined, fetch jobs from other workspaces + // as obscured unless we're in the admins workspace. This is to show the + // potential concurrency races without showing jobs that don't belong to + // the workspace. + // To avoid infering information through filtering, don't return obscured + // jobs if the filters are too specific + if should_fetch_obscured_jobs && w_id != "admins" { + // Get the obscured jobs from all workspaces (concurrency key could be global) + let (sqlb_q, sqlb_c) = ( + filter_list_queue_query( + sqlb_q, + &ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() }, + "admins", + true, + ), + filter_list_completed_query( + sqlb_c, + &ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() }, + "admins", + true, + ), + ); + + sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str(), true); + sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str(), true); + + let sql_q_user = sqlb_q_user.query()?; + let sql_c_user = sqlb_c_user.query()?; + let sql_q = sqlb_q.query()?; + let sql_c = sqlb_c.query()?; + + // This first transaction uses the user_db to know which uuids are + // accessible to the user. + let mut tx = user_db.begin(&authed).await?; + let running_jobs_user: Vec = if lq.success.is_none() { + sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await? + } else { + vec![] + }; + let completed_jobs_user: Vec = if lq.running.is_none() { + sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await? + } else { + vec![] + }; + tx.commit().await?; + + // This second transaction uses the db, so it will fetch information + // potentially forbidden to the user. It must be obscured before + // returning it + let running_jobs_db: Vec = if lq.success.is_none() { + sqlx::query_as(&sql_q).fetch_all(&db).await? + } else { + vec![] + }; + let completed_jobs_db: Vec = if lq.running.is_none() { + sqlx::query_as(&sql_c).fetch_all(&db).await? + } else { + vec![] + }; + + let obscured_jobs = running_jobs_db + .iter() + .filter(|j| !running_jobs_user.iter().any(|id| j.id == *id)) + .chain( + completed_jobs_db + .iter() + .filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)), + ) + .map(|j| ObscuredJob { + typ: j.typ.clone(), + started_at: j.started_at, + duration_ms: j.duration_ms, + }) + .collect(); + + let jobs = running_jobs_db + .into_iter() + .filter(|j| running_jobs_user.iter().any(|id| j.id == *id)) + .chain( + completed_jobs_db + .into_iter() + .filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)), + ) + .map(From::from) + .collect(); + + Ok(Json(ExtendedJobs { + jobs, + obscured_jobs, + omitted_obscured_jobs: !should_fetch_obscured_jobs, + })) + } else { + sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str(), true); + sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str(), true); + let sql_q = sqlb_q.query()?; + let sql_c = sqlb_c.query()?; + + let mut tx = user_db.begin(&authed).await?; + let running_jobs: Vec = if lq.success.is_none() { + sqlx::query_as(&sql_q).fetch_all(&mut *tx).await? + } else { + vec![] + }; + let completed_jobs: Vec = if lq.running.is_none() { + sqlx::query_as(&sql_c).fetch_all(&mut *tx).await? + } else { + vec![] + }; + tx.commit().await?; + + let jobs = running_jobs + .into_iter() + .chain(completed_jobs.into_iter()) + .map(From::from) + .collect(); + + Ok(Json(ExtendedJobs { + jobs, + obscured_jobs: vec![], + omitted_obscured_jobs: !should_fetch_obscured_jobs, + })) + } +} + +async fn get_concurrency_key( + Extension(db): Extension, + Path(job_id): Path, +) -> JsonResult> { + let key = sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id) + .fetch_optional(&db) + .await?; + Ok(Json(key)) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs new file mode 100644 index 0000000000..08aebe4ddd --- /dev/null +++ b/backend/windmill-api-jobs/src/execution.rs @@ -0,0 +1,892 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Job execution primitives: push, cancel, tag/license checks, wait-for-result. + +use axum::{ + response::{IntoResponse, Response}, + Json, +}; +use http::{HeaderMap, HeaderName, HeaderValue}; +use hyper::StatusCode; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::str::FromStr; +use std::sync::Arc; +use tokio::sync::RwLock; +use uuid::Uuid; +use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed}; +use windmill_common::{ + db::{UserDB, UserDbWithAuthed}, + error::{self, Error}, + flow_conversations::{add_message_to_conversation_tx, MessageType}, + get_latest_flow_version_info_for_path, + jobs::{ + check_tag_available_for_workspace_internal, format_result, script_path_to_payload, + JobPayload, + }, + triggers::TriggerMetadata, + users::username_to_permissioned_as, + utils::StripPath, + FlowVersionInfo, DB, +}; +use windmill_queue::{ + cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned, + PushIsolationLevel, +}; + +use crate::types::RunJobQuery; + +// --------------------------------------------------------------------------- +// Tag / license checks +// --------------------------------------------------------------------------- + +pub async fn check_tag_available_for_workspace( + db: &DB, + w_id: &str, + tag: &Option, + authed: &ApiAuthed, +) -> error::Result<()> { + if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { + let tags = get_scope_tags(authed); + check_tag_available_for_workspace_internal(&db, w_id, tag, &authed.email, tags).await + } else { + Ok(()) + } +} + +#[cfg(feature = "enterprise")] +pub async fn check_license_key_valid() -> error::Result<()> { + use windmill_common::ee_oss::LICENSE_KEY_VALID; + + let valid = *LICENSE_KEY_VALID.read().await; + if !valid { + return Err(error::Error::BadRequest( + "License key is not valid. Go to your superadmin settings to update your license key." + .to_string(), + )); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Cancel jobs +// --------------------------------------------------------------------------- + +pub async fn cancel_jobs( + jobs: Vec, + db: &DB, + username: &str, + w_id: &str, + force_cancel: bool, +) -> error::JsonResult> { + let mut uuids = vec![]; + tracing::info!("Cancelling jobs: {:?}", jobs); + let mut tx = db.begin().await?; + let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj + ( workspace_id + , id + , duration_ms + , result + , canceled_by + , canceled_reason + , flow_status + , status + , worker + ) + SELECT q.workspace_id + , q.id + , 0 + , $4 + , $1 + , 'cancel all' + , (SELECT flow_status FROM v2_job_status WHERE id = q.id) + , 'canceled'::job_status + , worker + FROM v2_job_queue q + JOIN v2_job USING (id) + WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule' + FOR UPDATE SKIP LOCKED + ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) + .fetch_all(&mut *tx) + .await?.into_iter().map(|x| x.id).collect::>(); + + sqlx::query!( + "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", + &trivial_jobs, + w_id + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + for job_id in jobs.into_iter() { + if trivial_jobs.contains(&job_id) { + continue; + } + match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move { + let tx = db.begin().await?; + let (tx, _) = cancel_job( + username, + None, + job_id.clone(), + w_id, + tx, + db, + force_cancel, + false, + ) + .await?; + tx.commit().await?; + Ok::<_, anyhow::Error>(()) + }) + .await + { + Ok(result) => match result { + Ok(_) => { + uuids.push(job_id); + } + Err(e) => { + tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e); + } + }, + Err(_) => { + tracing::error!( + "Timeout while trying to cancel job {:?} after 5 seconds", + job_id + ); + } + } + } + + uuids.extend(trivial_jobs); + + Ok(Json(uuids)) +} + +// --------------------------------------------------------------------------- +// Wait-for-result infrastructure +// --------------------------------------------------------------------------- + +struct Guard { + done: bool, + id: Uuid, + w_id: String, + db: DB, + username: String, +} + +impl Drop for Guard { + fn drop(&mut self) { + if !&self.done { + let id = self.id; + let w_id = self.w_id.clone(); + let db = self.db.clone(); + let username = self.username.clone(); + + tracing::info!("http connection broke, marking job {id} as canceled"); + tokio::spawn(async move { + let cancel_f = async { + let tx = db.begin().await?; + let (tx, _) = cancel_job( + &username, + Some("http connection broke".to_string()), + id, + &w_id, + tx, + &db, + false, + false, + ) + .await?; + tx.commit().await?; + Ok::<_, anyhow::Error>(()) + }; + + if let Err(e) = cancel_f.await { + tracing::error!( + "Error marking job as canceled after http connection broke: {e}" + ); + } + }); + } + } +} + +lazy_static::lazy_static! { + pub static ref TIMEOUT_WAIT_RESULT: Arc>> = Arc::new(RwLock::new( + std::env::var("TIMEOUT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse::().ok()) + )); + + pub static ref QUEUE_LIMIT_WAIT_RESULT: Option = std::env::var("QUEUE_LIMIT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse().ok()); + + pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(50); + + pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(2); + + pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(200); +} + +#[derive(Deserialize)] +pub struct WindmillCompositeResult { + windmill_status_code: Option, + windmill_content_type: Option, + windmill_headers: Option>, + result: Option>, +} + +pub async fn run_wait_result_internal( + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, +) -> error::Result<(Box, bool)> { + let mut result = None; + let mut success = false; + let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600); + let timeout_ms = if timeout <= 0 { + 2000 + } else { + (timeout * 1000) as u64 + }; + + let mut g = Guard { + done: false, + id: uuid, + w_id: w_id.to_string(), + db: db.clone(), + username: username.to_string(), + }; + + let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000; + let mut accumulated_delay = 0 as u64; + + loop { + if let Some(node_id_for_empty_return) = node_id_for_empty_return.as_ref() { + let result_and_success = get_result_and_success_by_id_from_flow( + &db, + w_id, + &uuid, + node_id_for_empty_return, + None, + ) + .await + .ok(); + if let Some((r, s)) = result_and_success { + result = Some(r); + success = s; + } + } + + if result.is_none() { + let row = sqlx::query!( + " + SELECT + result AS \"result: sqlx::types::Json>\", + result_columns, + status = 'success' AS \"success!\" + FROM + v2_job_completed + WHERE + id = $1 AND + workspace_id = $2 + ", + uuid, + &w_id + ) + .fetch_optional(db) + .await?; + if let Some(mut raw_result) = row { + format_result( + raw_result.result_columns.as_ref(), + raw_result.result.as_mut(), + ); + result = raw_result.result.map(|x| x.0); + success = raw_result.success; + } + } + + if result.is_some() { + break; + } + + let delay = if accumulated_delay <= fast_poll_duration { + *WAIT_RESULT_FAST_POLL_INTERVAL_MS + } else { + *WAIT_RESULT_SLOW_POLL_INTERVAL_MS + }; + accumulated_delay += delay; + if accumulated_delay > timeout_ms { + break; + }; + tokio::time::sleep(core::time::Duration::from_millis(delay)).await; + } + + if let Some(result) = result { + g.done = true; + Ok((result, success)) + } else { + Err(Error::ExecutionErr(format!("timeout after {}s", timeout))) + } +} + +pub fn result_to_response(result: Box, success: bool) -> error::Result { + let composite_result = serde_json::from_str::(result.get()); + match composite_result { + Ok(WindmillCompositeResult { + windmill_status_code, + windmill_content_type, + windmill_headers, + result: result_value, + }) => { + if windmill_content_type.is_none() + && windmill_status_code.is_none() + && windmill_headers.is_none() + { + return Ok(( + if success { + StatusCode::OK + } else { + StatusCode::UNPROCESSABLE_ENTITY + }, + Json(result), + ) + .into_response()); + } + + let status_code_or_default = windmill_status_code + .map(|val| match StatusCode::from_u16(val) { + Ok(sc) => Ok(sc), + Err(_) => Err(Error::ExecutionErr("Invalid status code".to_string())), + }) + .unwrap_or_else(|| { + if !success { + Ok(StatusCode::UNPROCESSABLE_ENTITY) + } else if result_value.is_some() { + Ok(StatusCode::OK) + } else { + Ok(StatusCode::NO_CONTENT) + } + })?; + + let mut headers = HeaderMap::new(); + + if let Some(windmill_headers) = windmill_headers { + for (k, v) in windmill_headers { + let k = HeaderName::from_str(k.as_str()).map_err(|err| { + Error::internal_err(format!("Invalid header name {k}: {err}")) + })?; + let v = HeaderValue::from_str(v.as_str()).map_err(|err| { + Error::internal_err(format!("Invalid header value {v}: {err}")) + })?; + headers.insert(k, v); + } + } + + if let Some(content_type) = windmill_content_type { + let serialized_json_result = result_value + .map(|val| val.get().to_owned()) + .unwrap_or_else(String::new); + let serialized_result = + serde_json::from_str::(serialized_json_result.as_str()) + .ok() + .unwrap_or(serialized_json_result); + headers.insert( + http::header::CONTENT_TYPE, + HeaderValue::from_str(content_type.as_str()).map_err(|err| { + Error::internal_err(format!("Invalid content type {content_type}: {err}")) + })?, + ); + return Ok((status_code_or_default, headers, serialized_result).into_response()); + } + if let Some(result_value) = result_value { + return Ok((status_code_or_default, headers, Json(result_value)).into_response()); + } else { + Ok((status_code_or_default, headers).into_response()) + } + } + _ => Ok(( + if success { + StatusCode::OK + } else { + StatusCode::UNPROCESSABLE_ENTITY + }, + Json(result), + ) + .into_response()), + } +} + +pub async fn run_wait_result( + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, +) -> error::Result { + let (result, success) = + run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username).await?; + + result_to_response(result, success) +} + +// --------------------------------------------------------------------------- +// Delete job metadata +// --------------------------------------------------------------------------- + +pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> { + sqlx::query!( + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", + job_uuid, + ) + .execute(db) + .await?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Queue limit check +// --------------------------------------------------------------------------- + +pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { + if let Some(limit) = queue_limit { + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() + OR (q.suspend_until IS NOT NULL + AND (q.suspend <= 0 OR q.suspend_until <= now())))", + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + if count > queue_limit.unwrap() { + return Err(Error::internal_err(format!( + "Number of queued job is too high: {count} > {limit}" + ))); + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Flow run helpers +// --------------------------------------------------------------------------- + +pub async fn set_flow_memory_id( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + job_id: Uuid, + memory_id: Uuid, +) -> error::Result<()> { + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = jsonb_set( + flow_status, + '{memory_id}', + to_jsonb($2::uuid) + ) + WHERE id = $1", + job_id, + memory_id + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Apply flow-specific query parameters after job creation +pub async fn process_flow_run_query_params( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + job_id: Uuid, + run_query: &RunJobQuery, +) -> error::Result<()> { + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(tx, job_id, memory_id).await?; + } + Ok(()) +} + +pub async fn handle_chat_conversation_messages( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + authed: &ApiAuthed, + w_id: &str, + flow_path: &str, + run_query: &RunJobQuery, + user_message_raw: Option<&Box>, +) -> error::Result<()> { + let memory_id = run_query.memory_id.ok_or_else(|| { + windmill_common::error::Error::BadRequest( + "memory_id is required for chat-enabled flows".to_string(), + ) + })?; + + let user_message_raw = user_message_raw.ok_or_else(|| { + windmill_common::error::Error::BadRequest( + "user_message argument is required for chat-enabled flows".to_string(), + ) + })?; + + let user_message: String = serde_json::from_str(user_message_raw.get()).map_err(|e| { + windmill_common::error::Error::BadRequest(format!( + "Failed to deserialize user_message: {}", + e + )) + })?; + + windmill_common::flow_conversations::get_or_create_conversation_with_id( + tx, + w_id, + flow_path, + &authed.username, + &user_message, + memory_id, + ) + .await?; + + add_message_to_conversation_tx( + tx, + memory_id, + None, + &user_message, + MessageType::User, + None, + true, + ) + .await?; + + Ok(()) +} + +pub async fn run_flow<'c>( + authed: &ApiAuthed, + db: &DB, + tx_o: Option>, + user_db: UserDB, + w_id: &str, + flow_path: &str, + flow_version_info: FlowVersionInfo, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + let FlowVersionInfo { + version, + tag, + dedicated_worker, + has_preprocessor, + chat_input_enabled, + on_behalf_of_email, + edited_by, + early_return, + .. + } = flow_version_info; + + let tag = run_query.tag.clone().or(tag); + + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let scheduled_for = run_query.get_scheduled_for(&db).await?; + + let return_tx = tx_o.is_some(); + + let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { + ( + &authed.email, + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Transaction(tx), + ) + } else if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { + ( + on_behalf_of_email, + username_to_permissioned_as(&edited_by), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ) + } else { + ( + &authed.email, + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), + ) + }; + + let (uuid, mut tx) = push( + &db, + tx, + &w_id, + JobPayload::Flow { + path: flow_path.to_string(), + dedicated_worker, + version, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + }, + PushArgs { args: &args.args, extra: args.extra }, + authed.display_username(), + email, + permissioned_as, + authed.token_prefix.as_deref(), + scheduled_for, + None, + run_query.parent_job, + None, + run_query.root_job, + run_query.job_id, + false, + false, + None, + !run_query.invisible_to_owner.unwrap_or(false), + tag, + None, + None, + None, + push_authed.as_ref(), + false, + None, + trigger, + run_query.suspended_mode, + ) + .await?; + + // Set memory_id if provided (for agent memory) + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(&mut tx, uuid, memory_id).await?; + } + + // Handle conversation messages for chat-enabled flows + if chat_input_enabled.unwrap_or(false) { + handle_chat_conversation_messages( + &mut tx, + &authed, + &w_id, + &flow_path.to_string(), + &run_query, + args.args.get("user_message"), + ) + .await?; + } + + // If we were given a transaction, return it; otherwise commit it + if return_tx { + Ok((uuid, early_return, Some(tx))) + } else { + tx.commit().await?; + Ok((uuid, early_return, None)) + } +} + +pub async fn run_flow_and_wait_result( + authed: &ApiAuthed, + db: &DB, + user_db: UserDB, + w_id: &str, + flow_path: &str, + flow_version_info: FlowVersionInfo, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result { + let (uuid, early_return, _) = run_flow( + authed, + db, + None, + user_db, + w_id, + flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await?; + + run_wait_result(&db, uuid, w_id, early_return, &authed.username).await +} + +// --------------------------------------------------------------------------- +// Push job by path +// --------------------------------------------------------------------------- + +pub async fn push_flow_job_by_path_into_queue<'c>( + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let flow_path = flow_path.to_path(); + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + + let flow_version_info = + get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) + .await?; + + run_flow( + &authed, + &db, + tx_o, + user_db, + &w_id, + flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await +} + +pub async fn push_script_job_by_path_into_queue<'c>( + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + script_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload( + script_path, + Some(userdb_authed), + db.clone(), + &w_id, + run_query.skip_preprocessor, + ) + .await?; + let scheduled_for = run_query.get_scheduled_for(&db).await?; + + let tag = run_query.tag.clone().or(tag); + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + + let return_tx = tx_o.is_some(); + + let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Transaction(tx), + ) + } else if let Some(on_behalf_of) = on_behalf_of.as_ref() { + ( + on_behalf_of.email.as_str(), + on_behalf_of.permissioned_as.clone(), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ) + } else { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db, authed.clone().into()), + ) + }; + + let (uuid, tx) = push( + &db, + tx, + &w_id, + job_payload, + PushArgs { args: &args.args, extra: args.extra }, + authed.display_username(), + email, + permissioned_as, + authed.token_prefix.as_deref(), + scheduled_for, + None, + run_query.parent_job, + None, + run_query.root_job, + run_query.job_id, + false, + false, + None, + !run_query.invisible_to_owner.unwrap_or(false), + tag, + timeout, + None, + // If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher. + if run_query.parent_job.is_some() || run_query.root_job.is_some() { + Some(2) + } else { + None + }, + push_authed.as_ref(), + false, + None, + trigger, + run_query.suspended_mode, + ) + .await?; + + // If we were given a transaction, return it; otherwise commit it + if return_tx { + Ok((uuid, delete_after_use, Some(tx))) + } else { + tx.commit().await?; + Ok((uuid, delete_after_use, None)) + } +} diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs new file mode 100644 index 0000000000..aba8a5f25c --- /dev/null +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -0,0 +1,266 @@ +use windmill_common::DB; + +use axum::{ + extract::Path, + routing::{get, post}, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use tower_http::cors::{Any, CorsLayer}; +use uuid::Uuid; +use windmill_common::{ + error::{self, Error}, + job_metrics::{ + record_metric, register_metric_for_job, JobStatsRecord, MetricKind, MetricNumericValue, + }, +}; + +pub fn workspaced_service() -> Router { + let cors = CorsLayer::new() + .allow_methods([http::Method::GET, http::Method::POST]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .allow_origin(Any); + + Router::new() + .route("/get/:id", post(get_job_metrics).layer(cors.clone())) + .route( + "/set_progress/:id", + post(set_job_progress).layer(cors.clone()), + ) + .route( + "/get_progress/:id", + get(get_job_progress).layer(cors.clone()), + ) +} + +#[derive(Deserialize)] +struct JobStatsRequest { + from_timestamp: Option>, + to_timestamp: Option>, + timeseries_max_datapoints: Option, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all +} + +#[derive(Serialize)] +struct JobStatsResponse { + metrics_metadata: Vec, + scalar_metrics: Vec, + timeseries_metrics: Vec, +} + +#[derive(Serialize)] +pub struct MetricsMetadata { + pub id: String, + pub name: Option, +} + +#[derive(Serialize)] +struct ScalarMetric { + metric_id: String, + value: f64, +} + +#[derive(Serialize)] +struct TimeseriesMetric { + metric_id: String, + values: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DataPoint { + pub timestamp: chrono::DateTime, + pub value: f64, +} + +async fn get_job_metrics( + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json< + JobStatsRequest, + >, +) -> error::JsonResult { + let records = sqlx::query_as::<_, JobStatsRecord>( + "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", + ) + .bind(w_id) + .bind(job_id) + .fetch_all(&db) + .await?; + + let mut metrics_metadata: Vec = vec![]; + let mut scalar_metrics: Vec = vec![]; + let mut timeseries_metrics: Vec = vec![]; + + for record in records { + let metric_id = record.metric_id; + match record.metric_kind { + MetricKind::ScalarInt => { + let value = record.scalar_int.unwrap_or_default() as f64; + scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); + } + MetricKind::ScalarFloat => { + let value = record.scalar_float.unwrap_or_default() as f64; + scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); + } + MetricKind::TimeseriesInt => { + if record.timestamps.clone().unwrap_or_default().len() + != record.timeseries_int.clone().unwrap_or_default().len() + { + tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) + } + let (timestamps, timeseries_int) = timeseries_sample( + from_timestamp, + to_timestamp, + timeseries_max_datapoints, + record.timestamps.unwrap_or_default(), + record.timeseries_int.unwrap_or_default(), + ); + let mut values: Vec = vec![]; + for (idx, value) in timeseries_int.iter().enumerate() { + values.push(DataPoint { + timestamp: timestamps[idx], + value: value.to_owned() as f64, + }); + } + timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); + } + MetricKind::TimeseriesFloat => { + if record.timestamps.clone().unwrap_or_default().len() + != record.timeseries_int.clone().unwrap_or_default().len() + { + tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) + } + let (timestamps, timeseries_float) = timeseries_sample( + from_timestamp, + to_timestamp, + timeseries_max_datapoints, + record.timestamps.unwrap_or_default(), + record.timeseries_float.unwrap_or_default(), + ); + let mut values: Vec = vec![]; + for (idx, value) in timeseries_float.iter().enumerate() { + values.push(DataPoint { + timestamp: timestamps[idx], + value: value.to_owned() as f64, + }); + } + timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); + } + }; + metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name }); + } + + let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; + Ok(Json(response)) +} +#[derive(Deserialize)] +struct JobProgressSetRequest { + percent: i32, + /// Optional parent flow id + /// Used to modify flow status + /// Specifically `progress` field in corresponding FlowStatusModule in `InProgress` state + flow_job_id: Option, +} + +async fn set_job_progress( + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Json(JobProgressSetRequest { percent, flow_job_id }): Json, +) -> error::JsonResult<()> { + // If flow_job_id exists, than we should modify flow_status of corresponding module + // Individual jobs and flows are handled differently + if let Some(flow_job_id) = flow_job_id { + // TODO: Return error if trying to set completed job? + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1) + WHERE id = $2", + serde_json::json!(percent.clamp(0, 99)), + flow_job_id + ) + .execute(&db) + .await?; + } + + let record_progress = || { + record_metric( + &db, + w_id.clone(), + job_id, + "progress_perc".to_owned(), + MetricNumericValue::Integer(percent), + ) + }; + + // Try to record + if let Err(err) = record_progress().await { + if matches!(err, Error::MetricNotFound(..)) { + // Register + // TODO: Reset progress after job is finished (in case it reruns same job)? + _ = register_metric_for_job( + &db, + w_id.clone(), + job_id, + "progress_perc".to_string(), + MetricKind::ScalarInt, + Some("Job Execution Progress (%)".to_owned()), + ) + .await?; + // Retry recording progress + record_progress().await.map_err(|err| { + // If for some reason it still returns same error, this error will be converted to BadRequest and returned + if let Error::MetricNotFound(body) = err { + Error::BadRequest(body) + } else { + err + } + })?; + } else { + return Err(err); + } + }; + return Ok(Json(())); +} + +async fn get_job_progress( + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, +) -> error::JsonResult> { + let progress: Option> = sqlx::query_scalar!( + "SELECT (scalar_int)::int FROM job_stats WHERE job_id = $1 AND workspace_id = $2 AND metric_id = 'progress_perc'", + job_id, w_id) + .fetch_optional(&db) + .await?; + + let respond_value = if let Some(Some(progress)) = progress { + Some(progress.clamp(0, 99)) + } else { + None + }; + + Ok(Json(respond_value)) +} + +fn timeseries_sample( + from: Option>, + to: Option>, + _datapoints: Option, + timestamps: Vec>, + values: Vec, +) -> (Vec>, Vec) { + if timestamps.len() != values.len() { + tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len()); + return (vec![], vec![]); + } + let mut filtered_timestamp: Vec> = vec![]; + let mut filtered_values: Vec = vec![]; + for (idx, timestamp) in timestamps.iter().enumerate() { + if *timestamp > from.unwrap_or(chrono::DateTime::::MIN_UTC) + && *timestamp < to.unwrap_or(chrono::DateTime::::MAX_UTC) + { + filtered_timestamp.push(timestamps[idx]); + filtered_values.push(values[idx]); + } + } + // TODO: implement sampling + return (filtered_timestamp, filtered_values); +} diff --git a/backend/windmill-api-jobs/src/jobs.rs b/backend/windmill-api-jobs/src/jobs.rs new file mode 100644 index 0000000000..afee1ab953 --- /dev/null +++ b/backend/windmill-api-jobs/src/jobs.rs @@ -0,0 +1,4 @@ +// benchmark_modification + +// benchmark +pub fn _benchmark_test_fn() -> bool { true } diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs new file mode 100644 index 0000000000..5de2c51795 --- /dev/null +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -0,0 +1,690 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Path, Query}, + Extension, Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error, + jobs::{JobKind, JobStatus, JobTriggerKind}, + scripts::ScriptLang, + utils::{paginate, paginate_without_limits, require_admin, Pagination}, +}; + +use windmill_api_auth::ApiAuthed; + +#[derive(Serialize, Deserialize)] +pub struct ExportableCompletedJob { + pub id: Uuid, + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>>, + pub tag: String, + pub created_at: chrono::DateTime, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub kind: JobKind, + pub runnable_id: Option, + pub runnable_path: Option, + pub parent_job: Option, + pub root_job: Option, + pub script_lang: Option, + pub script_entrypoint_override: Option, + pub flow_step: Option, + pub flow_step_id: Option, + pub flow_innermost_root_job: Option, + pub trigger: Option, + pub trigger_kind: Option, + pub same_worker: bool, + pub visible_to_owner: bool, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub timeout: Option, + pub priority: Option, + pub preprocessed: Option, + pub args: Option>>, + pub labels: Option>, + pub pre_run_error: Option, + pub duration_ms: i64, + pub result: Option>>, + pub deleted: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub flow_status: Option>>, + pub started_at: Option>, + pub memory_peak: Option, + pub status: JobStatus, + pub completed_at: chrono::DateTime, + pub worker: Option, + pub workflow_as_code_status: Option>>, + pub result_columns: Option>, + pub retries: Option>, + pub extras: Option>>, + pub logs: Option, + pub log_offset: Option, + pub log_file_index: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct ExportableQueuedJob { + // v2_job columns (excluding workspace_id) + pub id: Uuid, + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>>, + pub tag: String, + pub created_at: chrono::DateTime, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub kind: JobKind, + pub runnable_id: Option, + pub runnable_path: Option, + pub parent_job: Option, + pub root_job: Option, + pub script_lang: Option, + pub script_entrypoint_override: Option, + pub flow_step: Option, + pub flow_step_id: Option, + pub flow_innermost_root_job: Option, + pub trigger: Option, + pub trigger_kind: Option, + pub same_worker: bool, + pub visible_to_owner: bool, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub timeout: Option, + pub priority: Option, + pub preprocessed: Option, + pub args: Option>>, + pub labels: Option>, + pub pre_run_error: Option, + + // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) + pub started_at: Option>, + pub scheduled_for: chrono::DateTime, + pub running: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub suspend: Option, + pub suspend_until: Option>, + pub worker: Option, + pub extras: Option>>, + + // v2_job_runtime columns (excluding id) + pub ping: Option>, + pub memory_peak: Option, + + // v2_job_status columns (excluding id) + pub flow_status: Option>>, + pub flow_leaf_jobs: Option>>, + pub workflow_as_code_status: Option>>, + + // concurrency_key table + pub concurrency_key: Option, +} + +pub async fn export_completed_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let (per_page, offset) = paginate(pagination); + let per_page = per_page as i64; + let offset = offset as i64; + + let mut tx = user_db.begin(&authed).await?; + + let jobs = sqlx::query_as!( + ExportableCompletedJob, + r#" + SELECT + v2_job.id, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow as "raw_flow: _", + v2_job.tag, + v2_job.created_at, + v2_job.created_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job.kind as "kind: _", + v2_job.runnable_id, + v2_job.runnable_path, + v2_job.parent_job, + v2_job.root_job, + v2_job.script_lang as "script_lang: _", + v2_job.script_entrypoint_override, + v2_job.flow_step, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job.trigger, + v2_job.trigger_kind as "trigger_kind: _", + v2_job.same_worker, + v2_job.visible_to_owner, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.cache_ttl, + v2_job.timeout, + v2_job.priority, + v2_job.preprocessed, + v2_job.args as "args: _", + v2_job.labels, + v2_job.pre_run_error, + v2_job_completed.duration_ms, + v2_job_completed.result as "result: _", + v2_job_completed.deleted, + v2_job_completed.canceled_by, + v2_job_completed.canceled_reason, + v2_job_completed.flow_status as "flow_status: _", + v2_job_completed.started_at, + v2_job_completed.memory_peak, + v2_job_completed.status as "status: _", + v2_job_completed.completed_at, + v2_job_completed.worker, + v2_job_completed.workflow_as_code_status as "workflow_as_code_status: _", + v2_job_completed.result_columns, + v2_job_completed.retries, + v2_job_completed.extras as "extras: _", + job_logs.logs AS logs, + job_logs.log_offset, + job_logs.log_file_index + FROM v2_job_completed + INNER JOIN v2_job ON v2_job.id = v2_job_completed.id + LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id + LEFT JOIN job_logs ON job_logs.job_id = v2_job.id + WHERE v2_job_completed.workspace_id = $1 + ORDER BY v2_job.created_at DESC + LIMIT $2 + OFFSET $3 + "#, + w_id, + per_page, + offset + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(jobs)) +} + +pub async fn export_queued_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let (per_page, offset) = paginate_without_limits(pagination); + let per_page = per_page as i64; + let offset = offset as i64; + + let mut tx = user_db.begin(&authed).await?; + + let jobs = sqlx::query_as!( + ExportableQueuedJob, + r#" + SELECT + v2_job.id, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow as "raw_flow: _", + v2_job.tag, + v2_job.created_at, + v2_job.created_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job.kind as "kind: _", + v2_job.runnable_id, + v2_job.runnable_path, + v2_job.parent_job, + v2_job.root_job, + v2_job.script_lang as "script_lang: _", + v2_job.script_entrypoint_override, + v2_job.flow_step, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job."trigger", + v2_job.trigger_kind as "trigger_kind: _", + v2_job.same_worker, + v2_job.visible_to_owner, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.cache_ttl, + v2_job.timeout, + v2_job.priority, + v2_job.preprocessed, + v2_job.args as "args: _", + v2_job.labels, + v2_job.pre_run_error, + + v2_job_queue.started_at, + v2_job_queue.scheduled_for, + v2_job_queue.running, + v2_job_queue.canceled_by, + v2_job_queue.canceled_reason, + v2_job_queue.suspend, + v2_job_queue.suspend_until, + v2_job_queue.worker, + v2_job_queue.extras as "extras: _", + + v2_job_runtime.ping, + v2_job_runtime.memory_peak, + + v2_job_status.flow_status as "flow_status: _", + v2_job_status.flow_leaf_jobs as "flow_leaf_jobs: _", + v2_job_status.workflow_as_code_status as "workflow_as_code_status: _", + + concurrency_key.key as "concurrency_key?" + FROM v2_job_queue + INNER JOIN v2_job ON v2_job.id = v2_job_queue.id + LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id + LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id + LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id + WHERE v2_job_queue.workspace_id = $1 + AND v2_job_queue.running = false + AND v2_job.parent_job IS NULL + AND v2_job.trigger_kind IS DISTINCT FROM 'schedule' + ORDER BY v2_job.created_at DESC + LIMIT $2 + OFFSET $3 + "#, + w_id, + per_page, + offset + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(jobs)) +} + +pub async fn import_completed_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(jobs): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + for job in jobs { + sqlx::query!( + r#" + INSERT INTO v2_job ( + id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by, + permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path, + parent_job, root_job, script_lang, script_entrypoint_override, flow_step, + flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker, + visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl, + timeout, priority, preprocessed, args, labels, pre_run_error + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, job.raw_code, job.raw_lock, job.raw_flow as _, &job.tag, &w_id, + job.created_at, &job.created_by, &job.permissioned_as, job.permissioned_as_email, + job.kind as _, job.runnable_id, job.runnable_path, job.parent_job, job.root_job, + job.script_lang as _, job.script_entrypoint_override, job.flow_step, + job.flow_step_id, job.flow_innermost_root_job, job.trigger, job.trigger_kind as _, + job.same_worker, job.visible_to_owner, job.concurrent_limit, + job.concurrency_time_window_s, job.cache_ttl, job.timeout, job.priority, + job.preprocessed, job.args as _, job.labels as _, job.pre_run_error + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + r#" + INSERT INTO v2_job_completed ( + id, workspace_id, started_at, completed_at, duration_ms, result, deleted, + canceled_by, canceled_reason, flow_status, memory_peak, status, worker, + workflow_as_code_status, result_columns, retries, extras + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + &w_id, + job.started_at, + job.completed_at, + job.duration_ms, + job.result as _, + job.deleted, + job.canceled_by, + job.canceled_reason, + job.flow_status as _, + job.memory_peak, + job.status as _, + job.worker, + job.workflow_as_code_status as _, + job.result_columns as _, + job.retries as _, + job.extras as _ + ) + .execute(&mut *tx) + .await?; + + if let Some(logs) = &job.logs { + sqlx::query!( + r#" + INSERT INTO job_logs ( + job_id, workspace_id, logs, log_offset, log_file_index + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (job_id) DO NOTHING + "#, + job.id, + &w_id, + logs, + job.log_offset, + job.log_file_index as _ + ) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + Ok(format!("Successfully imported jobs")) +} + +pub async fn import_queued_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(jobs): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + for job in jobs { + sqlx::query!( + r#" + INSERT INTO v2_job ( + id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by, + permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path, + parent_job, root_job, script_lang, script_entrypoint_override, flow_step, + flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker, + visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl, + timeout, priority, preprocessed, args, labels, pre_run_error + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, + $31, $32, $33 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.raw_code, + job.raw_lock, + job.raw_flow as _, + &job.tag, + &w_id, + job.created_at, + &job.created_by, + &job.permissioned_as, + &job.permissioned_as_email, + job.kind as _, + job.runnable_id, + job.runnable_path, + job.parent_job, + job.root_job, + job.script_lang as _, + job.script_entrypoint_override, + job.flow_step, + job.flow_step_id, + job.flow_innermost_root_job, + job.trigger, + job.trigger_kind as _, + job.same_worker, + job.visible_to_owner, + job.concurrent_limit, + job.concurrency_time_window_s, + job.cache_ttl, + job.timeout, + job.priority, + job.preprocessed, + job.args as _, + job.labels as _, + job.pre_run_error, + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + r#" + INSERT INTO v2_job_queue ( + id, workspace_id, started_at, scheduled_for, running, canceled_by, + canceled_reason, suspend, suspend_until, worker, extras, tag, priority + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + &w_id, + job.started_at, + job.scheduled_for, + job.running, + job.canceled_by, + job.canceled_reason, + job.suspend, + job.suspend_until, + job.worker, + job.extras as _, + &job.tag, + job.priority + ) + .execute(&mut *tx) + .await?; + + if job.ping.is_some() || job.memory_peak.is_some() { + sqlx::query!( + r#" + INSERT INTO v2_job_runtime (id, ping, memory_peak) + VALUES ($1, $2, $3) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.ping, + job.memory_peak + ) + .execute(&mut *tx) + .await?; + } + + if job.flow_status.is_some() + || job.flow_leaf_jobs.is_some() + || job.workflow_as_code_status.is_some() + { + sqlx::query!( + r#" + INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs, workflow_as_code_status) + VALUES ($1, $2, $3, $4) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.flow_status as _, + job.flow_leaf_jobs as _, + job.workflow_as_code_status as _ + ) + .execute(&mut *tx) + .await?; + } + + if let Some(ref concurrency_key) = job.concurrency_key { + sqlx::query!( + r#" + WITH inserted_concurrency_counter AS ( + INSERT INTO concurrency_counter (concurrency_id, job_uuids) + VALUES ($1, '{}'::jsonb) + ON CONFLICT DO NOTHING + ) + INSERT INTO concurrency_key(key, job_id) + VALUES ($1, $2) + ON CONFLICT (job_id) DO NOTHING + "#, + concurrency_key, + job.id + ) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + Ok(format!("Successfully imported jobs")) +} + +pub async fn delete_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(job_ids): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + if job_ids.is_empty() { + return Ok("No jobs to delete".to_string()); + } + + let mut tx = user_db.begin(&authed).await?; + + let logs_deleted = sqlx::query!( + "DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let perms_deleted = sqlx::query!( + "DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let stats_deleted = sqlx::query!( + "DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let resume_deleted = sqlx::query!("DELETE FROM resume_job WHERE job = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + let runtime_deleted = sqlx::query!("DELETE FROM v2_job_runtime WHERE id = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + let status_deleted = sqlx::query!("DELETE FROM v2_job_status WHERE id = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + let concurrency_key_deleted = sqlx::query!( + "DELETE FROM concurrency_key WHERE job_id = ANY($1)", + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let queue_deleted = sqlx::query!( + "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let completed_deleted = sqlx::query!( + "DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let zombie_deleted = sqlx::query!( + "DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let jobs_deleted = sqlx::query!( + "DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + tx.commit().await?; + + let total_rows_deleted = logs_deleted + + perms_deleted + + stats_deleted + + resume_deleted + + runtime_deleted + + status_deleted + + concurrency_key_deleted + + queue_deleted + + completed_deleted + + zombie_deleted + + jobs_deleted; + + tracing::info!( + "Successfully deleted {} jobs ({} total rows across all tables) from workspace {}", + job_ids.len(), + total_rows_deleted, + w_id + ); + + Ok(format!( + "Successfully deleted {} jobs ({} total rows across all tables)", + jobs_deleted, total_rows_deleted + )) +} diff --git a/backend/windmill-api-jobs/src/lib.rs b/backend/windmill-api-jobs/src/lib.rs new file mode 100644 index 0000000000..7479ec1496 --- /dev/null +++ b/backend/windmill-api-jobs/src/lib.rs @@ -0,0 +1,18 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod concurrency_groups; +pub mod execution; +pub mod job_metrics; +pub mod jobs_export; +pub mod query; +pub mod types; + +pub use execution::*; +pub use query::*; +pub use types::*; diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs new file mode 100644 index 0000000000..ee7877438a --- /dev/null +++ b/backend/windmill-api-jobs/src/query.rs @@ -0,0 +1,811 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Query builders for filtering job lists (queue and completed). + +use sql_builder::prelude::*; +use sql_builder::SqlBuilder; +use windmill_common::utils::{paginate_without_limits, Pagination}; + +use crate::types::{ListCompletedQuery, ListQueueQuery}; + +pub fn filter_list_queue_query( + mut sqlb: SqlBuilder, + lq: &ListQueueQuery, + w_id: &str, + join_outstanding_wait_times: bool, +) -> SqlBuilder { + sqlb.join("v2_job").on_eq("v2_job_queue.id", "v2_job.id"); + + if join_outstanding_wait_times { + sqlb.left() + .join("outstanding_wait_time") + .on_eq("v2_job_queue.id", "outstanding_wait_time.job_id"); + } + + if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { + sqlb.and_where_eq("v2_job_queue.workspace_id", "?".bind(&w_id)); + } + + if let Some(w) = &lq.worker { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job_queue.worker", w.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job_queue.worker", "?".bind(w)); + } + } + + if let Some(ps) = &lq.script_path_start { + sqlb.and_where_like_left("runnable_path", ps); + } + if let Some(p) = &lq.script_path_exact { + sqlb.and_where_eq("runnable_path", "?".bind(p)); + } + if let Some(p) = &lq.schedule_path { + sqlb.and_where_eq("trigger", "?".bind(p)); + sqlb.and_where_eq("trigger_kind", "'schedule'"); + } + if let Some(h) = &lq.script_hash { + sqlb.and_where_eq("runnable_id", "?".bind(h)); + } + if let Some(cb) = &lq.created_by { + sqlb.and_where_eq("created_by", "?".bind(cb)); + } + if let Some(t) = &lq.tag { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job.tag", "?".bind(t)); + } + } + + if let Some(r) = &lq.running { + sqlb.and_where_eq("running", &r); + } + if let Some(pj) = &lq.parent_job { + sqlb.and_where_eq("parent_job", "?".bind(pj)); + } + if let Some(dt) = &lq.started_before { + sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.started_after { + sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(fs) = &lq.is_flow_step { + if *fs { + sqlb.and_where_is_not_null("flow_step_id"); + } else { + sqlb.and_where_is_null("flow_step_id"); + } + } + if let Some(fs) = &lq.has_null_parent { + if *fs { + sqlb.and_where_is_null("parent_job"); + } + } + + if let Some(dt) = &lq.created_before { + sqlb.and_where_le("v2_job.created_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.created_after { + sqlb.and_where_ge("v2_job.created_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.created_or_started_after { + let ts = dt.timestamp_millis(); + sqlb.and_where(format!("(started_at IS NOT NULL AND started_at >= to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at >= to_timestamp({} / 1000.0))", ts, ts)); + } + + if let Some(dt) = &lq.created_or_started_before { + let ts = dt.timestamp_millis(); + sqlb.and_where(format!("(started_at IS NOT NULL AND started_at < to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at < to_timestamp({} / 1000.0))", ts, ts)); + } + + if let Some(s) = &lq.suspended { + if *s { + sqlb.and_where_gt("suspend", 0); + } else { + sqlb.and_where_is_null("suspend_until"); + } + } + + if let Some(jk) = &lq.job_kinds { + sqlb.and_where_in( + "kind", + &jk.split(',').into_iter().map(quote).collect::>(), + ); + } + + if let Some(args) = &lq.args { + sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + } + + if lq.scheduled_for_before_now.is_some_and(|x| x) { + sqlb.and_where_le("scheduled_for", "now()"); + } + + if lq.is_not_schedule.unwrap_or(false) { + sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); + } + + if let Some(tk) = &lq.trigger_kind { + sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); + } + + if let Some(tp) = &lq.trigger_path { + sqlb.and_where_eq("trigger", "?".bind(tp)); + } + + sqlb +} + +pub fn list_queue_jobs_query( + w_id: &str, + lq: &ListQueueQuery, + fields: &[&str], + pagination: Pagination, + join_outstanding_wait_times: bool, + tags: Option>, +) -> SqlBuilder { + let (limit, offset) = paginate_without_limits(pagination); + let mut sqlb = SqlBuilder::select_from("v2_job_queue") + .fields(fields) + .order_by("v2_job.created_at", lq.order_desc.unwrap_or(true)) + .limit(limit) + .offset(offset) + .clone(); + + if let Some(tags) = tags { + sqlb.and_where_in( + "v2_job.tag", + &tags.iter().map(|x| quote(x)).collect::>(), + ); + } + + filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times) +} + +pub fn filter_list_completed_query( + mut sqlb: SqlBuilder, + lq: &ListCompletedQuery, + w_id: &str, + join_outstanding_wait_times: bool, +) -> SqlBuilder { + sqlb.join("v2_job") + .on_eq("v2_job_completed.id", "v2_job.id"); + + if join_outstanding_wait_times { + sqlb.left() + .join("outstanding_wait_time") + .on_eq("v2_job_completed.id", "outstanding_wait_time.job_id"); + } + + if let Some(label) = &lq.label { + if lq.allow_wildcards.unwrap_or(false) { + let wh = format!( + "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", + &label.replace("*", "%").replace("'", "''") + ); + sqlb.and_where("result ? 'wm_labels'"); + sqlb.and_where(&wh); + } else { + let mut wh = format!("result->'wm_labels' ? "); + wh.push_str(&format!("'{}'", &label.replace("'", "''"))); + sqlb.and_where("result ? 'wm_labels'"); + sqlb.and_where(&wh); + } + } + + if let Some(worker) = &lq.worker { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job_completed.worker", worker.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job_completed.worker", "?".bind(worker)); + } + } + + if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { + sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)) + .and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); + } + + if let Some(p) = &lq.schedule_path { + sqlb.and_where_eq("trigger", "?".bind(p)); + sqlb.and_where_eq("trigger_kind", "'schedule'"); + } + + if let Some(ps) = &lq.script_path_start { + sqlb.and_where_like_left("runnable_path", ps); + } + if let Some(p) = &lq.script_path_exact { + sqlb.and_where_eq("runnable_path", "?".bind(p)); + } + if let Some(h) = &lq.script_hash { + sqlb.and_where_eq("runnable_id", "?".bind(h)); + } + if let Some(t) = &lq.tag { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job.tag", "?".bind(t)); + } + } + + if let Some(cb) = &lq.created_by { + sqlb.and_where_eq("created_by", "?".bind(cb)); + } + if let Some(r) = &lq.success { + if *r { + sqlb.and_where_eq("status", "'success'") + .or_where_eq("status", "'skipped'"); + } else { + sqlb.and_where_eq("status", "'failure'") + .or_where_eq("status", "'canceled'"); + } + } + if let Some(pj) = &lq.parent_job { + sqlb.and_where_eq("parent_job", "?".bind(pj)); + } + if let Some(dt) = &lq.started_before { + sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.started_after { + sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.created_or_started_before { + sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.created_or_started_after { + let ts = dt.to_rfc3339(); + sqlb.and_where(format!( + "(created_at >= '{}' OR started_at >= '{}')", + ts.replace("'", "''"), + ts.replace("'", "''") + )); + } + + if let Some(dt) = &lq.created_before { + sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.created_after { + sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.created_or_started_after_completed_jobs { + sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.completed_after { + sqlb.and_where_ge("completed_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.completed_before { + sqlb.and_where_le("completed_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(sk) = &lq.is_skipped { + if *sk { + sqlb.and_where_eq("status", "'skipped'"); + } else { + sqlb.and_where_ne("status", "'skipped'"); + } + } + if let Some(fs) = &lq.is_flow_step { + if *fs { + sqlb.and_where_is_not_null("flow_step_id"); + } else { + sqlb.and_where_is_null("flow_step_id"); + } + } + if let Some(fs) = &lq.has_null_parent { + if *fs { + sqlb.and_where_is_null("parent_job"); + } + } + if let Some(jk) = &lq.job_kinds { + sqlb.and_where_in( + "kind", + &jk.split(',').into_iter().map(quote).collect::>(), + ); + } + + if let Some(args) = &lq.args { + sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + } + + if let Some(result) = &lq.result { + sqlb.and_where("result @> ?".bind(&result.replace("'", "''"))); + } + + if lq.is_not_schedule.unwrap_or(false) { + sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); + } + + if let Some(tk) = &lq.trigger_kind { + sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); + } + + if let Some(tp) = &lq.trigger_path { + sqlb.and_where_eq("trigger", "?".bind(tp)); + } + + sqlb +} + +pub fn list_completed_jobs_query( + w_id: &str, + per_page: Option, + offset: usize, + lq: &ListCompletedQuery, + fields: &[&str], + join_outstanding_wait_times: bool, + tags: Option>, +) -> SqlBuilder { + let mut sqlb = SqlBuilder::select_from("v2_job_completed") + .fields(fields) + .order_by( + if lq.completed_before.is_some() || lq.completed_after.is_some() { + "v2_job_completed.completed_at" + } else { + "v2_job.created_at" + }, + lq.order_desc.unwrap_or(true), + ) + .offset(offset) + .clone(); + if let Some(per_page) = per_page { + sqlb.limit(per_page); + } + + if let Some(tags) = tags { + sqlb.and_where_in( + "v2_job.tag", + &tags.iter().map(|x| quote(x)).collect::>(), + ); + } + + filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_queue_query() -> ListQueueQuery { + ListQueueQuery { + script_path_start: None, + script_path_exact: None, + script_hash: None, + created_by: None, + started_before: None, + started_after: None, + created_before: None, + created_after: None, + created_or_started_before: None, + created_or_started_after: None, + running: None, + parent_job: None, + order_desc: None, + job_kinds: None, + suspended: None, + args: None, + tag: None, + schedule_path: None, + scheduled_for_before_now: None, + all_workspaces: None, + is_flow_step: None, + has_null_parent: None, + is_not_schedule: None, + concurrency_key: None, + worker: None, + allow_wildcards: None, + trigger_kind: None, + trigger_path: None, + include_args: None, + } + } + + fn empty_completed_query() -> ListCompletedQuery { + ListCompletedQuery { + script_path_start: None, + script_path_exact: None, + script_hash: None, + created_by: None, + started_before: None, + started_after: None, + created_before: None, + created_after: None, + created_or_started_before: None, + created_or_started_after: None, + created_or_started_after_completed_jobs: None, + created_before_queue: None, + created_after_queue: None, + completed_after: None, + completed_before: None, + success: None, + running: None, + parent_job: None, + order_desc: None, + job_kinds: None, + is_skipped: None, + is_flow_step: None, + suspended: None, + schedule_path: None, + args: None, + result: None, + tag: None, + scheduled_for_before_now: None, + all_workspaces: None, + has_null_parent: None, + label: None, + is_not_schedule: None, + concurrency_key: None, + worker: None, + allow_wildcards: None, + trigger_kind: None, + trigger_path: None, + include_args: None, + } + } + + fn build_sql(sqlb: SqlBuilder) -> String { + sqlb.sql().unwrap_or_default() + } + + // --- Queue query tests --- + + #[test] + fn test_queue_basic_query() { + let lq = empty_queue_query(); + let sqlb = list_queue_jobs_query( + "test_ws", + &lq, + &["v2_job_queue.id"], + Pagination { page: Some(1), per_page: Some(10) }, + false, + None, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("v2_job_queue")); + assert!(sql.contains("v2_job_queue.workspace_id")); + } + + #[test] + fn test_queue_filter_script_path_start() { + let lq = ListQueueQuery { + script_path_start: Some("f/test".to_string()), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("runnable_path")); + assert!(sql.contains("LIKE")); + } + + #[test] + fn test_queue_filter_script_path_exact() { + let lq = ListQueueQuery { + script_path_exact: Some("f/test/script".to_string()), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("runnable_path")); + } + + #[test] + fn test_queue_filter_running() { + let lq = ListQueueQuery { + running: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("running")); + } + + #[test] + fn test_queue_filter_job_kinds() { + let lq = ListQueueQuery { + job_kinds: Some("script,flow".to_string()), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("kind")); + assert!(sql.contains("IN")); + } + + #[test] + fn test_queue_filter_suspended() { + let lq = ListQueueQuery { + suspended: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("suspend")); + } + + #[test] + fn test_queue_filter_is_not_schedule() { + let lq = ListQueueQuery { + is_not_schedule: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("trigger_kind IS DISTINCT FROM")); + } + + #[test] + fn test_queue_filter_has_null_parent() { + let lq = ListQueueQuery { + has_null_parent: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("parent_job IS NULL")); + } + + #[test] + fn test_queue_filter_is_flow_step_true() { + let lq = ListQueueQuery { + is_flow_step: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("flow_step_id IS NOT NULL")); + } + + #[test] + fn test_queue_filter_is_flow_step_false() { + let lq = ListQueueQuery { + is_flow_step: Some(false), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("flow_step_id IS NULL")); + } + + #[test] + fn test_queue_admins_all_workspaces() { + let lq = ListQueueQuery { + all_workspaces: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "admins", + false, + ); + let sql = build_sql(sqlb); + assert!(!sql.contains("workspace_id")); + } + + #[test] + fn test_queue_non_admins_ignores_all_workspaces() { + let lq = ListQueueQuery { + all_workspaces: Some(true), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "other_ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("workspace_id")); + } + + #[test] + fn test_queue_with_outstanding_wait_times() { + let lq = empty_queue_query(); + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + true, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("outstanding_wait_time")); + } + + #[test] + fn test_queue_schedule_path_filter() { + let lq = ListQueueQuery { + schedule_path: Some("f/test/schedule".to_string()), + ..empty_queue_query() + }; + let sqlb = filter_list_queue_query( + SqlBuilder::select_from("v2_job_queue").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("trigger")); + assert!(sql.contains("'schedule'")); + } + + // --- Completed query tests --- + + #[test] + fn test_completed_basic_query() { + let lq = empty_completed_query(); + let sqlb = list_completed_jobs_query("test_ws", Some(10), 0, &lq, &["id"], false, None); + let sql = build_sql(sqlb); + assert!(sql.contains("v2_job_completed")); + } + + #[test] + fn test_completed_filter_success_true() { + let lq = ListCompletedQuery { + success: Some(true), + ..empty_completed_query() + }; + let sqlb = filter_list_completed_query( + SqlBuilder::select_from("v2_job_completed").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("'success'")); + } + + #[test] + fn test_completed_filter_success_false() { + let lq = ListCompletedQuery { + success: Some(false), + ..empty_completed_query() + }; + let sqlb = filter_list_completed_query( + SqlBuilder::select_from("v2_job_completed").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("'failure'")); + } + + #[test] + fn test_completed_order_by_completed_at() { + let lq = ListCompletedQuery { + completed_after: Some(chrono::Utc::now()), + ..empty_completed_query() + }; + let sqlb = list_completed_jobs_query("ws", Some(10), 0, &lq, &["id"], false, None); + let sql = build_sql(sqlb); + assert!(sql.contains("completed_at")); + } + + #[test] + fn test_completed_filter_label() { + let lq = ListCompletedQuery { + label: Some("deploy".to_string()), + ..empty_completed_query() + }; + let sqlb = filter_list_completed_query( + SqlBuilder::select_from("v2_job_completed").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("wm_labels")); + } + + #[test] + fn test_completed_filter_is_skipped() { + let lq = ListCompletedQuery { + is_skipped: Some(true), + ..empty_completed_query() + }; + let sqlb = filter_list_completed_query( + SqlBuilder::select_from("v2_job_completed").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("'skipped'")); + } + + #[test] + fn test_completed_with_tags() { + let lq = empty_completed_query(); + let sqlb = list_completed_jobs_query( + "ws", + Some(10), + 0, + &lq, + &["id"], + false, + Some(vec!["tag1", "tag2"]), + ); + let sql = build_sql(sqlb); + assert!(sql.contains("v2_job.tag")); + assert!(sql.contains("IN")); + } + + #[test] + fn test_completed_no_limit() { + let lq = empty_completed_query(); + let sqlb = list_completed_jobs_query("ws", None, 0, &lq, &["id"], false, None); + let sql = build_sql(sqlb); + assert!(!sql.contains("LIMIT")); + } + + #[test] + fn test_completed_result_filter() { + let lq = ListCompletedQuery { + result: Some(r#"{"status": "ok"}"#.to_string()), + ..empty_completed_query() + }; + let sqlb = filter_list_completed_query( + SqlBuilder::select_from("v2_job_completed").clone(), + &lq, + "ws", + false, + ); + let sql = build_sql(sqlb); + assert!(sql.contains("result @>")); + } +} diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs new file mode 100644 index 0000000000..5ca13645e7 --- /dev/null +++ b/backend/windmill-api-jobs/src/types.rs @@ -0,0 +1,846 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Shared types for the jobs subsystem. + +use axum::{ + extract::{FromRequest, Json, Query, Request}, + response::{IntoResponse, Response}, +}; +use http::StatusCode; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use uuid::Uuid; +use windmill_common::{ + error, + jobs::{CompletedJob, JobKind, JobTriggerKind, QueuedJob}, + scripts::{ScriptHash, ScriptLang}, + utils::now_from_db, + DB, +}; + +use windmill_api_sse::{Job, JobExtended}; + +// ------------ RunJobQuery ------------ + +#[derive(Debug, Deserialize, Clone, Default)] +pub struct RunJobQuery { + pub scheduled_for: Option>, + pub scheduled_in_secs: Option, + pub parent_job: Option, + pub root_job: Option, + pub invisible_to_owner: Option, + pub queue_limit: Option, + pub payload: Option, + pub job_id: Option, + pub tag: Option, + pub timeout: Option, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub skip_preprocessor: Option, + pub poll_delay_ms: Option, + pub memory_id: Option, + pub trigger_external_id: Option, + pub service_name: Option, + pub suspended_mode: Option, +} + +impl RunJobQuery { + pub async fn get_scheduled_for( + &self, + db: &DB, + ) -> error::Result>> { + if let Some(scheduled_for) = self.scheduled_for { + Ok(Some(scheduled_for)) + } else if let Some(scheduled_in_secs) = self.scheduled_in_secs { + let now = now_from_db(db).await?; + Ok(Some( + now + chrono::Duration::try_seconds(scheduled_in_secs).unwrap_or_default(), + )) + } else { + Ok(None) + } + } + + pub fn payload_as_args(&self) -> error::Result>> { + let payload_r = self.payload.clone().map(decode_payload).map(|x| { + x.map_err(|e| { + error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) + }) + }); + + let payload_as_args = if let Some(payload) = payload_r { + payload? + } else { + HashMap::new() + }; + + Ok(payload_as_args) + } +} + +// ------------ List query types ------------ + +#[derive(Deserialize, Clone)] +pub struct ListQueueQuery { + pub script_path_start: Option, + pub script_path_exact: Option, + pub script_hash: Option, + pub created_by: Option, + pub started_before: Option>, + pub started_after: Option>, + pub created_before: Option>, + pub created_after: Option>, + pub created_or_started_before: Option>, + pub created_or_started_after: Option>, + pub running: Option, + pub schedule_path: Option, + pub parent_job: Option, + pub order_desc: Option, + pub job_kinds: Option, + pub suspended: Option, + pub worker: Option, + // filter by matching a subset of the args using base64 encoded json subset + pub args: Option, + pub tag: Option, + pub scheduled_for_before_now: Option, + pub all_workspaces: Option, + pub is_flow_step: Option, + pub has_null_parent: Option, + pub is_not_schedule: Option, + pub concurrency_key: Option, + pub allow_wildcards: Option, + pub trigger_kind: Option, + pub trigger_path: Option, + pub include_args: Option, +} + +#[derive(Deserialize, Clone)] +pub struct ListCompletedQuery { + pub script_path_start: Option, + pub script_path_exact: Option, + pub script_hash: Option, + pub created_by: Option, + pub started_before: Option>, + pub started_after: Option>, + pub created_before: Option>, + pub created_after: Option>, + pub created_or_started_before: Option>, + pub created_or_started_after: Option>, + pub created_or_started_after_completed_jobs: Option>, + pub created_before_queue: Option>, + pub created_after_queue: Option>, + pub completed_after: Option>, + pub completed_before: Option>, + pub success: Option, + pub running: Option, + pub parent_job: Option, + pub order_desc: Option, + pub job_kinds: Option, + pub is_skipped: Option, + pub is_flow_step: Option, + pub suspended: Option, + pub schedule_path: Option, + // filter by matching a subset of the args using base64 encoded json subset + pub args: Option, + // filter by matching a subset of the result using base64 encoded json subset + pub result: Option, + pub tag: Option, + pub scheduled_for_before_now: Option, + pub all_workspaces: Option, + pub has_null_parent: Option, + pub label: Option, + pub is_not_schedule: Option, + pub concurrency_key: Option, + pub worker: Option, + pub allow_wildcards: Option, + pub trigger_kind: Option, + pub trigger_path: Option, + pub include_args: Option, +} + +impl From for ListQueueQuery { + fn from(lcq: ListCompletedQuery) -> Self { + Self { + script_path_start: lcq.script_path_start, + script_path_exact: lcq.script_path_exact, + script_hash: lcq.script_hash, + created_by: lcq.created_by, + started_before: lcq.started_before, + started_after: lcq.started_after, + created_before: lcq.created_before_queue.or(lcq.created_before), + created_after: lcq.created_after_queue.or(lcq.created_after), + created_or_started_before: lcq.created_or_started_before, + created_or_started_after: lcq.created_or_started_after, + worker: lcq.worker, + running: lcq.running, + parent_job: lcq.parent_job, + order_desc: lcq.order_desc, + job_kinds: lcq.job_kinds, + suspended: lcq.suspended, + args: lcq.args, + tag: lcq.tag, + schedule_path: lcq.schedule_path, + scheduled_for_before_now: lcq.scheduled_for_before_now, + all_workspaces: lcq.all_workspaces, + is_flow_step: lcq.is_flow_step, + has_null_parent: lcq.has_null_parent, + is_not_schedule: lcq.is_not_schedule, + concurrency_key: lcq.concurrency_key, + allow_wildcards: lcq.allow_wildcards, + trigger_kind: lcq.trigger_kind, + trigger_path: lcq.trigger_path, + include_args: lcq.include_args, + } + } +} + +// ------------ ListableCompletedJob ------------ + +#[derive(Debug, sqlx::FromRow, Serialize)] +pub struct ListableCompletedJob { + pub r#type: String, + pub workspace_id: String, + pub id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub duration_ms: i64, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_path: Option, + pub deleted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_code: Option, + pub canceled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_reason: Option, + pub job_kind: JobKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + pub permissioned_as: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_flow: Option, + pub is_flow_step: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + pub is_skipped: bool, + pub email: String, + pub visible_to_owner: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + pub tag: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub args: Option, +} + +// ------------ UnifiedJob ------------ + +#[derive(sqlx::FromRow)] +pub struct UnifiedJob { + pub workspace_id: String, + pub typ: String, + pub id: Uuid, + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub completed_at: Option>, + pub scheduled_for: Option>, + pub running: Option, + pub script_hash: Option, + pub script_path: Option, + pub args: Option, + pub duration_ms: Option, + pub success: Option, + pub deleted: bool, + pub canceled: bool, + pub canceled_by: Option, + pub job_kind: JobKind, + pub schedule_path: Option, + pub permissioned_as: String, + pub is_flow_step: bool, + pub language: Option, + pub is_skipped: bool, + pub email: String, + pub visible_to_owner: bool, + pub suspend: Option, + pub mem_peak: Option, + pub tag: String, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub priority: Option, + pub labels: Option, + pub self_wait_time_ms: Option, + pub aggregate_wait_time_ms: Option, + pub preprocessed: Option, + pub worker: Option, + pub runnable_settings_handle: Option, +} + +const CJ_FIELDS: &[&str] = &[ + "'CompletedJob' as typ", + "v2_job_completed.id", + "v2_job_completed.workspace_id", + "v2_job.parent_job", + "v2_job.created_by", + "v2_job.created_at", + "v2_job_completed.started_at", + "null as scheduled_for", + "v2_job_completed.completed_at", + "null as running", + "v2_job.runnable_id as script_hash", + "v2_job.runnable_path as script_path", + "null as args", + "v2_job_completed.duration_ms", + "v2_job_completed.status = 'success' OR v2_job_completed.status = 'skipped' as success", + "false as deleted", + "v2_job_completed.status = 'canceled' as canceled", + "v2_job_completed.canceled_by", + "v2_job.kind as job_kind", + "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", + "v2_job.permissioned_as", + "v2_job.flow_step_id IS NOT NULL as is_flow_step", + "v2_job.script_lang as language", + "v2_job_completed.status = 'skipped' as is_skipped", + "v2_job.permissioned_as_email as email", + "v2_job.visible_to_owner", + "null as suspend", + "v2_job_completed.memory_peak as mem_peak", + "v2_job.tag", + "null as concurrent_limit", + "null as concurrency_time_window_s", + "v2_job.priority", + "v2_job_completed.result->'wm_labels' as labels", + "self_wait_time_ms", + "aggregate_wait_time_ms", + "v2_job.preprocessed", + "v2_job_completed.worker", + "null as runnable_settings_handle", +]; + +const QJ_FIELDS: &[&str] = &[ + "'QueuedJob' as typ", + "v2_job_queue.id", + "v2_job_queue.workspace_id", + "v2_job.parent_job", + "v2_job.created_by", + "v2_job_queue.created_at", + "v2_job_queue.started_at", + "v2_job_queue.scheduled_for", + "null as completed_at", + "v2_job_queue.running", + "v2_job.runnable_id as script_hash", + "v2_job.runnable_path as script_path", + "null as args", + "null as duration_ms", + "null as success", + "false as deleted", + "v2_job_queue.canceled_by IS NOT NULL as canceled", + "v2_job_queue.canceled_by", + "v2_job.kind as job_kind", + "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", + "v2_job.permissioned_as", + "v2_job.flow_step_id IS NOT NULL as is_flow_step", + "v2_job.script_lang as language", + "false as is_skipped", + "v2_job.permissioned_as_email as email", + "v2_job.visible_to_owner", + "v2_job_queue.suspend", + "null as mem_peak", + "v2_job.tag", + "v2_job.concurrent_limit", + "v2_job.concurrency_time_window_s", + "v2_job.priority", + "null as labels", + "self_wait_time_ms", + "aggregate_wait_time_ms", + "v2_job.preprocessed", + "v2_job_queue.worker", + "v2_job_queue.runnable_settings_handle", +]; + +impl UnifiedJob { + pub fn completed_job_fields() -> &'static [&'static str] { + CJ_FIELDS + } + pub fn queued_job_fields() -> &'static [&'static str] { + QJ_FIELDS + } +} + +impl From for Job { + fn from(uj: UnifiedJob) -> Self { + let args = uj.args.and_then(|v| serde_json::from_value(v).ok()); + match uj.typ.as_ref() { + "CompletedJob" => Job::CompletedJob(JobExtended::new( + uj.self_wait_time_ms, + uj.aggregate_wait_time_ms, + CompletedJob { + workspace_id: uj.workspace_id, + id: uj.id, + parent_job: uj.parent_job, + created_by: uj.created_by, + created_at: uj.created_at, + started_at: uj.started_at, + completed_at: uj.completed_at, + duration_ms: uj.duration_ms.unwrap(), + success: uj.success.unwrap(), + script_hash: uj.script_hash, + script_path: uj.script_path, + args: args.clone(), + result: None, + result_columns: None, + logs: None, + flow_status: None, + workflow_as_code_status: None, + deleted: uj.deleted, + canceled: uj.canceled, + canceled_by: uj.canceled_by, + canceled_reason: None, + job_kind: uj.job_kind, + schedule_path: uj.schedule_path, + permissioned_as: uj.permissioned_as, + is_flow_step: uj.is_flow_step, + language: uj.language, + is_skipped: uj.is_skipped, + email: uj.email, + visible_to_owner: uj.visible_to_owner, + mem_peak: uj.mem_peak, + tag: uj.tag, + priority: uj.priority, + labels: uj.labels, + preprocessed: uj.preprocessed, + }, + )), + "QueuedJob" => Job::QueuedJob(JobExtended::new( + uj.self_wait_time_ms, + uj.aggregate_wait_time_ms, + QueuedJob { + workspace_id: uj.workspace_id, + id: uj.id, + parent_job: uj.parent_job, + created_by: uj.created_by, + created_at: uj.created_at, + started_at: uj.started_at, + scheduled_for: uj.scheduled_for.unwrap(), + running: uj.running.unwrap(), + script_hash: uj.script_hash, + script_path: uj.script_path, + script_entrypoint_override: None, + args, + logs: None, + canceled: uj.canceled, + canceled_by: uj.canceled_by, + canceled_reason: None, + last_ping: None, + job_kind: uj.job_kind, + schedule_path: uj.schedule_path, + permissioned_as: uj.permissioned_as, + flow_status: None, + workflow_as_code_status: None, + is_flow_step: uj.is_flow_step, + language: uj.language, + same_worker: false, + pre_run_error: None, + email: uj.email, + visible_to_owner: uj.visible_to_owner, + suspend: uj.suspend, + mem_peak: uj.mem_peak, + root_job: None, + leaf_jobs: None, + tag: uj.tag, + concurrent_limit: uj.concurrent_limit, + concurrency_time_window_s: uj.concurrency_time_window_s, + timeout: None, + flow_step_id: None, + cache_ttl: None, + cache_ignore_s3_path: None, + priority: uj.priority, + preprocessed: uj.preprocessed, + runnable_settings_handle: uj.runnable_settings_handle, + }, + )), + t => panic!("job type {} not valid", t), + } + } +} + +// ------------ Approval types ------------ + +#[derive(Deserialize, Debug)] +pub struct QueryApprover { + pub approver: Option, + /// If true, generate/verify resume URLs for the parent flow instead of the specific step. + /// This allows pre-approvals that can be consumed by any later suspend step in the same flow. + pub flow_level: Option, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Debug)] +pub struct ResumeUrls { + pub approvalPage: String, + pub cancel: String, + pub resume: String, +} + +// ------------ QueryOrBody extractor ------------ + +pub struct QueryOrBody(pub Option); + +#[axum::async_trait] +impl FromRequest for QueryOrBody +where + D: DeserializeOwned, + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request( + req: Request, + state: &S, + ) -> std::result::Result { + return if req.method() == axum::http::Method::GET { + let Query(InPayload { payload }) = Query::from_request(req, state) + .await + .map_err(IntoResponse::into_response)?; + payload + .map(|p| { + decode_payload(p) + .map(QueryOrBody) + .map_err(|err| (StatusCode::BAD_REQUEST, format!("{err:#?}"))) + .map_err(IntoResponse::into_response) + }) + .unwrap_or(Ok(QueryOrBody(None))) + } else { + Json::from_request(req, state) + .await + .map(|Json(v)| QueryOrBody(Some(v))) + .map_err(IntoResponse::into_response) + }; + + #[derive(Deserialize)] + struct InPayload { + payload: Option, + } + } +} + +// ------------ Utility functions ------------ + +pub fn decode_payload(t: String) -> anyhow::Result { + let vec = base64::engine::general_purpose::STANDARD + .decode(t) + .context("invalid base64")?; + serde_json::from_slice(vec.as_slice()).context("invalid json") +} + +pub fn add_raw_string( + raw_string: Option, + mut args: serde_json::Map, +) -> serde_json::Map { + if let Some(raw_string) = raw_string { + args.insert( + "raw_string".to_string(), + serde_json::Value::String(raw_string), + ); + } + return args; +} + +use anyhow::Context; +use base64::Engine; + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + // --- decode_payload --- + + #[test] + fn test_decode_payload_valid() { + let payload = base64::engine::general_purpose::STANDARD + .encode(r#"{"key": "value"}"#); + let result: HashMap = decode_payload(payload).unwrap(); + assert_eq!(result["key"], json!("value")); + } + + #[test] + fn test_decode_payload_invalid_base64() { + let result: anyhow::Result> = + decode_payload("not-valid-base64!!!".to_string()); + assert!(result.is_err()); + } + + #[test] + fn test_decode_payload_invalid_json() { + let payload = base64::engine::general_purpose::STANDARD.encode("not json"); + let result: anyhow::Result> = decode_payload(payload); + assert!(result.is_err()); + } + + #[test] + fn test_decode_payload_empty_object() { + let payload = base64::engine::general_purpose::STANDARD.encode("{}"); + let result: HashMap = decode_payload(payload).unwrap(); + assert!(result.is_empty()); + } + + // --- add_raw_string --- + + #[test] + fn test_add_raw_string_some() { + let args = serde_json::Map::new(); + let result = add_raw_string(Some("body content".to_string()), args); + assert_eq!( + result["raw_string"], + serde_json::Value::String("body content".to_string()) + ); + } + + #[test] + fn test_add_raw_string_none() { + let args = serde_json::Map::new(); + let result = add_raw_string(None, args); + assert!(!result.contains_key("raw_string")); + } + + #[test] + fn test_add_raw_string_preserves_existing() { + let mut args = serde_json::Map::new(); + args.insert("existing".to_string(), json!("value")); + let result = add_raw_string(Some("body".to_string()), args); + assert_eq!(result["existing"], json!("value")); + assert_eq!(result["raw_string"], json!("body")); + } + + // --- RunJobQuery --- + + #[test] + fn test_run_job_query_payload_as_args_none() { + let q = RunJobQuery::default(); + let result = q.payload_as_args().unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn test_run_job_query_payload_as_args_valid() { + let encoded = base64::engine::general_purpose::STANDARD + .encode(r#"{"x": 42}"#); + let q = RunJobQuery { + payload: Some(encoded), + ..Default::default() + }; + let result = q.payload_as_args().unwrap(); + assert!(result.contains_key("x")); + } + + #[test] + fn test_run_job_query_payload_as_args_invalid() { + let q = RunJobQuery { + payload: Some("invalid!!!".to_string()), + ..Default::default() + }; + assert!(q.payload_as_args().is_err()); + } + + // --- ListCompletedQuery -> ListQueueQuery conversion --- + + #[test] + fn test_list_completed_to_queue_query_conversion() { + let lcq = ListCompletedQuery { + script_path_start: Some("f/test".to_string()), + script_path_exact: None, + script_hash: None, + created_by: Some("admin".to_string()), + started_before: None, + started_after: None, + created_before: Some(chrono::Utc::now()), + created_after: None, + created_or_started_before: None, + created_or_started_after: None, + created_or_started_after_completed_jobs: None, + created_before_queue: None, + created_after_queue: None, + completed_after: None, + completed_before: None, + success: None, + running: Some(true), + parent_job: None, + order_desc: Some(true), + job_kinds: Some("script,flow".to_string()), + is_skipped: None, + is_flow_step: None, + suspended: None, + schedule_path: None, + args: None, + result: None, + tag: Some("custom".to_string()), + scheduled_for_before_now: None, + all_workspaces: None, + has_null_parent: None, + label: None, + is_not_schedule: None, + concurrency_key: None, + worker: None, + allow_wildcards: None, + trigger_kind: None, + trigger_path: None, + include_args: None, + }; + + let lqq: ListQueueQuery = lcq.into(); + assert_eq!(lqq.script_path_start, Some("f/test".to_string())); + assert_eq!(lqq.created_by, Some("admin".to_string())); + assert_eq!(lqq.running, Some(true)); + assert_eq!(lqq.job_kinds, Some("script,flow".to_string())); + assert_eq!(lqq.tag, Some("custom".to_string())); + } + + #[test] + fn test_list_completed_to_queue_prefers_queue_created_fields() { + let specific_time = chrono::Utc::now(); + let other_time = specific_time - chrono::Duration::hours(1); + + let lcq = ListCompletedQuery { + script_path_start: None, + script_path_exact: None, + script_hash: None, + created_by: None, + started_before: None, + started_after: None, + created_before: Some(other_time), + created_after: Some(other_time), + created_or_started_before: None, + created_or_started_after: None, + created_or_started_after_completed_jobs: None, + created_before_queue: Some(specific_time), + created_after_queue: Some(specific_time), + completed_after: None, + completed_before: None, + success: None, + running: None, + parent_job: None, + order_desc: None, + job_kinds: None, + is_skipped: None, + is_flow_step: None, + suspended: None, + schedule_path: None, + args: None, + result: None, + tag: None, + scheduled_for_before_now: None, + all_workspaces: None, + has_null_parent: None, + label: None, + is_not_schedule: None, + concurrency_key: None, + worker: None, + allow_wildcards: None, + trigger_kind: None, + trigger_path: None, + include_args: None, + }; + + let lqq: ListQueueQuery = lcq.into(); + assert_eq!(lqq.created_before, Some(specific_time)); + assert_eq!(lqq.created_after, Some(specific_time)); + } + + // --- UnifiedJob field constants --- + + #[test] + fn test_completed_job_fields_not_empty() { + let fields = UnifiedJob::completed_job_fields(); + assert!(!fields.is_empty()); + assert!(fields.iter().any(|f| f.contains("typ"))); + assert!(fields.iter().any(|f| f.contains("workspace_id"))); + } + + #[test] + fn test_queued_job_fields_not_empty() { + let fields = UnifiedJob::queued_job_fields(); + assert!(!fields.is_empty()); + assert!(fields.iter().any(|f| f.contains("typ"))); + assert!(fields.iter().any(|f| f.contains("scheduled_for"))); + } + + #[test] + fn test_completed_and_queued_fields_same_count() { + assert_eq!( + UnifiedJob::completed_job_fields().len(), + UnifiedJob::queued_job_fields().len(), + "CJ and QJ field lists must have the same number of columns for UNION queries" + ); + } + + // --- ListableCompletedJob serialization --- + + #[test] + fn test_listable_completed_job_skip_none() { + let job = ListableCompletedJob { + r#type: "CompletedJob".to_string(), + workspace_id: "test".to_string(), + id: Uuid::nil(), + parent_job: None, + created_by: "admin".to_string(), + created_at: chrono::Utc::now(), + started_at: None, + duration_ms: 100, + success: true, + script_hash: None, + script_path: None, + deleted: false, + raw_code: None, + canceled: false, + canceled_by: None, + canceled_reason: None, + job_kind: JobKind::Script, + schedule_path: None, + permissioned_as: "u/admin".to_string(), + flow_status: None, + raw_flow: None, + is_flow_step: false, + language: None, + is_skipped: false, + email: "admin@test.com".to_string(), + visible_to_owner: true, + mem_peak: None, + tag: "default".to_string(), + priority: None, + labels: None, + args: None, + }; + + let json = serde_json::to_value(&job).unwrap(); + let obj = json.as_object().unwrap(); + assert!(!obj.contains_key("parent_job")); + assert!(!obj.contains_key("script_hash")); + assert!(!obj.contains_key("raw_code")); + assert!(!obj.contains_key("canceled_by")); + assert!(!obj.contains_key("mem_peak")); + assert!(!obj.contains_key("labels")); + assert!(obj.contains_key("type")); + assert!(obj.contains_key("workspace_id")); + } +} diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml new file mode 100644 index 0000000000..407348ee72 --- /dev/null +++ b/backend/windmill-api-scripts/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "windmill-api-scripts" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_scripts" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +private = ["windmill-common/private"] +python = ["dep:windmill-parser-py"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-queue.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +windmill-dep-map.workspace = true +windmill-parser-ts.workspace = true + +axum.workspace = true +futures.workspace = true +http.workspace = true +hyper.workspace = true +itertools.workspace = true +quick_cache.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +sql-builder.workspace = true +sqlx.workspace = true +tracing.workspace = true +chrono.workspace = true +lazy_static.workspace = true +tokio.workspace = true +windmill-parser-py = { workspace = true, optional = true } diff --git a/backend/windmill-api-scripts/src/flows.rs b/backend/windmill-api-scripts/src/flows.rs new file mode 100644 index 0000000000..adeb24d068 --- /dev/null +++ b/backend/windmill-api-scripts/src/flows.rs @@ -0,0 +1,1945 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::collections::HashMap; + +use axum::response::IntoResponse; +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use windmill_api_auth::{ + auth::{list_tokens_internal, TruncatedTokenWithEmail}, + check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed, +}; +use windmill_common::{ + utils::{WithStarredInfoQuery, HTTP_CLIENT}, + webhook::{WebhookMessage, WebhookShared}, + DB, +}; +use windmill_queue::schedule::clear_schedule; + +use hyper::StatusCode; +use serde::{Deserialize, Serialize}; +use sql_builder::prelude::*; +use sqlx::{FromRow, Postgres, Transaction}; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::assets::{clear_static_asset_usage, AssetUsageKind}; +use windmill_common::min_version::{ + MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2, +}; +use windmill_common::runnable_settings::RunnableSettingsTrait; +use windmill_common::utils::query_elems_from_hub; +use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; +use windmill_common::HUB_BASE_URL; +use windmill_common::{ + db::UserDB, + error::{self, to_anyhow, Error, JsonResult, Result}, + flows::{Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, + jobs::JobPayload, + schedule::Schedule, + scripts::Schema, + utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, +}; +use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_queue::WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT; +use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_flows)) + .route("/list_search", get(list_search_flows)) + .route("/create", post(create_flow)) + .route("/update/*path", post(update_flow)) + .route("/archive/*path", post(archive_flow_by_path)) + .route("/delete/*path", delete(delete_flow_by_path)) + .route("/list_tokens/*path", get(list_tokens)) + .route("/get/*path", get(get_flow_by_path)) + .route("/deployment_status/p/*path", get(get_deployment_status)) + .route("/get/draft/*path", get(get_flow_by_path_w_draft)) + .route("/exists/*path", get(exists_flow_by_path)) + .route("/list_paths", get(list_paths)) + .route("/history/p/*path", get(get_flow_history)) + .route("/get_latest_version/*path", get(get_latest_version)) + .route( + "/list_paths_from_workspace_runnable/:runnable_kind/*path", + get(list_paths_from_workspace_runnable), + ) + .route("/history_update/v/:version", post(update_flow_history)) + .route("/get/v/:version", get(get_flow_version_by_id)) + .route("/get/v/:version/p/*path", get(get_flow_version)) + .route( + "/toggle_workspace_error_handler/*path", + post(toggle_workspace_error_handler), + ) +} + +pub fn global_service() -> Router { + Router::new() + .route("/hub/list", get(list_hub_flows)) + .route("/hub/get/:id", get(get_hub_flow_by_id)) +} + +#[derive(Serialize, FromRow)] +pub struct SearchFlow { + path: String, + value: sqlx::types::Json>, +} +async fn list_search_flows( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + #[cfg(feature = "enterprise")] + let n = 1000; + + #[cfg(not(feature = "enterprise"))] + let n = 3; + let mut tx = user_db.begin(&authed).await?; + + let rows = sqlx::query_as::<_, SearchFlow>( + "SELECT flow.path, flow_version.value + FROM flow + LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.workspace_id = $1 LIMIT $2", + ) + .bind(&w_id) + .bind(n) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_flows( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, + Query(lq): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + + let mut sqlb = SqlBuilder::select_from("flow as o") + .fields(&[ + "o.workspace_id", + "o.path", + "summary", + if !lq.without_description.unwrap_or(false) { + "description" + } else { + "NULL as description" + }, + "fv.created_by as edited_by", + "fv.created_at as edited_at", + "archived", + "extra_perms", + "favorite.path IS NOT NULL as starred", + "draft.path IS NOT NULL as has_draft", + "draft_only", + "ws_error_handler_muted" + ]) + .left() + .join("favorite") + .on( + "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?" + .bind(&authed.username), + ) + .left() + .join("draft") + .on( + "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'" + ) + .left() + .join("flow_version fv") + .on( + "fv.id = o.versions[array_upper(o.versions, 1)]" + ) + .order_desc("favorite.path IS NOT NULL") + .order_by("fv.created_at", lq.order_desc.unwrap_or(true)) + .and_where("o.workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + + sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false)); + + if let Some(ps) = &lq.path_start { + sqlb.and_where_like_left("o.path", ps); + } + if let Some(p) = &lq.path_exact { + sqlb.and_where_eq("o.path", "?".bind(p)); + } + if let Some(cb) = &lq.edited_by { + sqlb.and_where_eq("fv.created_by", "?".bind(cb)); + } + if lq.starred_only.unwrap_or(false) { + sqlb.and_where_is_not_null("favorite.path"); + } + + if !lq.include_draft_only.unwrap_or(false) || authed.is_operator { + sqlb.and_where("o.draft_only IS NOT TRUE"); + } + if let Some(dw) = &lq.dedicated_worker { + sqlb.and_where_eq("dedicated_worker", dw); + } + + if lq.with_deployment_msg.unwrap_or(false) { + sqlb.join("deployment_metadata dm") + .left() + .on("dm.flow_version = o.versions[array_upper(o.versions, 1)]") + .fields(&["dm.deployment_msg"]); + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as::<_, ListableFlow>(&sql) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_hub_flows(Extension(db): Extension) -> impl IntoResponse { + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + &format!( + "{}/searchFlowData?approved=true", + *HUB_BASE_URL.read().await + ), + None, + &db, + ) + .await?; + Ok::<_, Error>((status_code, headers, response)) +} + +async fn list_paths( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + + let flows = sqlx::query_scalar!( + "SELECT distinct(path) FROM flow WHERE workspace_id = $1", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(flows)) +} + +pub async fn get_hub_flow_by_id( + Path(id): Path, + Extension(db): Extension, +) -> JsonResult> { + let value = http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id), + false, + None, + Some(&db), + ) + .await? + .json() + .await + .map_err(to_anyhow)?; + Ok(Json(value)) +} + +#[derive(Deserialize)] +pub struct ToggleWorkspaceErrorHandler { + #[cfg(feature = "enterprise")] + pub muted: Option, +} + +#[cfg(not(feature = "enterprise"))] +async fn toggle_workspace_error_handler( + _authed: ApiAuthed, + Extension(_user_db): Extension, + Path((_w_id, _path)): Path<(String, StripPath)>, + Json(_req): Json, +) -> Result { + return Err(Error::BadRequest( + "Muting the error handler for certain flow is only available in enterprise version" + .to_string(), + )); +} + +#[cfg(feature = "enterprise")] +async fn toggle_workspace_error_handler( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(req): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + let error_handler_maybe: Option = sqlx::query_scalar!( + r#" + SELECT + error_handler->>'path' + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(None); + + let response = match error_handler_maybe { + Some(_) => { + sqlx::query_scalar!( + r#" + UPDATE + flow + SET + ws_error_handler_muted = $3 + WHERE + path = $1 AND + workspace_id = $2 + "#, + path.to_path(), + w_id, + req.muted, + ) + .execute(&mut *tx) + .await?; + Ok("".to_string()) + } + None => Err(Error::BadRequest( + "Workspace error handler needs to be defined".to_string(), + )), + }; + + tx.commit().await?; + + return response; +} + +async fn check_path_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + path: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!("Flow {} already exists", path))); + } + return Ok(()); +} + +#[derive(Deserialize)] +struct ListPathsFromWorkspaceRunnableQuery { + match_path_start: Option, +} + +async fn list_paths_from_workspace_runnable( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, + Query(query): Query, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || { + format!("flows:read:{}", format!("{}/{}", runnable_kind, path)) + })?; + let mut tx = user_db.begin(&authed).await?; + + let runnables = if query.match_path_start.unwrap_or(false) { + sqlx::query_scalar!( + r#"SELECT DISTINCT f.path + FROM workspace_runnable_dependencies wru + JOIN flow f + ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id + WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + path, + matches!(runnable_kind, RunnableKind::Flow), + w_id + ) + .fetch_all(&mut *tx) + .await? + } else { + sqlx::query_scalar!( + r#"SELECT f.path + FROM workspace_runnable_dependencies wru + JOIN flow f + ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id + WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, + path, + matches!(runnable_kind, RunnableKind::Flow), + w_id + ) + .fetch_all(&mut *tx) + .await? + }; + + tx.commit().await?; + Ok(Json(runnables)) +} + +async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { + #[cfg(not(feature = "enterprise"))] + if new_flow.ws_error_handler_muted.is_some_and(|val| val) { + return Err(Error::BadRequest( + "Muting the error handler for certain flow is only available in enterprise version" + .to_string(), + )); + } + + guard_flow_from_debounce_data(new_flow).await?; + + return Ok(()); +} + +async fn create_flow( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(nf): Json, +) -> Result<(StatusCode, String)> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot create flows for security reasons".to_string(), + )); + } + check_scopes(&authed, || format!("flows:write:{}", nf.path))?; + validate_flow(&nf).await?; + if *CLOUD_HOSTED { + let nb_flows = + sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id) + .fetch_one(&db) + .await?; + if nb_flows.unwrap_or(0) >= 1000 { + return Err(Error::BadRequest( + "You have reached the maximum number of flows (1000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + if nf.summary.len() > 300 { + return Err(Error::BadRequest( + "Summary must be less than 300 characters on cloud".to_string(), + )); + } + if nf + .description + .as_ref() + .is_some_and(|desc| desc.len() > 3000) + { + return Err(Error::BadRequest( + "Description must be less than 3000 characters on cloud".to_string(), + )); + } + } + + // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?; + let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await; + + let mut tx = user_db.clone().begin(&authed).await?; + + check_path_conflict(&mut tx, &w_id, &nf.path).await?; + check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; + + let schema_str = nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()); + sqlx::query!( + r#"INSERT INTO flow ( + workspace_id, path, summary, description, + dependency_job, lock_error_logs, draft_only, tag, + dedicated_worker, visible_to_runner_only, on_behalf_of_email, + ws_error_handler_muted, + value, schema, edited_by, edited_at + ) VALUES ( + $1, $2, $3, $4, + NULL, '', $5, $6, + $7, $8, $9, + $10, + $11, $12::text::json, $13, now() + )"#, + w_id, + nf.path, + nf.summary, + nf.description.as_deref().unwrap_or(""), + nf.draft_only, + nf.tag, + nf.dedicated_worker, + nf.visible_to_runner_only.unwrap_or(false), + nf.on_behalf_of_email.and(Some(&authed.email)), + nf.ws_error_handler_muted.unwrap_or(false), + sqlx::types::Json(&nf.value) as _, + schema_str, + &authed.username, + ) + .execute(&mut *tx) + .await?; + + let version = sqlx::query_scalar!( + "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) + VALUES ($1, $2, $3, $4::text::json, $5) + RETURNING id", + w_id, + nf.path, + sqlx::types::Json(nf.value) as _, + schema_str, + &authed.username, + ) + .fetch_one(&mut *tx) + .await?; + + sqlx::query!( + "UPDATE flow SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3", + version, + nf.path, + w_id + ).execute(&mut *tx).await?; + + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'", + nf.path, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "flows.create", + ActionKind::Create, + &w_id, + Some(&nf.path.to_string()), + Some( + [Some(("flow", nf.path.as_str()))] + .into_iter() + .flatten() + .collect(), + ), + ) + .await?; + + let mut args: HashMap> = HashMap::new(); + if let Some(dm) = nf.deployment_message { + args.insert("deployment_message".to_string(), to_raw_value(&dm)); + } + + let tx = PushIsolationLevel::Transaction(tx); + let (dependency_job_uuid, mut new_tx) = push( + &db, + tx, + &w_id, + JobPayload::FlowDependencies { + path: nf.path.clone(), + dedicated_worker: nf.dedicated_worker, + version: version, + debouncing_settings: Default::default(), + }, + windmill_queue::PushArgs { args: &args, extra: None }, + &authed.username, + &authed.email, + windmill_common::users::username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + None, + ) + .await?; + + sqlx::query!( + "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3", + dependency_job_uuid, + nf.path, + w_id + ) + .execute(&mut *new_tx) + .await?; + + // Store the job_id in deployment_metadata for this flow deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + nf.path, + version, + dependency_job_uuid + ) + .execute(&mut *new_tx) + .await?; + + new_tx.commit().await?; + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, + ); + + Ok((StatusCode::CREATED, nf.path.to_string())) +} + +async fn check_schedule_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + path: &str, +) -> error::Result<()> { + let exists_flow = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2 AND path != \ + script_path)", + path, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists_flow { + return Err(error::Error::BadConfig(format!( + "A flow cannot have the same path as a schedule if the schedule does not trigger that \ + same flow: {path}", + ))); + }; + Ok(()) +} + +pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { + return windmill_api_auth::require_is_writer( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM flow WHERE path = $1 AND workspace_id = $2", + "flow", + ) + .await; +} + +#[derive(Serialize)] +pub struct FlowVersion { + pub id: i64, + pub created_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, +} + +async fn get_flow_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let flows = sqlx::query_as!( + FlowVersion, + "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC", + path, + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(flows)) +} + +async fn get_latest_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let version = sqlx::query_as!( + FlowVersion, + "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(version)) +} + +async fn get_flow_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, version, path)): Path<(String, i64, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let flow = sqlx::query_as::<_, Flow>( + "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by + FROM flow + LEFT JOIN flow_version ON flow_version.path = flow.path AND flow_version.workspace_id = flow.workspace_id + WHERE flow.path = $1 AND flow.workspace_id = $2 AND flow_version.id = $3", + ) + .bind(path) + .bind(w_id) + .bind(version) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + let flow = not_found_if_none(flow, "Flow version", version.to_string())?; + + Ok(Json(flow)) +} + +async fn get_flow_version_by_id( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, version)): Path<(String, i64)>, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + + // First, fetch the path to perform authorization check early + let path: Option = + sqlx::query_scalar("SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2") + .bind(version) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + let path = not_found_if_none( + path, + "Flow version", + format!("{} in workspace {}", version, w_id), + )?; + + // Perform authorization check before fetching full data + check_scopes(&authed, || format!("flows:read:{}", path))?; + + // Now fetch the full flow data with INNER JOIN to ensure flow exists + let flow = sqlx::query_as::<_, Flow>( + "SELECT + flow.workspace_id, + flow.path, + flow.summary, + flow.description, + flow.archived, + flow.extra_perms, + flow.draft_only, + flow.dedicated_worker, + flow.tag, + flow.ws_error_handler_muted, + flow.timeout, + flow.visible_to_runner_only, + flow.on_behalf_of_email, + flow_version.schema, + flow_version.value, + flow_version.created_at as edited_at, + flow_version.created_by as edited_by + FROM flow + INNER JOIN flow_version + ON flow_version.path = flow.path + AND flow_version.workspace_id = flow.workspace_id + WHERE flow_version.id = $1 AND flow.workspace_id = $2", + ) + .bind(version) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + let flow = not_found_if_none( + flow, + "Flow", + format!("for version {} (flow may have been deleted)", version), + )?; + + Ok(Json(flow)) +} + +#[derive(Deserialize)] +pub struct FlowHistoryUpdate { + pub deployment_msg: String, +} + +async fn update_flow_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, version)): Path<(String, i64)>, + Json(history_update): Json, +) -> Result<()> { + let mut tx = user_db.begin(&authed).await?; + + // Fetch path and perform authorization check early + let path: Option = + sqlx::query_scalar("SELECT path FROM flow_version WHERE workspace_id = $1 AND id = $2") + .bind(&w_id) + .bind(version) + .fetch_optional(&mut *tx) + .await?; + + let path = not_found_if_none( + path, + "Flow version", + format!("{} in workspace {}", version, w_id), + )?; + + // Perform authorization check before any modifications + check_scopes(&authed, || format!("flows:write:{}", path))?; + + // Insert or update deployment metadata + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", + &w_id, + path, + version, + history_update.deployment_msg, + ) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + Ok(()) +} + +async fn update_flow( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Extension(webhook): Extension, + Path((w_id, flow_path)): Path<(String, StripPath)>, + Json(nf): Json, +) -> Result { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot update flows for security reasons".to_string(), + )); + } + let flow_path = flow_path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", flow_path))?; + validate_flow(&nf).await?; + + let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await; + let mut tx = user_db.clone().begin(&authed).await?; + + check_schedule_conflict(&mut tx, &w_id, flow_path).await?; + + let schema = nf.schema.map(|x| x.0); + let old_dep_job = sqlx::query_scalar!( + "SELECT dependency_job FROM flow WHERE path = $1 AND workspace_id = $2", + flow_path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?; + let is_new_path = nf.path != flow_path; + let schema_str = schema.and_then(|x| serde_json::to_string(&x).ok()); + + sqlx::query!( + " + UPDATE + flow + SET + path = $1, + summary = $2, + description = $3, + dependency_job = NULL, + lock_error_logs = '', + draft_only = NULL, + tag = $4, + dedicated_worker = $5, + visible_to_runner_only = $6, + on_behalf_of_email = $7, + ws_error_handler_muted = $8, + value = $9, + schema = $10::text::json, + edited_by = $11, + edited_at = now() + WHERE + path = $12 AND workspace_id = $13", + if is_new_path { flow_path } else { &nf.path }, + nf.summary, + nf.description.as_deref().unwrap_or(""), + nf.tag, + nf.dedicated_worker, + nf.visible_to_runner_only.unwrap_or(false), + nf.on_behalf_of_email.and(Some(&authed.email)), + nf.ws_error_handler_muted.unwrap_or(false), + sqlx::types::Json(&nf.value) as _, + schema_str, + authed.username, + flow_path, + w_id, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}")) + })?; + + if is_new_path { + // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint + sqlx::query!( + "INSERT INTO flow + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) + SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at + FROM flow + WHERE path = $2 AND workspace_id = $3", + nf.path, + flow_path, + w_id + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Error updating flow due to create new flow: {e:#}")) + })?; + + sqlx::query!( + "UPDATE flow_version SET path = $1 WHERE path = $2 AND workspace_id = $3", + nf.path, + flow_path, + w_id + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating flow due to updating flow history path: {e:#}" + )) + })?; + + sqlx::query!( + "DELETE FROM flow WHERE path = $1 AND workspace_id = $2", + flow_path, + w_id + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating flow due to deleting old flow: {e:#}" + )) + })?; + + sqlx::query!( + "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS TRUE", + nf.path, + flow_path, + w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS TRUE", + nf.path, + flow_path, + w_id + ) + .execute(&mut *tx) + .await?; + } + + // tracing::error!("Updating flow: {:?}", nf.value.get()); + + // This will lock anyone who is trying to iterate on flow_versions with given path and parameters. + let version = sqlx::query_scalar!( + "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) RETURNING id", + w_id, + nf.path, + sqlx::types::Json(nf.value) as _, + schema_str, + &authed.username, + ) + .fetch_one(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating flow due to flow history insert: {e:#}" + )) + })?; + + // TODO: This should happen only after we are done with dependency job. + sqlx::query!( + "UPDATE flow SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3", + version, nf.path, w_id + ).execute(&mut *tx).await?; + + if is_new_path { + check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; + + if !authed.is_admin { + require_owner_of_path(&authed, flow_path)?; + } + } + + let mut schedulables: Vec = sqlx::query_as::<_, Schedule>( + "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *") + .bind(&nf.path) + .bind(&flow_path) + .bind(&w_id) + .fetch_all(&mut *tx) + .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to related schedules update: {e:#}")))?; + + let schedule = sqlx::query_as::<_, Schedule>( + "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *") + .bind(&nf.path) + .bind(&flow_path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to related schedule update: {e:#}")))?; + + if let Some(schedule) = schedule { + clear_schedule(&mut tx, &flow_path, &w_id).await?; + schedulables.push(schedule); + } + + for schedule in schedulables.into_iter() { + clear_schedule(&mut tx, &schedule.path, &w_id).await?; + + if schedule.enabled { + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; + } + } + + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'", + flow_path, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "flows.update", + ActionKind::Create, + &w_id, + Some(&nf.path.to_string()), + Some( + [Some(("flow", nf.path.as_str()))] + .into_iter() + .flatten() + .collect(), + ), + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateFlow { + workspace: w_id.clone(), + old_path: flow_path.to_owned(), + new_path: nf.path.clone(), + }, + ); + + let tx = PushIsolationLevel::Transaction(tx); + + let mut args: HashMap> = HashMap::new(); + if let Some(dm) = nf.deployment_message { + args.insert("deployment_message".to_string(), to_raw_value(&dm)); + } + args.insert("parent_path".to_string(), to_raw_value(&flow_path)); + + let (dependency_job_uuid, mut new_tx) = push( + &db, + tx, + &w_id, + JobPayload::FlowDependencies { + path: nf.path.clone(), + dedicated_worker: nf.dedicated_worker, + version, + debouncing_settings: Default::default(), + }, + windmill_queue::PushArgs { args: &args, extra: None }, + &authed.username, + &authed.email, + windmill_common::users::username_to_permissioned_as(&authed.username), + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + None, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + None, + ) + .await?; + + sqlx::query!( + "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3", + dependency_job_uuid, + nf.path, + w_id + ) + .execute(&mut *new_tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating flow due to updating dependency job field: {e:#}" + )) + })?; + + // Store the job_id in deployment_metadata for this flow deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + nf.path, + version, + dependency_job_uuid + ) + .execute(&mut *new_tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating deployment_metadata with job_id: {e:#}" + )) + })?; + + if let Some(old_dep_job) = old_dep_job { + sqlx::query!( + "UPDATE v2_job_queue SET + canceled_by = $2, + canceled_reason = 're-deployment' + WHERE id = $1", + old_dep_job, + &authed.username + ) + .execute(&mut *new_tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating flow due to cancelling dependency job: {e:#}" + )) + })?; + } + + new_tx.commit().await?; + + Ok(nf.path.to_string()) +} + +async fn list_tokens( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + list_tokens_internal(&db, &w_id, &path, true).await +} + +#[derive(Serialize)] +struct DeploymentStatus { + lock_error_logs: Option, + job_id: Option, +} +async fn get_deployment_status( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let mut tx = db.begin().await?; + let status_o = sqlx::query!( + "SELECT f.lock_error_logs, dm.job_id + FROM flow f + LEFT JOIN deployment_metadata dm ON f.versions[array_upper(f.versions, 1)] = dm.flow_version + AND f.workspace_id = dm.workspace_id AND f.path = dm.path + WHERE f.path = $1 AND f.workspace_id = $2", + path, + w_id, + ) + .fetch_optional(&mut *tx) + .await?; + + let status = not_found_if_none(status_o, "DeploymentStatus", path)?; + + let deployment_status = + DeploymentStatus { lock_error_logs: status.lock_error_logs, job_id: status.job_id }; + + tx.commit().await?; + Ok(Json(deployment_status)) +} + +async fn get_flow_by_path( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let flow_o = if query.with_starred_info.unwrap_or(false) { + sqlx::query_as::<_, FlowWithStarred>( + r#" + SELECT + flow.workspace_id, + flow.path, + flow.lock_error_logs, + flow.summary, + flow.description, + flow.archived, + flow.extra_perms, + flow.draft_only, + flow.dedicated_worker, + flow.tag, + flow.ws_error_handler_muted, + flow.timeout, + flow.visible_to_runner_only, + flow.on_behalf_of_email, + flow_version.id AS version_id, + flow_version.schema, + flow_version.value, + flow_version.created_at AS edited_at, + flow_version.created_by AS edited_by, + favorite.path IS NOT NULL AS starred + FROM flow + LEFT JOIN favorite + ON favorite.favorite_kind = 'flow' + AND favorite.workspace_id = flow.workspace_id + AND favorite.path = flow.path + AND favorite.usr = $3 + LEFT JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 AND flow.workspace_id = $2 + "#, + ) + .bind(path) + .bind(w_id) + .bind(&authed.username) + .fetch_optional(&mut *tx) + .await? + } else { + sqlx::query_as::<_, FlowWithStarred>( + r#" + SELECT + flow.workspace_id, + flow.path, + flow.lock_error_logs, + flow.summary, + flow.description, + flow.archived, + flow.extra_perms, + flow.draft_only, + flow.dedicated_worker, + flow.tag, + flow.ws_error_handler_muted, + flow.timeout, + flow.visible_to_runner_only, + flow.on_behalf_of_email, + flow_version.id AS version_id, + flow_version.schema, + flow_version.value, + flow_version.created_at AS edited_at, + flow_version.created_by AS edited_by, + NULL AS starred + FROM flow + LEFT JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 AND flow.workspace_id = $2 + "#, + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await? + }; + + tx.commit().await?; + + let flow = not_found_if_none(flow_o, "Flow", path)?; + Ok(Json(flow)) +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct FlowWDraft { + pub path: String, + pub summary: String, + pub description: String, + pub schema: Option, + pub value: sqlx::types::Json>, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dedicated_worker: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub visible_to_runner_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_behalf_of_email: Option, +} + +async fn get_flow_by_path_w_draft( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let flow_o = sqlx::query_as::<_, FlowWDraft>( + "SELECT + flow.path, + flow.summary, + flow.description, + flow_version.schema, + flow_version.value, + flow.extra_perms, + flow.draft_only, + flow.ws_error_handler_muted, + flow.dedicated_worker, + draft.value AS draft, + flow.tag, + flow.visible_to_runner_only, + flow.on_behalf_of_email + FROM flow + LEFT JOIN draft + ON flow.path = draft.path + AND draft.workspace_id = $2 + AND draft.typ = 'flow' + LEFT JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 + AND flow.workspace_id = $2", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + let flow = not_found_if_none(flow_o, "Flow", path)?; + Ok(Json(flow)) +} + +async fn exists_flow_by_path( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +#[derive(Deserialize)] +struct Archived { + archived: Option, +} + +async fn archive_flow_by_path( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(archived): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3", + archived.archived.unwrap_or(true), + path, + &w_id + ) + .execute(&mut *tx) + .await?; + + clear_static_asset_usage(&mut *tx, &w_id, path, AssetUsageKind::Flow).await?; + + audit_log( + &mut *tx, + &authed, + "flows.archive", + ActionKind::Delete, + &w_id, + Some(path), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + + ScopedDependencyMap::clear_map_for_item(path, &w_id, "flow", tx, &None) + .await + .commit() + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Flow { + path: path.to_string(), + parent_path: Some(path.to_string()), + version: 0, // dummy version as it will not get inserted in db + }, + Some(format!( + "Flow '{}' {}", + path, + if archived.archived.unwrap_or(true) { + "archived" + } else { + "unarchived" + } + )), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::ArchiveFlow { workspace: w_id, path: path.to_owned() }, + ); + + Ok(format!("Flow {path} archived")) +} + +/// Validates that flow debouncing configuration is supported by all workers +/// Returns an error if debouncing is configured but workers are behind required version +async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> { + if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await + && !nf.parse_flow_value()?.debouncing_settings.is_default() + { + tracing::warn!( + "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" + ); + Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() }) + } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await + && !nf + .parse_flow_value()? + .debouncing_settings + .is_legacy_compatible() + && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT + { + tracing::warn!( + "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" + ); + Err(Error::WorkersAreBehind { + feature: "V2 Debouncing".into(), + min_version: "1.597.0".into(), + }) + } else { + Ok(()) + } +} + +#[derive(Deserialize)] +struct DeleteFlowQuery { + keep_captures: Option, +} + +async fn delete_flow_by_path( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("flows:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'", + path, + &w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM flow WHERE path = $1 AND workspace_id = $2", + path, + &w_id + ) + .execute(&mut *tx) + .await?; + + if !query.keep_captures.unwrap_or(false) { + sqlx::query!( + "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", + path, + &w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", + path, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "flows.delete", + ActionKind::Delete, + &w_id, + Some(path), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Flow { + path: path.to_string(), + parent_path: Some(path.to_string()), + version: 0, // dummy version as it will not get inserted in db + }, + Some(format!("Flow '{}' deleted", path)), + true, + None, + ) + .await?; + + sqlx::query!( + "DELETE FROM deployment_metadata WHERE path = $1 AND workspace_id = $2 AND script_hash IS NULL and app_version IS NULL", + path, + w_id + ) + .execute(&db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" + )) + })?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteFlow { workspace: w_id, path: path.to_owned() }, + ); + + Ok(format!("Flow {path} deleted")) +} + +#[cfg(test)] +mod tests { + + use std::{collections::HashMap, time::Duration}; + + use windmill_common::{ + flows::{ + ConstantDelay, ExponentialDelay, FlowModule, FlowModuleValue, FlowValue, + InputTransform, Retry, StopAfterIf, + }, + runnable_settings::{ + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, + }, + scripts, + }; + + const SECOND: Duration = Duration::from_secs(1); + + #[test] + fn flowmodule_serde() { + let fv = FlowValue { + modules: vec![ + FlowModule { + id: "a".to_string(), + value: windmill_common::worker::to_raw_value(&FlowModuleValue::Script { + path: "test".to_string(), + input_transforms: [( + "test".to_string(), + InputTransform::Static { + value: windmill_common::worker::to_raw_value(&"test2".to_string()), + }, + )] + .into(), + hash: None, + tag_override: None, + is_trigger: None, + pass_flow_input_directly: None, + }), + stop_after_if: None, + stop_after_all_iters_if: None, + summary: None, + suspend: Default::default(), + retry: None, + sleep: None, + cache_ttl: None, + cache_ignore_s3_path: None, + mock: None, + timeout: None, + priority: None, + delete_after_use: None, + continue_on_error: None, + skip_if: None, + apply_preprocessor: None, + pass_flow_input_directly: None, + }, + FlowModule { + id: "b".to_string(), + value: windmill_common::worker::to_raw_value(&FlowModuleValue::RawScript { + input_transforms: HashMap::new(), + content: "test".to_string(), + language: scripts::ScriptLang::Deno, + path: None, + lock: None, + tag: None, + is_trigger: None, + assets: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + }), + stop_after_if: Some(StopAfterIf { + expr: "foo = 'bar'".to_string(), + ..Default::default() + }), + stop_after_all_iters_if: None, + summary: None, + suspend: Default::default(), + retry: None, + sleep: None, + cache_ttl: None, + cache_ignore_s3_path: None, + mock: None, + timeout: None, + priority: None, + delete_after_use: None, + continue_on_error: None, + skip_if: None, + apply_preprocessor: None, + pass_flow_input_directly: None, + }, + FlowModule { + id: "c".to_string(), + value: windmill_common::worker::to_raw_value(&FlowModuleValue::ForloopFlow { + iterator: InputTransform::Static { + value: windmill_common::worker::to_raw_value(&[1, 2, 3]), + }, + modules: vec![], + modules_node: None, + skip_failures: true, + parallel: false, + parallelism: None, + squash: None, + }), + stop_after_if: Some(StopAfterIf { + expr: "previous.isEmpty()".to_string(), + ..Default::default() + }), + stop_after_all_iters_if: None, + summary: None, + suspend: Default::default(), + retry: None, + sleep: None, + cache_ttl: None, + cache_ignore_s3_path: None, + mock: None, + timeout: None, + priority: None, + delete_after_use: None, + continue_on_error: None, + skip_if: None, + apply_preprocessor: None, + pass_flow_input_directly: None, + }, + ], + failure_module: Some(Box::new(FlowModule { + id: "d".to_string(), + value: FlowModuleValue::Script { + path: "test".to_string(), + input_transforms: HashMap::new(), + hash: None, + tag_override: None, + is_trigger: None, + pass_flow_input_directly: None, + } + .into(), + stop_after_if: Some(StopAfterIf { + expr: "previous.isEmpty()".to_string(), + ..Default::default() + }), + stop_after_all_iters_if: None, + summary: None, + suspend: Default::default(), + retry: None, + sleep: None, + cache_ttl: None, + cache_ignore_s3_path: None, + mock: None, + timeout: None, + priority: None, + delete_after_use: None, + continue_on_error: None, + skip_if: None, + apply_preprocessor: None, + pass_flow_input_directly: None, + })), + preprocessor_module: None, + same_worker: false, + skip_expr: None, + cache_ttl: None, + cache_ignore_s3_path: None, + priority: None, + early_return: None, + chat_input_enabled: None, + flow_env: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), + }; + let expect = serde_json::json!({ + "modules": [ + { + "id": "a", + "value": { + "input_transforms": { + "test": { + "type": "static", + "value": "test2" + } + }, + "type": "script", + "path": "test", + }, + }, + { + "id": "b", + "value": { + "input_transforms": {}, + "type": "rawscript", + "content": "test", + "language": "deno" + }, + "stop_after_if": { + "expr": "foo = 'bar'", + "skip_if_stopped": false, + "error_message": null + } + }, + { + "id": "c", + "value": { + "type": "forloopflow", + "iterator": { + "type": "static", + "value": [ + 1, + 2, + 3 + ] + }, + "parallel": false, + "skip_failures": true, + "modules": [] + }, + "stop_after_if": { + "expr": "previous.isEmpty()", + "skip_if_stopped": false, + "error_message": null + } + } + ], + "failure_module": { + "id": "d", + "value": { + "input_transforms": {}, + "type": "script", + "path": "test", + }, + "stop_after_if": { + "expr": "previous.isEmpty()", + "skip_if_stopped": false, + "error_message": null + } + }, + }); + assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect)); + } + + #[test] + fn retry_serde() { + assert_eq!(Retry::default(), serde_json::from_str(r#"{}"#).unwrap()); + + assert_eq!( + Retry::default(), + serde_json::from_str( + r#" + { + "constant": { + "seconds": 0 + }, + "exponential": { + "multiplier": 1, + "seconds": 0 + }, + "retry_if": null + } + "# + ) + .unwrap() + ); + + assert_eq!( + Retry { + constant: Default::default(), + exponential: ExponentialDelay { + attempts: 0, + multiplier: 1, + seconds: 123, + random_factor: None + }, + retry_if: None + }, + serde_json::from_str( + r#" + { + "constant": {}, + "exponential": { "seconds": 123 }, + "retry_if" : null + } + "# + ) + .unwrap() + ); + } + + #[test] + fn retry_exponential() { + let retry = Retry { + constant: ConstantDelay::default(), + exponential: ExponentialDelay { + attempts: 3, + multiplier: 4, + seconds: 3, + random_factor: None, + }, + retry_if: None, + }; + assert_eq!( + vec![ + Some(12 * SECOND), + Some(36 * SECOND), + Some(108 * SECOND), + None + ], + (0..4) + .map(|previous_attempts| retry.interval(previous_attempts, false)) + .collect::>() + ); + + assert_eq!(Some(108 * SECOND), retry.max_interval()); + } + + #[test] + fn retry_both() { + let retry = Retry { + constant: ConstantDelay { attempts: 2, seconds: 4 }, + exponential: ExponentialDelay { + attempts: 2, + multiplier: 1, + seconds: 3, + random_factor: None, + }, + retry_if: None, + }; + assert_eq!( + vec![ + Some(4 * SECOND), + Some(4 * SECOND), + Some(27 * SECOND), + Some(81 * SECOND), + None, + ], + (0..5) + .map(|previous_attempts| retry.interval(previous_attempts, false)) + .collect::>() + ); + + assert_eq!(Some(81 * SECOND), retry.max_interval()); + } +} diff --git a/backend/windmill-api-scripts/src/lib.rs b/backend/windmill-api-scripts/src/lib.rs new file mode 100644 index 0000000000..aab68a4d52 --- /dev/null +++ b/backend/windmill-api-scripts/src/lib.rs @@ -0,0 +1,10 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod flows; +pub mod scripts; diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs new file mode 100644 index 0000000000..f73ea4c04b --- /dev/null +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -0,0 +1,2318 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::extract::Multipart; +use windmill_api_auth::{ + auth::{list_tokens_internal, AuthCache, TruncatedTokenWithEmail}, + check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed, +}; +use windmill_common::{ + utils::{BulkDeleteRequest, WithStarredInfoQuery, HTTP_CLIENT}, + webhook::{WebhookMessage, WebhookShared}, + DB, +}; +use windmill_queue::schedule::clear_schedule; + +use axum::{ + extract::{Extension, Path, Query}, + response::IntoResponse, + routing::{delete, get, post}, + Json, Router, +}; +use futures::future::try_join_all; +use http::header; +use hyper::StatusCode; +use itertools::Itertools; +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use serde_json::value::RawValue; +use sql_builder::prelude::*; +use sqlx::{FromRow, Postgres, Transaction}; +use std::{collections::HashMap, sync::Arc}; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_dep_map::process_relative_imports; +use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; + +use windmill_common::{ + assets::{ + clear_static_asset_usage, clear_static_asset_usage_by_script_hash, + insert_static_asset_usage, AssetUsageKind, AssetWithAltAccessType, + }, + error::{self, to_anyhow}, + min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2}, + runnable_settings::{ + min_version_supports_runnable_settings_v0, RunnableSettings, RunnableSettingsTrait, + }, + s3_helpers::upload_artifact_to_store, + scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, + utils::{paginate_without_limits, WarnAfterExt}, + worker::CLOUD_HOSTED, +}; + +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + jobs::JobPayload, + schedule::Schedule, + schema::should_validate_schema, + scripts::{ + to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash, + ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred, + }, + users::username_to_permissioned_as, + utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath}, + worker::to_raw_value, + HUB_BASE_URL, +}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_parser_ts::remove_pinned_imports; +use windmill_queue::{ + schedule::push_scheduled_job, PushIsolationLevel, WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT, +}; + +const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20; + +#[derive(Serialize, sqlx::FromRow)] +pub struct ScriptWDraft { + pub hash: ScriptHash, + pub path: String, + pub summary: String, + pub description: String, + pub content: String, + pub language: ScriptLang, + pub kind: ScriptKind, + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft: Option>>, + pub schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub envs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dedicated_worker: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub restart_unless_cancelled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub visible_to_runner_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_main_func: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, + #[serde(flatten)] + #[sqlx(flatten)] + pub runnable_settings: SR, +} + +impl ScriptWDraft { + pub async fn prefetch_cached<'a>( + self, + db: &DB, + ) -> error::Result> { + let (debouncing_settings, concurrency_settings) = + windmill_common::runnable_settings::prefetch_cached_from_handle( + self.runnable_settings.runnable_settings_handle, + db, + ) + .await?; + + Ok(ScriptWDraft { + runnable_settings: ScriptRunnableSettingsInline { + concurrency_settings: concurrency_settings.maybe_fallback( + self.runnable_settings.concurrency_key, + self.runnable_settings.concurrent_limit, + self.runnable_settings.concurrency_time_window_s, + ), + debouncing_settings: debouncing_settings.maybe_fallback( + self.runnable_settings.debounce_key, + self.runnable_settings.debounce_delay_s, + ), + }, + hash: self.hash, + path: self.path, + summary: self.summary, + description: self.description, + content: self.content, + language: self.language, + kind: self.kind, + tag: self.tag, + draft: self.draft, + schema: self.schema, + draft_only: self.draft_only, + envs: self.envs, + cache_ttl: self.cache_ttl, + cache_ignore_s3_path: self.cache_ignore_s3_path, + dedicated_worker: self.dedicated_worker, + ws_error_handler_muted: self.ws_error_handler_muted, + priority: self.priority, + restart_unless_cancelled: self.restart_unless_cancelled, + delete_after_use: self.delete_after_use, + timeout: self.timeout, + visible_to_runner_only: self.visible_to_runner_only, + no_main_func: self.no_main_func, + has_preprocessor: self.has_preprocessor, + on_behalf_of_email: self.on_behalf_of_email, + assets: self.assets, + }) + } +} + +pub fn global_service() -> Router { + Router::new() + .route("/hub/top", get(get_top_hub_scripts)) + .route("/hub/get/*path", get(get_hub_script_by_path)) + .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) + .route("/hub/pick/*path", get(pick_hub_script_by_path)) +} + +pub fn global_unauthed_service() -> Router { + Router::new() + .route( + "/tokened_raw/:workspace/:token/*path", + get(get_tokened_raw_script_by_path), + ) + .route("/empty_ts/*path", get(get_empty_ts_script_by_path)) +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_scripts)) + .route("/list_search", get(list_search_scripts)) + .route("/create", post(create_script)) + .route("/create_snapshot", post(create_snapshot_script)) + .route("/archive/p/*path", post(archive_script_by_path)) + .route("/get/draft/*path", get(get_script_by_path_w_draft)) + .route("/get/p/*path", get(get_script_by_path)) + .route("/list_tokens/*path", get(list_tokens)) + .route("/raw/p/*path", get(raw_script_by_path)) + .route("/raw_unpinned/p/*path", get(raw_script_by_path_unpinned)) + .route("/exists/p/*path", get(exists_script_by_path)) + .route("/archive/h/:hash", post(archive_script_by_hash)) + .route("/delete/h/:hash", post(delete_script_by_hash)) + .route("/delete/p/*path", post(delete_script_by_path)) + .route("/delete_bulk", delete(delete_scripts_bulk)) + .route("/get/h/:hash", get(get_script_by_hash)) + .route("/raw/h/:hash", get(raw_script_by_hash)) + .route("/deployment_status/h/:hash", get(get_deployment_status)) + .route("/list_paths", get(list_paths)) + .route( + "/toggle_workspace_error_handler/p/*path", + post(toggle_workspace_error_handler), + ) + .route("/history/p/*path", get(get_script_history)) + .route("/get_latest_version/*path", get(get_latest_version)) + .route( + "/list_paths_from_workspace_runnable/*path", + get(list_paths_from_workspace_runnable), + ) + .route( + "/history_update/h/:hash/p/*path", + post(update_script_history), + ) +} + +#[derive(Serialize, FromRow)] +pub struct SearchScript { + path: String, + content: String, +} +async fn list_search_scripts( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + #[cfg(feature = "enterprise")] + let n = 10000; + + #[cfg(not(feature = "enterprise"))] + let n = 10; + + let rows = sqlx::query_as!( + SearchScript, + "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2", + &w_id, + n + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_scripts( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, + Query(lq): Query, +) -> JsonResult> { + let (per_page, offset) = paginate_without_limits(pagination); + let mut sqlb = SqlBuilder::select_from("script as o") + .fields(&[ + "hash", + "o.path", + "summary", + "COALESCE(draft.created_at, o.created_at) as created_at", + "archived", + "extra_perms", + if !lq.without_description.unwrap_or(false) { + "description" + } else { + "NULL as description" + }, + "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors", + "language", + "favorite.path IS NOT NULL as starred", + "tag", + "draft.path IS NOT NULL as has_draft", + "draft_only", + "ws_error_handler_muted", + "no_main_func", + "codebase IS NOT NULL as use_codebase", + "kind" + ]) + .left() + .join("favorite") + .on( + "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?" + .bind(&authed.username), + ) + .left() + .join("draft") + .on( + "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'" + ) + .order_desc("favorite.path IS NOT NULL") + .order_by("created_at", lq.order_desc.unwrap_or(true)) + .and_where("o.workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + + let lowercased_kinds: Option> = lq + .kinds + .map(|x| x.split(",").map(&str::to_lowercase).collect()); + + if (!lq.include_without_main.unwrap_or(false) + && lowercased_kinds + .as_ref() + .map(|x| !x.contains(&"preprocessor".to_string())) + .unwrap_or(true)) + || authed.is_operator + { + // only include scripts that have a main function + // do not hide scripts without main if preprocessor is in the kinds + sqlb.and_where("o.no_main_func IS NOT TRUE"); + } + + if !lq.include_draft_only.unwrap_or(false) || authed.is_operator { + sqlb.and_where("draft_only IS NOT TRUE"); + } + + if lq.show_archived.unwrap_or(false) { + sqlb.and_where_eq( + "o.ctid", + "(SELECT ctid FROM script + WHERE path = o.path + AND workspace_id = ? + ORDER BY created_at DESC + LIMIT 1)" + .bind(&w_id), + ); + sqlb.and_where_eq("archived", true); + } else { + sqlb.and_where_eq("archived", false); + } + if let Some(ps) = &lq.path_start { + sqlb.and_where_like_left("o.path", ps); + } + if let Some(p) = &lq.path_exact { + sqlb.and_where_eq("o.path", "?".bind(p)); + } + if let Some(cb) = &lq.created_by { + sqlb.and_where_eq("created_by", "?".bind(cb)); + } + if let Some(ph) = &lq.first_parent_hash { + sqlb.and_where_eq("parent_hashes[1]", &ph.0); + } + if let Some(ph) = &lq.last_parent_hash { + sqlb.and_where_eq("parent_hashes[array_upper(parent_hashes, 1)]", &ph.0); + } + if let Some(ph) = &lq.parent_hash { + sqlb.and_where_eq("any(parent_hashes)", &ph.0); + } + if let Some(it) = &lq.is_template { + sqlb.and_where_eq("is_template", it); + } + if let Some(dw) = &lq.dedicated_worker { + sqlb.and_where_eq("dedicated_worker", dw); + } + if authed.is_operator { + sqlb.and_where_eq("kind", quote("script")); + } else if let Some(lowercased_kinds) = lowercased_kinds { + let safe_kinds = lowercased_kinds + .into_iter() + .map(sql_builder::quote) + .collect_vec(); + if safe_kinds.len() > 0 { + sqlb.and_where_in("kind", safe_kinds.as_slice()); + } + } + if lq.starred_only.unwrap_or(false) { + sqlb.and_where_is_not_null("favorite.path"); + } + + if lq.with_deployment_msg.unwrap_or(false) { + sqlb.join("deployment_metadata dm") + .left() + .on("dm.script_hash = o.hash") + .fields(&["dm.deployment_msg"]); + } + + if let Some(languages) = lq.languages { + sqlb.and_where_in( + "language", + &languages + .iter() + .map(|language| quote(language.as_str())) + .collect_vec(), + ); + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as::<_, ListableScript>(&sql) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct TopHubScriptsQuery { + limit: Option, + app: Option, + kind: Option, +} + +async fn get_top_hub_scripts( + Query(query): Query, + Extension(db): Extension, +) -> impl IntoResponse { + let mut query_params = vec![]; + if let Some(query_limit) = query.limit { + query_params.push(("limit", query_limit.to_string().clone())); + } + if let Some(query_app) = query.app { + query_params.push(("app", query_app.to_string().clone())); + } + if let Some(query_kind) = query.kind { + query_params.push(("kind", query_kind.to_string().clone())); + } + + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/top", *HUB_BASE_URL.read().await), + Some(query_params), + &db, + ) + .await?; + Ok::<_, Error>((status_code, headers, response)) +} + +async fn create_snapshot_script( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(webhook): Extension, + Extension(db): Extension, + Path(w_id): Path, + mut multipart: Multipart, +) -> Result<(StatusCode, String)> { + // TODO: Check for debouncing here as well. + let mut script_hash = None; + let mut tx = None; + let mut uploaded = false; + let mut handle_deployment_metadata = None; + while let Some(field) = multipart.next_field().await.unwrap() { + let name = field.name().unwrap().to_string(); + let data = field.bytes().await.unwrap(); + if name == "script" { + let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap(); + let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar")); + let use_esm = ns.codebase.as_ref().is_some_and(|x| x.contains(".esm")); + let (new_hash, ntx, hdm) = create_script_internal( + ns, + w_id.clone(), + authed.clone(), + db.clone(), + user_db.clone(), + webhook.clone(), + ) + .await?; + let mut nh = new_hash.to_string(); + if use_esm { + nh = format!("{nh}.esm"); + } + if is_tar { + nh = format!("{nh}.tar"); + } + script_hash = Some(nh); + tx = Some(ntx); + handle_deployment_metadata = hdm; + } + if name == "file" { + let hash = script_hash.as_ref().ok_or_else(|| { + Error::BadRequest( + "script need to be passed first in the multipart upload".to_string(), + ) + })?; + + uploaded = true; + + let path = windmill_common::s3_helpers::bundle(&w_id, &hash); + upload_artifact_to_store( + &path, + data, + &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, + ) + .await?; + } + // println!("Length of `{}` is {} bytes", name, data.len()); + } + if !uploaded { + return Err(Error::BadRequest("No file uploaded".to_string())); + } + if script_hash.is_none() { + return Err(Error::BadRequest( + "No script found in the uploaded file".to_string(), + )); + } + + tx.unwrap().commit().await?; + if let Some(hdm) = handle_deployment_metadata { + hdm.handle(&db).await?; + } + return Ok((StatusCode::CREATED, format!("{}", script_hash.unwrap()))); +} + +async fn list_paths_from_workspace_runnable( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let runnables = sqlx::query_scalar!( + r#"SELECT importer_path FROM dependency_map + WHERE workspace_id = $1 AND imported_path = $2"#, + w_id, + path.to_path(), + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(runnables)) +} + +async fn create_script( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(webhook): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(ns): Json, +) -> Result<(StatusCode, String)> { + let (hash, tx, hdm) = + create_script_internal(ns, w_id, authed, db.clone(), user_db, webhook).await?; + tx.commit().await?; + if let Some(hdm) = hdm { + hdm.handle(&db).await?; + } + Ok((StatusCode::CREATED, format!("{}", hash))) +} + +struct HandleDeploymentMetadata { + email: String, + created_by: String, + w_id: String, + obj: DeployedObject, + deployment_message: Option, + renamed_from: Option, +} + +impl HandleDeploymentMetadata { + async fn handle(self, db: &DB) -> Result<()> { + handle_deployment_metadata( + &self.email, + &self.created_by, + &db, + &self.w_id, + self.obj, + self.deployment_message, + false, + self.renamed_from.as_deref(), + ) + .await + } +} + +async fn create_script_internal<'c>( + ns: NewScript, + w_id: String, + authed: ApiAuthed, + db: sqlx::Pool, + user_db: UserDB, + webhook: WebhookShared, +) -> Result<( + ScriptHash, + Transaction<'c, Postgres>, + Option, +)> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot create scripts for security reasons".to_string(), + )); + } + check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; + + guard_script_from_debounce_data(&ns).await?; + + let codebase = ns.codebase.as_ref(); + #[cfg(not(feature = "enterprise"))] + if ns.ws_error_handler_muted.is_some_and(|val| val) { + return Err(Error::BadRequest( + "Muting the error handler for certain script is only available in enterprise version" + .to_string(), + )); + } + if *CLOUD_HOSTED { + let nb_scripts = + sqlx::query_scalar!("SELECT COUNT(*) FROM script WHERE workspace_id = $1", &w_id) + .fetch_one(&db) + .await?; + if nb_scripts.unwrap_or(0) >= 5000 { + return Err(Error::BadRequest( + "You have reached the maximum number of scripts (5000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + + if ns.summary.len() > 300 { + return Err(Error::BadRequest( + "Summary must be less than 300 characters on cloud".to_string(), + )); + } + if ns.description.len() > 3000 { + return Err(Error::BadRequest( + "Description must be less than 3000 characters on cloud".to_string(), + )); + } + } + let script_path = ns.path.clone(); + let hash = ScriptHash(hash_script(&ns)); + let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await; + let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + if sqlx::query_scalar!( + "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", + hash.0, + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .is_some() + { + return Err(Error::BadRequest( + "A script with same hash (hence same path, description, summary, content) already \ + exists!" + .to_owned(), + )); + }; + let clashing_script = sqlx::query_as::<_, Script>( + "SELECT * FROM script WHERE path = $1 AND archived = false AND workspace_id = $2", + ) + .bind(&ns.path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + struct ParentInfo { + p_hashes: Vec, + perms: serde_json::Value, + p_path: String, + } + let parent_hashes_and_perms: Option = match (&ns.parent_hash, clashing_script) { + (None, None) => Ok(None), + (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!( + "Path conflict for {} with non-archived hash {}", + &ns.path, &s.hash + ))), + (None, Some(s)) => { + sqlx::query!( + "DELETE FROM script WHERE hash = $1 AND workspace_id = $2", + s.hash.0, + &w_id + ) + .execute(&mut *tx) + .await?; + Ok(None) + } + (Some(p_hash), o) => { + // Lock the parent row to prevent concurrent updates with the same parent_hash + // This ensures linear lineage - only one script can have a given parent at a time + if sqlx::query_scalar!( + "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2 FOR UPDATE", + p_hash.0, + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .is_none() + { + return Err(Error::BadRequest( + "The parent hash does not seem to exist".to_owned(), + )); + }; + + let clashing_hash_o = sqlx::query_scalar!( + "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", + p_hash.0, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(clashing_hash) = clashing_hash_o { + return Err(Error::BadRequest(format!( + "A script with hash {} with same parent_hash has been found. However, the \ + lineage must be linear: no 2 scripts can have the same parent", + ScriptHash(clashing_hash) + ))); + }; + + let ScriptWithStarred { script: ps, .. } = + get_script_by_hash_internal(&mut tx, &w_id, p_hash, None).await?; + + if ps.path != ns.path { + require_owner_of_path(&authed, &ps.path)?; + } + + let ph = { + let v = ps.parent_hashes.map(|x| x.0).unwrap_or_default(); + let mut v: Vec = v + .into_iter() + .take(MAX_HASH_HISTORY_LENGTH_STORED - 1) + .collect(); + v.insert(0, p_hash.0); + v + }; + let r: Result> = match o { + Some(clashing_script) + if clashing_script.path == ns.path && clashing_script.hash.0 != p_hash.0 => + { + Err(Error::BadRequest(format!( + "Path conflict for {} with non-archived hash {}", + &ns.path, &clashing_script.hash + ))) + } + Some(_) | None => Ok(Some(ParentInfo { + p_hashes: ph, + perms: ps.extra_perms, + p_path: ps.path, + })), + }; + sqlx::query!( + "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2", + p_hash.0, + &w_id + ) + .execute(&mut *tx) + .await?; + + clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + + r + } + }?; + let p_hashes = parent_hashes_and_perms.as_ref().map(|v| &v.p_hashes[..]); + let extra_perms = parent_hashes_and_perms + .as_ref() + .map(|v| v.perms.clone()) + .unwrap_or(json!({})); + let lock = if ns.codebase.is_some() { + Some(String::new()) + } else if !( + ns.language == ScriptLang::Python3 + || ns.language == ScriptLang::Go + || ns.language == ScriptLang::Bun + || ns.language == ScriptLang::Bunnative + || ns.language == ScriptLang::Deno + || ns.language == ScriptLang::Rust + || ns.language == ScriptLang::Ansible + || ns.language == ScriptLang::CSharp + || ns.language == ScriptLang::Nu + || ns.language == ScriptLang::Php + || ns.language == ScriptLang::Java + || ns.language == ScriptLang::Ruby + // for related places search: ADD_NEW_LANG + ) { + Some(String::new()) + } else { + ns.lock.as_ref().and_then(|e| { + if e.is_empty() { + None + } else { + Some(e.to_string()) + } + }) + }; + + let needs_lock_gen = lock.is_none() && codebase.is_none(); + let envs = ns.envs.as_ref().map(|x| x.as_slice()); + let envs = if ns.envs.is_none() || ns.envs.as_ref().unwrap().is_empty() { + None + } else { + envs + }; + + let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&ns.content); + if anns.native { + ScriptLang::Bunnative + } else { + ScriptLang::Bun + } + } else { + ns.language.clone() + }; + + let validate_schema = should_validate_schema(&ns.content, &ns.language); + + let (no_main_func, has_preprocessor) = if matches!(ns.kind, Some(ScriptKind::Preprocessor)) { + (ns.no_main_func, ns.has_preprocessor) + } else { + match lang { + ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { + let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None); + match args { + Ok(args) => (args.no_main_func, args.has_preprocessor), + Err(e) => { + tracing::warn!( + "Error parsing deno signature when deploying script {}: {:?}", + ns.path, + e + ); + (None, None) + } + } + } + #[cfg(feature = "python")] + ScriptLang::Python3 => { + let args = windmill_parser_py::parse_python_signature(&ns.content, None, true); + match args { + Ok(args) => (args.no_main_func, args.has_preprocessor), + Err(e) => { + tracing::warn!( + "Error parsing python signature when deploying script {}: {:?}", + ns.path, + e + ); + (None, None) + } + } + } + _ => (ns.no_main_func, ns.has_preprocessor), + } + }; + + let runnable_settings_handle = windmill_common::runnable_settings::insert_rs(RunnableSettings { + debouncing_settings: ns.debouncing_settings.insert_cached(&db).await?, + concurrency_settings: ns.concurrency_settings.insert_cached(&db).await?, + }, &db) + .await?; + + let ( + guarded_concurrent_limit, + guarded_concurrency_time_window_s, + guarded_concurrency_key, + guarded_debounce_key, + guarded_debounce_delay_s, + ) = if min_version_supports_runnable_settings_v0().await { + Default::default() + } else { + ( + ns.concurrency_settings.concurrent_limit.clone(), + ns.concurrency_settings.concurrency_time_window_s.clone(), + ns.concurrency_settings.concurrency_key.clone(), + ns.debouncing_settings.debounce_key.clone(), + ns.debouncing_settings.debounce_delay_s.clone(), + ) + }; + + sqlx::query!( + "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ + content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38)", + &w_id, + &hash.0, + ns.path, + p_hashes, + ns.summary, + ns.description, + &ns.content, + &authed.username, + ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()), + ns.is_template.unwrap_or(false), + extra_perms, + lock, + lang as ScriptLang, + ns.kind.unwrap_or(ScriptKind::Script) as ScriptKind, + ns.tag, + ns.draft_only, + envs, + guarded_concurrent_limit, + guarded_concurrency_time_window_s, + ns.cache_ttl, + ns.dedicated_worker, + ns.ws_error_handler_muted.unwrap_or(false), + ns.priority, + ns.restart_unless_cancelled, + ns.delete_after_use, + ns.timeout, + guarded_concurrency_key, + ns.visible_to_runner_only, + no_main_func.filter(|x: &bool| *x), // should be Some(true) or None + codebase, + has_preprocessor.filter(|x: &bool| *x), // should be Some(true) or None + if ns.on_behalf_of_email.is_some() { + Some(&authed.email) + } else { + None + }, + validate_schema, + ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()), + guarded_debounce_key, + guarded_debounce_delay_s, + ns.cache_ignore_s3_path, + runnable_settings_handle + ) + .execute(&mut *tx) + .await?; + + let p_path_opt = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()); + if let Some(ref p_path) = p_path_opt { + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", + p_path, + &w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", + ns.path, + p_path, + w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", + ns.path, + p_path, + w_id + ) + .execute(&mut *tx) + .await?; + + let mut schedulables = sqlx::query_as::<_, Schedule>( + "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS false RETURNING *") + .bind(&ns.path) + .bind(&p_path) + .bind(&w_id) + .fetch_all(&mut *tx) + .await?; + + let schedule = sqlx::query_as::<_, Schedule>( + "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *") + .bind(&ns.path) + .bind(&p_path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + if let Some(schedule) = schedule { + schedulables.push(schedule); + } + + // Update dynamic_skip references when script is renamed + sqlx::query!( + "UPDATE schedule SET dynamic_skip = $1 WHERE dynamic_skip = $2 AND workspace_id = $3", + &ns.path, + &p_path, + &w_id + ) + .execute(&mut *tx) + .await?; + + for schedule in schedulables { + clear_schedule(&mut tx, &schedule.path, &w_id).await?; + + if schedule.enabled { + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; + } + } + } else { + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", + ns.path, + &w_id + ) + .execute(&mut *tx) + .await?; + } + if p_hashes.is_some() && !p_hashes.unwrap().is_empty() { + audit_log( + &mut *tx, + &authed, + "scripts.update", + ActionKind::Update, + &w_id, + Some(&ns.path), + Some([("hash", hash.to_string().as_str())].into()), + ) + .await?; + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateScript { + workspace: w_id.clone(), + path: ns.path.clone(), + hash: hash.to_string(), + }, + ); + } else { + audit_log( + &mut *tx, + &authed, + "scripts.create", + ActionKind::Create, + &w_id, + Some(&ns.path), + Some( + [ + ("workspace", w_id.as_str()), + ("hash", hash.to_string().as_str()), + ] + .into(), + ), + ) + .await?; + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateScript { + workspace: w_id.clone(), + path: ns.path.clone(), + hash: hash.to_string(), + }, + ); + } + + clear_static_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; + for asset in ns.assets.as_ref().into_iter().flatten() { + insert_static_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script) + .await?; + } + + let permissioned_as = username_to_permissioned_as(&authed.username); + if let Some(parent_hash) = ns.parent_hash { + tracing::info!( + "creating script {hash:?} at path {script_path} with parent {parent_hash} on workspace {w_id}", + ); + } else { + tracing::info!("creating script {hash:?} at path {script_path} on workspace {w_id}",); + } + if needs_lock_gen { + let tag = if ns.dedicated_worker.is_some_and(|x| x) { + Some(format!("{}:{}", &w_id, &ns.path,)) + } else if ns.tag.as_ref().is_some_and(|x| x.contains("$args[")) { + None + } else { + ns.tag + }; + + let mut args: HashMap> = HashMap::new(); + if let Some(dm) = ns.deployment_message { + args.insert("deployment_message".to_string(), to_raw_value(&dm)); + } + if let Some(ref p_path) = p_path_opt { + args.insert("parent_path".to_string(), to_raw_value(&p_path)); + } + + let tx = PushIsolationLevel::Transaction(tx); + let (job_id, mut new_tx) = windmill_queue::push( + &db, + tx, + &w_id, + JobPayload::Dependencies { + hash, + language: ns.language, + path: ns.path.clone(), + dedicated_worker: ns.dedicated_worker, + debouncing_settings: Default::default(), + }, + windmill_queue::PushArgs::from(&args), + &authed.username, + &authed.email, + permissioned_as, + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + tag, + None, + None, + None, + Some(&authed.clone().into()), + false, + None, + None, + None, + ) + .await?; + + // Store the job_id in deployment_metadata for this script deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, script_hash, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + ns.path, + hash.0, + job_id + ) + .execute(&mut *new_tx) + .await?; + + Ok((hash, new_tx, None)) + } else { + if codebase.is_none() { + let db2 = db.clone(); + let w_id2 = w_id.clone(); + let authed2 = authed.clone(); + let permissioned_as2 = permissioned_as.clone(); + let script_path2 = script_path.clone(); + let parent_path = p_path_opt.clone(); + let deployment_message = ns.deployment_message.clone(); + let content = ns.content.clone(); + let language = ns.language.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + if let Err(e) = process_relative_imports( + &db2, + None, + None, + &w_id2, + &script_path2, + parent_path, + deployment_message, + &content, + &Some(language), + &authed2.email, + &authed2.username, + &permissioned_as2, + ) + .await + { + tracing::error!(%e, "error processing relative imports"); + } + }); + } + + // handle_deployment_metadata( + // &authed.email, + // &authed.username, + // &db, + // &w_id, + // DeployedObject::Script { + // hash: hash.clone(), + // path: script_path.clone(), + // parent_path: p_path_opt, + // }, + // ns.deployment_message, + // false, + // ) + // .await?; + + Ok(( + hash, + tx, + Some(HandleDeploymentMetadata { + email: authed.email, + created_by: authed.username, + w_id, + obj: DeployedObject::Script { + hash: hash.clone(), + path: script_path.clone(), + parent_path: p_path_opt.clone(), + }, + deployment_message: ns.deployment_message, + renamed_from: p_path_opt, + }), + )) + } +} + +pub async fn get_hub_script_by_path( + Path(path): Path, + Extension(db): Extension, +) -> Result { + windmill_common::scripts::get_hub_script_by_path(path, &HTTP_CLIENT, &db).await +} + +pub async fn get_full_hub_script_by_path( + Path(path): Path, + Extension(db): Extension, +) -> JsonResult { + Ok(Json( + windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, Some(&db)) + .await?, + )) +} + +pub async fn pick_hub_script_by_path( + Path(path): Path, + Extension(db): Extension, +) -> impl IntoResponse { + let path_str = path.to_path(); + + // Extract version_id from path (format: {hub}/{version_id}/{summary}) + let version_id = path_str.split('/').nth(1).unwrap_or(""); + + let hub_base_url = HUB_BASE_URL.read().await.clone(); + + // Determine which hub to use based on version_id + // If version_id < PRIVATE_HUB_MIN_VERSION, use default hub + let target_hub_url = if version_id + .parse::() + .is_ok_and(|v| v < windmill_common::PRIVATE_HUB_MIN_VERSION) + { + windmill_common::DEFAULT_HUB_BASE_URL + } else { + &hub_base_url + }; + + // Call the hub's pick endpoint: /scripts/{version_id}/pick + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/{}/pick", target_hub_url, version_id), + None, + &db, + ) + .await?; + + Ok::<_, Error>((status_code, headers, response)) +} + +#[axum::debug_handler] +async fn get_script_by_path( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let script_o = if query.with_starred_info.unwrap_or(false) { + sqlx::query_as::<_, ScriptWithStarred>( + "SELECT s.*, favorite.path IS NOT NULL as starred + FROM script s + LEFT JOIN favorite + ON favorite.favorite_kind = 'script' + AND favorite.workspace_id = s.workspace_id + AND favorite.path = s.path + AND favorite.usr = $3 + WHERE s.path = $1 + AND s.workspace_id = $2 + ORDER BY s.created_at DESC LIMIT 1", + ) + .bind(path) + .bind(w_id) + .bind(&authed.username) + .fetch_optional(&mut *tx) + .await? + } else { + sqlx::query_as::<_, ScriptWithStarred>( + "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await? + }; + tx.commit().await?; + + let script = windmill_common::scripts::prefetch_cached_script_with_starred( + not_found_if_none(script_o, "Script", path)?, + &db, + ) + .await?; + + Ok(Json(script)) +} + +async fn list_tokens( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + list_tokens_internal(&db, &w_id, &path, false).await +} + +async fn get_script_by_path_w_draft( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let script_o = sqlx::query_as::<_, ScriptWDraft>( + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON + script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' + WHERE script.path = $1 AND script.workspace_id = $2 + ORDER BY script.created_at DESC LIMIT 1", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let script = not_found_if_none(script_o, "Script", path)?; + Ok(Json(script.prefetch_cached(&db).await?)) +} + +async fn get_script_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let query_result = sqlx::query!( + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg + FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash + WHERE s.workspace_id = $1 AND s.path = $2 + ORDER by s.created_at DESC", + w_id, + path, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + let result: Vec = query_result + .into_iter() + .map(|row| ScriptHistory { + script_hash: ScriptHash(row.hash), + deployment_msg: row.deployment_msg, + }) + .collect(); + return Ok(Json(result)); +} + +async fn get_latest_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + let row_o = sqlx::query!( + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg + FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash + WHERE s.workspace_id = $1 AND s.path = $2 + ORDER by s.created_at DESC LIMIT 1", + w_id, + path, + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + if let Some(row) = row_o { + let result = ScriptHistory { + script_hash: ScriptHash(row.hash), + deployment_msg: row.deployment_msg, // + }; + return Ok(Json(Some(result))); + } else { + return Ok(Json(None)); + } +} + +async fn update_script_history( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, script_hash, script_path)): Path<(String, ScriptHash, StripPath)>, + Json(script_history_update): Json, +) -> Result<()> { + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("scripts:write:{}", script_path))?; + + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL + DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", + w_id, + script_path, + script_hash.0, + script_history_update.deployment_msg, + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + return Ok(()); +} + +async fn list_paths( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + + let scripts = sqlx::query_scalar!( + "SELECT distinct(path) FROM script WHERE workspace_id = $1", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(scripts)) +} + +#[derive(Deserialize)] +pub struct ToggleWorkspaceErrorHandler { + #[cfg(feature = "enterprise")] + pub muted: Option, +} + +#[cfg(not(feature = "enterprise"))] +async fn toggle_workspace_error_handler( + _authed: ApiAuthed, + Extension(_user_db): Extension, + Path((_w_id, _path)): Path<(String, StripPath)>, + Json(_req): Json, +) -> Result { + return Err(Error::BadRequest( + "Muting the error handler for certain script is only available in enterprise version" + .to_string(), + )); +} + +#[cfg(feature = "enterprise")] +async fn toggle_workspace_error_handler( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(req): Json, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + let error_handler_maybe: Option = sqlx::query_scalar!( + "SELECT error_handler->>'path' FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(None); + + match error_handler_maybe { + Some(_) => { + sqlx::query_scalar!( + "UPDATE script + SET ws_error_handler_muted = $3 + WHERE ctid = ( + SELECT ctid FROM script + WHERE path = $1 AND workspace_id = $2 + ORDER BY created_at DESC + LIMIT 1 + ) +", + path.to_path(), + w_id, + req.muted, + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok("".to_string()) + } + None => { + tx.commit().await?; + Err(Error::BadRequest( + "Workspace error handler needs to be defined".to_string(), + )) + } + } +} + +async fn get_tokened_raw_script_by_path( + Extension(user_db): Extension, + Extension(db): Extension, + Extension(cache): Extension>, + Path((w_id, token, path)): Path<(String, String, StripPath)>, + Query(query): Query, +) -> Result { + let authed = cache + .get_authed(Some(w_id.clone()), &token) + .await + .ok_or_else(|| Error::NotAuthorized("Invalid token".to_string()))?; + return raw_script_by_path( + authed, + Extension(user_db), + Extension(db), + Path((w_id, path)), + Query(query), + ) + .await; +} + +async fn get_empty_ts_script_by_path() -> String { + return String::new(); +} + +#[derive(Deserialize)] +struct RawScriptByPathQuery { + // used to make cache immutable with respect to importer + cache_key: Option, + // used specifically for python to cache folders on import success to avoid extra db calls on package fetch + cache_folders: Option, +} + +struct StringWithLength(String); + +impl IntoResponse for StringWithLength { + fn into_response(self) -> axum::response::Response { + let len = self.0.len(); + ([(header::CONTENT_LENGTH, len.to_string())], self.0).into_response() + } +} + +async fn raw_script_by_path( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> Result { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {}", path.to_path()); + } + let r = raw_script_by_path_internal(path, user_db, db, authed, w_id, false, query).await?; + Ok(StringWithLength(r)) +} + +async fn raw_script_by_path_unpinned( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> Result { + let r = raw_script_by_path_internal(path, user_db, db, authed, w_id, true, query).await?; + Ok(StringWithLength(r)) +} + +lazy_static::lazy_static! { + static ref DEBUG_RAW_SCRIPT_ENDPOINTS: bool = + std::env::var("DEBUG_RAW_SCRIPT_ENDPOINTS").is_ok(); +} + +lazy_static::lazy_static! { + pub static ref RAW_SCRIPT_CACHE: Cache = Cache::new(1000); + pub static ref CACHE_FOLDERS_PATH: Cache = Cache::new(1000); + +} + +async fn raw_script_by_path_internal( + path: StripPath, + user_db: UserDB, + db: DB, + authed: ApiAuthed, + w_id: String, + unpin: bool, + query: RawScriptByPathQuery, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:read:{}", path))?; + let cache_path = query + .cache_key + .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); + if let Some(cache_path) = cache_path.clone() { + let cached_content = RAW_SCRIPT_CACHE.get(&cache_path); + if let Some(cached_content) = cached_content { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {} (cached)", path); + } + return Ok(cached_content); + } + } + + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {} (not cached)", path); + } + + if !path.ends_with(".py") + && !path.ends_with(".ts") + && !path.ends_with(".go") + && !path.ends_with(".sh") + { + return Err(Error::BadRequest(format!( + "Path must ends with a .py, .ts, .go. or .sh extension: {}", + path + ))); + } + let path = path + .trim_end_matches(".py") + .trim_end_matches(".bun.ts") + .trim_end_matches(".deno.ts") + .trim_end_matches(".ts") + .trim_end_matches(".go") + .trim_end_matches(".sh"); + + // folder cache is only useful for python given it needs to recuse over all intermediate folders to find the package. + // When a script exists in a folder, we can cache the fact that the folder exists to avoid extra db calls. + let mut split_path = path.split("/").collect::>(); + let folder_path = if query.cache_folders.is_some() && split_path.len() > 2 { + Some(format!("{w_id}:{path}/")) + } else { + None + }; + + let has_folder_cache = folder_path.is_some(); + if let Some(cache_folders) = folder_path { + let cached_content = CACHE_FOLDERS_PATH.get(&cache_folders); + if let Some(cached_ts) = cached_content { + if cached_ts >= chrono::Utc::now().timestamp() - 300 { + // 5 minutes + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {} (cached folders)", path); + } + return Ok("WINDMILL_IS_FOLDER".to_string()); + } else { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!( + "Raw script by path request: {} (cached folders expired)", + path + ); + } + } + } + } + + let mut tx = user_db.begin(&authed).await?; + + let content_o = sqlx::query_scalar!( + "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + path, + w_id + ) + .fetch_optional(&mut *tx) + .warn_after_seconds(5) + .await?; + tx.commit().await?; + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!( + "Raw script by path request: {} (content: {:?})", + path, + content_o + ); + } + + if content_o.is_none() { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1)", + path, + w_id + ) + .fetch_one(&db) + .warn_after_seconds(5) + .await? + .unwrap_or(false); + + if exists { + return Err(Error::NotFound(format!( + "Script {path} exists but {} does not have permissions to access it", + authed.username + ))); + } else { + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + let other_script_o = sqlx::query_scalar!( + "SELECT path FROM script WHERE workspace_id = $1 AND archived = false", + w_id + ) + .fetch_all(&db) + .await?; + let other_script_archived = sqlx::query_scalar!( + "SELECT distinct(path) FROM script WHERE workspace_id = $1 AND archived = true", + w_id + ) + .fetch_all(&db) + .await?; + tracing::warn!( + "Script {path} does not exist in workspace {w_id} but these paths do, non-archived: {:?} | archived: {:?}", + other_script_o.join(", "), + other_script_archived.join(", ") + ) + } + } + } + + let content = not_found_if_none(content_o, "Script", path)?; + + let content = if unpin { + remove_pinned_imports(&content)? + } else { + content + }; + + if has_folder_cache { + while split_path.len() >= 2 { + split_path.pop(); + let npath = split_path.join("/"); + CACHE_FOLDERS_PATH.insert(format!("{w_id}:{npath}/"), chrono::Utc::now().timestamp()); + } + } + + if let Some(cache_path) = cache_path { + RAW_SCRIPT_CACHE.insert(cache_path, content.clone()); + } + if *DEBUG_RAW_SCRIPT_ENDPOINTS { + tracing::warn!("Raw script by path request: {} (content response)", path); + } + Ok(content) +} + +async fn exists_script_by_path( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn get_script_by_hash_internal<'c>( + db: &mut Transaction<'c, Postgres>, + workspace_id: &str, + hash: &ScriptHash, + with_starred_info_for_username: Option<&str>, +) -> Result> { + let script_o = if let Some(username) = with_starred_info_for_username { + sqlx::query_as::<_, ScriptWithStarred>( + "SELECT s.*, favorite.path IS NOT NULL as starred + FROM script s + LEFT JOIN favorite + ON favorite.favorite_kind = 'script' + AND favorite.workspace_id = s.workspace_id + AND favorite.path = s.path + AND favorite.usr = $1 + WHERE s.hash = $2 AND s.workspace_id = $3", + ) + .bind(&username) + .bind(hash) + .bind(workspace_id) + .fetch_optional(&mut **db) + .await? + } else { + sqlx::query_as::<_, ScriptWithStarred>( + "SELECT *, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2", + ) + .bind(hash) + .bind(workspace_id) + .fetch_optional(&mut **db) + .await? + }; + + let script = not_found_if_none(script_o, "Script", hash.to_string())?; + Ok(script) +} + +#[derive(Deserialize)] +struct GetScriptByHashQuery { + authed: Option, +} +async fn get_script_by_hash( + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, hash)): Path<(String, ScriptHash)>, + Query(query): Query, + Query(query_auth): Query, + Extension(authed): Extension, +) -> JsonResult> { + let mut tx = if query_auth.authed.is_some_and(|x| x) { + user_db.begin(&authed).await? + } else { + db.begin().await? + }; + let r = get_script_by_hash_internal( + &mut tx, + &w_id, + &hash, + query.with_starred_info.and_then(|x| { + if x { + Some(authed.username.as_str()) + } else { + None + } + }), + ) + .await?; + + check_scopes(&authed, || format!("scripts:read:{}", &r.script.path))?; + + tx.commit().await?; + + Ok(Json(windmill_common::scripts::prefetch_cached_script_with_starred(r, &db).await?)) +} + +async fn raw_script_by_hash( + Extension(db): Extension, + Path((w_id, hash_str)): Path<(String, String)>, +) -> Result { + let mut tx = db.begin().await?; + let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| { + Error::BadRequest("Raw script path must end with .ts".to_string()) + })?)?); + let r = get_script_by_hash_internal(&mut tx, &w_id, &hash, None).await?; + tx.commit().await?; + + Ok(r.script.content) +} + +#[derive(Serialize)] +struct DeploymentStatus { + lock: Option, + lock_error_logs: Option, + job_id: Option, +} +async fn get_deployment_status( + Extension(db): Extension, + Path((w_id, hash)): Path<(String, ScriptHash)>, +) -> JsonResult { + let mut tx = db.begin().await?; + let status_o = sqlx::query!( + "SELECT s.lock, s.lock_error_logs, dm.job_id + FROM script s + LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash AND s.workspace_id = dm.workspace_id + WHERE s.hash = $1 AND s.workspace_id = $2", + hash.0, + w_id, + ) + .fetch_optional(&mut *tx) + .await?; + + let status = not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?; + + let deployment_status = DeploymentStatus { + lock: status.lock, + lock_error_logs: status.lock_error_logs, + job_id: status.job_id, + }; + + tx.commit().await?; + Ok(Json(deployment_status)) +} + +pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { + return windmill_api_auth::require_is_writer( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "script", + ) + .await; +} + +async fn archive_script_by_path( + authed: ApiAuthed, + Extension(webhook): Extension, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result<()> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot archive scripts for security reasons".to_string(), + )); + } + let path = path.to_path(); + check_scopes(&authed, || format!("scripts:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + require_owner_of_path(&authed, path)?; + + let hash: i64 = sqlx::query_scalar!( + "UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash", + path, + &w_id + ) + .fetch_one(&db) + .await + .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; + + clear_static_asset_usage(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + + audit_log( + &mut *tx, + &authed, + "scripts.archive", + ActionKind::Delete, + &w_id, + Some(&ScriptHash(hash).to_string()), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + + ScopedDependencyMap::clear_map_for_item(path, &w_id, "script", tx, &None) + .await + .commit() + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Script { + hash: ScriptHash(0), // dummy hash as it will not get inserted in db + path: path.to_string(), + parent_path: Some(path.to_string()), + }, + Some(format!("Script '{}' archived", path)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, + ); + + Ok(()) +} + +async fn archive_script_by_hash( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Extension(webhook): Extension, + Path((w_id, hash)): Path<(String, ScriptHash)>, +) -> JsonResult> { + if authed.is_operator { + return Err(Error::NotAuthorized( + "Operators cannot archive scripts for security reasons".to_string(), + )); + } + let mut tx = user_db.begin(&authed).await?; + + let script = sqlx::query_as::<_, Script>( + "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2 RETURNING *", + ) + .bind(&hash.0) + .bind(&w_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; + + check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; + clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + + audit_log( + &mut *tx, + &authed, + "scripts.archive", + ActionKind::Delete, + &w_id, + Some(&hash.to_string()), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + + ScopedDependencyMap::clear_map_for_item(&script.path, &w_id, "script", tx, &None) + .await + .commit() + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, + ); + + Ok(Json(windmill_common::scripts::prefetch_cached_script(script, &db).await?)) +} + +async fn delete_script_by_hash( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(webhook): Extension, + Extension(db): Extension, + Path((w_id, hash)): Path<(String, ScriptHash)>, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + + require_admin(authed.is_admin, &authed.username)?; + let script = sqlx::query_as::<_, Script>( + "UPDATE script SET content = '', archived = true, deleted = true, lock = '', schema = null WHERE hash = $1 AND \ + workspace_id = $2 RETURNING *", + ) + .bind(&hash.0) + .bind(&w_id) + .fetch_one(&db) + .await + .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; + + check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; + + clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + + audit_log( + &mut *tx, + &authed, + "scripts.delete", + ActionKind::Delete, + &w_id, + Some(&hash.to_string()), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + tx.commit().await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, + ); + + Ok(Json(windmill_common::scripts::prefetch_cached_script(script, &db).await?)) +} + +#[derive(Deserialize)] +struct DeleteScriptQuery { + keep_captures: Option, +} + +async fn delete_script_by_path( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(webhook): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> JsonResult { + let path = path.to_path(); + + check_scopes(&authed, || format!("scripts:write:{}", path))?; + + if path == "u/admin/hub_sync" && w_id == "admins" { + return Err(Error::BadRequest( + "Cannot delete the global setup app".to_string(), + )); + } + + let mut tx = user_db.begin(&authed).await?; + + let draft_only = sqlx::query_scalar!( + "SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + let script = if !draft_only { + require_admin(authed.is_admin, &authed.username)?; + sqlx::query_scalar!( + "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path", + path, + w_id + ) + .fetch_one(&db) + .await + .map_err(|e| Error::internal_err(format!("deleting script by path {w_id}: {e:#}")))? + } else { + sqlx::query_scalar!( + "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path", + path, + w_id + ) + .fetch_one(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("deleting script by path {w_id}: {e:#}")))? + }; + + sqlx::query!( + "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", + path, + w_id + ) + .execute(&db) + .await?; + + if !query.keep_captures.unwrap_or(false) { + sqlx::query!( + "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", + path, + w_id + ) + .execute(&db) + .await?; + + sqlx::query!( + "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", + path, + w_id + ) + .execute(&db) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "scripts.delete", + ActionKind::Delete, + &w_id, + Some(&path), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Script { + hash: ScriptHash(0), // Temporary value as it will get removed right after + path: path.to_string(), + parent_path: Some(path.to_string()), + }, + Some(format!("Script '{}' deleted", path)), + true, + None, + ) + .await?; + + sqlx::query!( + "DELETE FROM deployment_metadata WHERE path = $1 AND workspace_id = $2 AND script_hash IS NOT NULL", + path, + w_id + ) + .execute(&db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" + )) + })?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteScriptPath { workspace: w_id, path: path.to_owned() }, + ); + + Ok(Json(script)) +} + +async fn delete_scripts_bulk( + authed: ApiAuthed, + Extension(webhook): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult> { + for path in &request.paths { + check_scopes(&authed, || format!("scripts:write:{}", path))?; + } + + require_admin(authed.is_admin, &authed.username)?; + + if request.paths.contains(&"u/admin/hub_sync".to_string()) && w_id == "admins" { + return Err(Error::BadRequest( + "Cannot delete the global setup app".to_string(), + )); + } + + let mut tx = db.begin().await?; + + let mut deleted_paths = sqlx::query_scalar!( + "DELETE FROM script WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", + w_id, + &request.paths + ) + .fetch_all(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("deleting scripts in bulk {w_id}: {e:#}")))?; + + // remove duplicates from deleted_paths + deleted_paths.sort(); + deleted_paths.dedup(); + + sqlx::query!( + "DELETE FROM draft WHERE workspace_id = $1 AND path = ANY($2) AND typ = 'script'", + w_id, + &deleted_paths + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM capture_config WHERE workspace_id = $1 AND path = ANY($2) AND is_flow IS FALSE", + w_id, + &deleted_paths + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM capture WHERE workspace_id = $1 AND path = ANY($2) AND is_flow IS FALSE", + w_id, + &deleted_paths + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "scripts.delete_bulk", + ActionKind::Delete, + &w_id, + Some(&deleted_paths.join(", ")), + Some([("workspace", w_id.as_str())].into()), + ) + .await?; + + tx.commit().await?; + + try_join_all(deleted_paths.iter().map(|path| { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Script { hash: ScriptHash(0), path: path.clone(), parent_path: None }, + Some(format!("Script '{}' deleted", path)), + true, + None, + ) + })) + .await?; + + sqlx::query!( + "DELETE FROM deployment_metadata WHERE workspace_id = $1 AND path = ANY($2) AND script_hash IS NOT NULL", + w_id, + &deleted_paths + ) + .execute(&db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error deleting deployment metadata for scripts with paths {} in workspace {w_id}: {e:#}", deleted_paths.join(", ") + )) + })?; + + for path in &deleted_paths { + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteScriptPath { workspace: w_id.clone(), path: path.to_owned() }, + ); + } + + Ok(Json(deleted_paths)) +} + +/// Validates that script debouncing configuration is supported by all workers +/// Returns an error if debouncing is configured but workers are behind required version +async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { + if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await && !ns.debouncing_settings.is_default() { + tracing::warn!( + "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature" + ); + Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() }) + } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await + && !ns.debouncing_settings.is_legacy_compatible() + && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT + { + tracing::warn!( + "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature" + ); + Err(Error::WorkersAreBehind { + feature: "V2 Debouncing".into(), + min_version: "1.597.0".into(), + }) + } else { + Ok(()) + } +} diff --git a/backend/windmill-api-sse/Cargo.toml b/backend/windmill-api-sse/Cargo.toml new file mode 100644 index 0000000000..b8127c3d36 --- /dev/null +++ b/backend/windmill-api-sse/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "windmill-api-sse" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_sse" +path = "src/lib.rs" + +[features] +default = [] +enterprise = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +uuid.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api-sse/src/lib.rs b/backend/windmill-api-sse/src/lib.rs new file mode 100644 index 0000000000..1db4aa9433 --- /dev/null +++ b/backend/windmill-api-sse/src/lib.rs @@ -0,0 +1,341 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Shared SSE types for job update streaming. +//! +//! This crate contains the types used by the SSE job update system, +//! separated out so they can be depended on without pulling in the +//! full windmill-api-jobs crate. + +use serde::Serialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::ops::{Deref, DerefMut}; +use uuid::Uuid; +use windmill_common::flow_status::FlowStatus; +use windmill_common::jobs::{CompletedJob, JobKind, QueuedJob}; + +// ------------ Job types (from jobs.rs) ------------ + +#[derive(sqlx::FromRow, Debug, Serialize)] +pub struct JobExtended { + #[sqlx(flatten)] + #[serde(flatten)] + pub inner: T, + + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_lock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_flow: Option>>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub worker: Option, + + #[sqlx(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub self_wait_time_ms: Option, + #[sqlx(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub aggregate_wait_time_ms: Option, +} + +pub trait JobCommon { + fn job_kind(&self) -> &JobKind; +} + +impl JobCommon for QueuedJob { + fn job_kind(&self) -> &JobKind { + &self.job_kind + } +} + +impl JobCommon for CompletedJob { + fn job_kind(&self) -> &JobKind { + &self.job_kind + } +} + +impl JobExtended { + pub fn new( + self_wait_time_ms: Option, + aggregate_wait_time_ms: Option, + inner: T, + ) -> Self { + Self { + inner, + raw_code: None, + raw_lock: None, + raw_flow: None, + worker: None, + self_wait_time_ms, + aggregate_wait_time_ms, + } + } +} + +impl Deref for JobExtended { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for JobExtended { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +#[derive(Serialize, Debug)] +#[serde(tag = "type")] +pub enum Job { + QueuedJob(JobExtended), + CompletedJob(JobExtended), +} + +impl Job { + pub fn created_by(&self) -> &str { + match self { + Job::QueuedJob(job) => &job.created_by, + Job::CompletedJob(job) => &job.created_by, + } + } + + pub fn append_to_logs(&mut self, logs: &str) { + match self { + Job::QueuedJob(job) => { + if let Some(ref mut l) = job.logs { + l.push_str(logs); + } else { + job.logs = Some(logs.to_string()); + } + } + Job::CompletedJob(job) => { + if let Some(ref mut l) = job.logs { + l.push_str(logs); + } else { + job.logs = Some(logs.to_string()); + } + } + } + } + + pub fn log_len(&self) -> Option { + match self { + Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()), + Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()), + } + } + + pub fn logs(&self) -> Option { + match self { + Job::QueuedJob(job) => job.logs.clone(), + Job::CompletedJob(job) => job.logs.clone(), + } + } + + pub fn flow_status(&self) -> Option { + match self { + Job::QueuedJob(job) => job + .flow_status + .as_ref() + .and_then(|rf| serde_json::from_str(rf.0.get()).ok()), + Job::CompletedJob(job) => job + .flow_status + .as_ref() + .and_then(|rf| serde_json::from_str(rf.0.get()).ok()), + } + } + + pub fn is_flow_step(&self) -> bool { + match self { + Job::QueuedJob(job) => job.is_flow_step, + Job::CompletedJob(job) => job.is_flow_step, + } + } + + pub fn is_flow(&self) -> bool { + self.job_kind().is_flow() + } + + pub fn job_kind(&self) -> &JobKind { + match self { + Job::QueuedJob(job) => &job.job_kind, + Job::CompletedJob(job) => &job.job_kind, + } + } + + pub fn id(&self) -> Uuid { + match self { + Job::QueuedJob(job) => job.id, + Job::CompletedJob(job) => job.id, + } + } + + pub fn workspace_id(&self) -> &String { + match self { + Job::QueuedJob(job) => &job.workspace_id, + Job::CompletedJob(job) => &job.workspace_id, + } + } + + pub fn script_path(&self) -> &str { + match self { + Job::QueuedJob(job) => job.script_path.as_ref(), + Job::CompletedJob(job) => job.script_path.as_ref(), + } + .map(String::as_str) + .unwrap_or("tmp/main") + } + + pub fn args(&self) -> Option<&sqlx::types::Json>>> { + match self { + Job::QueuedJob(job) => job.args.as_ref(), + Job::CompletedJob(job) => job.args.as_ref(), + } + } + + pub fn full_path_with_workspace(&self) -> String { + format!( + "{}/{}/{}", + self.workspace_id(), + if self.is_flow() { "flow" } else { "script" }, + self.script_path(), + ) + } + + pub async fn concurrency_key( + &self, + db: &sqlx::Pool, + ) -> Result, sqlx::Error> { + sqlx::query_scalar!( + "SELECT key FROM concurrency_key WHERE job_id = $1", + self.id() + ) + .fetch_optional(db) + .await + } + + pub async fn fetch_outstanding_wait_time( + &mut self, + db: &sqlx::Pool, + ) -> Result<(), sqlx::Error> { + let r = sqlx::query!( + "SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1", + self.id() + ) + .fetch_optional(db) + .await?; + + let (self_wait_time, aggregate_wait_time) = r + .map(|x| (x.self_wait_time_ms, x.aggregate_wait_time_ms)) + .unwrap_or((None, None)); + + match self { + Job::QueuedJob(job) => { + job.self_wait_time_ms = self_wait_time; + job.aggregate_wait_time_ms = aggregate_wait_time; + } + Job::CompletedJob(job) => { + job.self_wait_time_ms = self_wait_time; + job.aggregate_wait_time_ms = aggregate_wait_time; + } + } + Ok(()) + } +} + +// ------------ SSE types ------------ + +#[derive(Serialize, Debug)] +pub struct JobUpdate { + #[serde(skip_serializing_if = "Option::is_none")] + pub running: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub completed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_logs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_result_stream: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub log_offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub only_result: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_stream_job_id: Option, +} + +impl JobUpdate { + pub fn hash_str(&self) -> String { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + format!("{:x}", hasher.finish()) + } +} + +impl Hash for JobUpdate { + fn hash(&self, state: &mut H) { + self.running.hash(state); + self.completed.hash(state); + self.log_offset.hash(state); + self.mem_peak.hash(state); + self.progress.hash(state); + self.stream_offset.hash(state); + self.flow_stream_job_id.hash(state); + if !self.completed.unwrap_or(false) { + self.flow_status.as_ref().map(|x| x.get().hash(state)); + self.workflow_as_code_status + .as_ref() + .map(|x| x.get().hash(state)); + } + } +} + +#[derive(serde::Deserialize)] +pub struct JobUpdateQuery { + pub running: Option, + pub log_offset: Option, + pub stream_offset: Option, + pub get_progress: Option, + pub no_logs: Option, + pub only_result: Option, + pub fast: Option, + pub is_flow: Option, + pub poll_delay_ms: Option, +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum JobUpdateSSEStream { + Update(JobUpdate), + Error { error: String }, + NotFound, + Timeout, + Ping, +} + +lazy_static::lazy_static! { + pub static ref TIMEOUT_SSE_STREAM: u64 = + std::env::var("TIMEOUT_SSE_STREAM").unwrap_or("60".to_string()).parse::().unwrap_or(60); +} diff --git a/backend/windmill-api-users/Cargo.toml b/backend/windmill-api-users/Cargo.toml new file mode 100644 index 0000000000..e720b37eb7 --- /dev/null +++ b/backend/windmill-api-users/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "windmill-api-users" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_users" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +private = ["windmill-common/private"] +parquet = [] +no_auth = ["windmill-api-auth/no_auth"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true + +argon2.workspace = true +axum.workspace = true +chrono.workspace = true +http.workspace = true +hyper.workspace = true +lazy_static.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +time.workspace = true +tokio.workspace = true +tower-cookies.workspace = true +tracing.workspace = true diff --git a/backend/windmill-api-users/src/lib.rs b/backend/windmill-api-users/src/lib.rs new file mode 100644 index 0000000000..97aab59b2a --- /dev/null +++ b/backend/windmill-api-users/src/lib.rs @@ -0,0 +1,2 @@ +pub mod users; +pub mod users_oss; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs new file mode 100644 index 0000000000..014ad44931 --- /dev/null +++ b/backend/windmill-api-users/src/users.rs @@ -0,0 +1,2351 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#![allow(non_snake_case)] + +use sqlx::{Postgres, Transaction}; + +use std::sync::atomic::AtomicBool; +use std::sync::Arc; +use std::time::Duration; + +use windmill_api_auth::ApiAuthed; + +pub use windmill_api_auth::Tokened; + +use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use axum::{ + extract::{Extension, Path, Query}, + response::{IntoResponse, Response}, + routing::{delete, get, post}, + Json, Router, +}; +use hyper::{header::LOCATION, StatusCode}; +use windmill_api_auth::require_super_admin; +use windmill_common::usernames::{ + generate_instance_wide_unique_username, get_instance_username_or_create_pending, +}; +use windmill_common::utils::{COOKIE_DOMAIN, IS_SECURE}; +use windmill_common::webhook::WebhookShared; +use windmill_common::DB; + +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use time::OffsetDateTime; +use tower_cookies::{Cookie, Cookies}; +use tracing::Instrument; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::audit::AuditAuthor; +use windmill_common::auth::TOKEN_PREFIX_LEN; +use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; +use windmill_common::oauth2::InstanceEvent; +use windmill_common::users::truncate_token; +use windmill_common::users::COOKIE_NAME; +use windmill_common::utils::paginate; +use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::BASE_URL; +use windmill_common::{ + auth::{get_folders_for_user, get_groups_for_user}, + db::UserDB, + error::{self, Error, JsonResult, Result}, + utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, +}; +use windmill_git_sync::handle_deployment_metadata; + +const COOKIE_PATH: &str = "/"; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_users)) + .route("/list_usage", get(list_user_usage)) + .route("/list_usernames", get(list_usernames)) + .route("/exists", post(exists_username)) + .route("/get/:user", get(get_workspace_user)) + .route("/update/:user", post(update_workspace_user)) + .route("/delete/:user", delete(delete_workspace_user)) + .route("/convert_to_group/:user", post(convert_user_to_group)) + .route("/is_owner/*path", get(is_owner_of_path)) + .route("/whois/:username", get(whois)) + .route("/whoami", get(whoami)) + .route("/leave", post(leave_workspace)) + .route("/username_to_email/:username", get(username_to_email)) +} + +pub fn global_service() -> Router { + Router::new() + .route("/exists/:email", get(exists_email)) + .route("/email", get(get_email)) + .route("/whoami", get(global_whoami)) + .route("/list_invites", get(list_invites)) + .route("/decline_invite", post(decline_invite)) + .route("/accept_invite", post(accept_invite)) + .route("/list_as_super_admin", get(list_users_as_super_admin)) + .route("/set_login_type/:user", post(set_login_type)) + .route("/update/:user", post(update_user)) + .route("/delete/:user", delete(delete_user)) + .route("/username_info/:user", get(get_instance_username_info)) + .route("/tokens/create", post(create_token)) + .route("/tokens/delete/:token_prefix", delete(delete_token)) + .route("/tokens/list", get(list_tokens)) + .route("/tokens/impersonate", post(impersonate)) + .route("/usage", get(get_usage)) + .route("/all_runnables", get(get_all_runnables)) + .route("/refresh_token", get(refresh_token)) + .route( + "/tutorial_progress", + post(update_tutorial_progress).get(get_tutorial_progress), + ) + .route("/leave_instance", post(leave_instance)) + .route("/export", get(export_global_users)) + .route("/overwrite", post(overwrite_global_users)) +} + +pub fn make_unauthed_service() -> Router { + Router::new() + .route("/login", post(login)) + .route("/logout", post(logout).get(logout)) + .route("/is_first_time_setup", get(is_first_time_setup)) + .route("/request_password_reset", post(request_password_reset)) + .route("/is_smtp_configured", get(is_smtp_configured)) +} + +pub use windmill_api_auth::{ + create_token_internal, fetch_api_authed, get_scope_tags, maybe_refresh_folders, + require_is_writer, require_path_read_access_for_preview, NewToken, OptAuthed, +}; + +#[cfg(feature = "parquet")] +pub use windmill_api_auth::fetch_api_authed_from_permissioned_as; + +#[derive(FromRow, Serialize)] +pub struct User { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub created_at: chrono::DateTime, + pub operator: bool, + pub disabled: bool, + pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub added_via: Option, +} + +#[derive(Serialize)] +pub struct UserWithUsage { + pub email: String, + pub executions: Option, +} + +#[derive(FromRow, Serialize, Debug)] +pub struct GlobalUserInfo { + email: String, + login_type: Option, + super_admin: bool, + devops: bool, + verified: bool, + name: Option, + company: Option, + username: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operator_only: Option, + first_time_user: bool, +} + +#[derive(Serialize, Debug)] +pub struct UserInfo { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub is_super_admin: bool, + pub created_at: chrono::DateTime, + pub groups: Vec, + pub operator: bool, + pub disabled: bool, + pub role: Option, + pub folders_read: Vec, + pub folders: Vec, + pub folders_owners: Vec, + pub name: Option, +} + +#[derive(FromRow, Serialize)] +pub struct WorkspaceInvite { + pub workspace_id: String, + pub email: String, + pub is_admin: bool, + pub operator: bool, + pub parent_workspace_id: Option, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct NewUser { + pub email: String, + pub password: String, + pub super_admin: bool, + pub name: Option, + pub company: Option, + pub skip_email: Option, +} + +#[derive(Deserialize)] +pub struct AcceptInvite { + pub workspace_id: String, + pub username: Option, +} + +#[derive(Deserialize)] +pub struct DeclineInvite { + pub workspace_id: String, +} + +#[derive(Deserialize)] +pub struct EditUser { + pub is_super_admin: Option, + pub is_devops: Option, + pub name: Option, +} + +#[derive(Deserialize)] +pub struct EditWorkspaceUser { + pub is_admin: Option, + pub operator: Option, + pub disabled: Option, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct EditPassword { + pub password: String, +} + +#[derive(Deserialize)] +pub struct EditLoginType { + pub login_type: String, +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedToken { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, +} + +// NewToken is re-exported from windmill-api-auth above + +#[derive(Deserialize)] +pub struct Login { + pub email: String, + pub password: String, +} + +lazy_static::lazy_static! { + static ref FIRST_TIME_SETUP: Arc = Arc::new(AtomicBool::new(true)); +} + +pub async fn is_first_time_setup(Extension(db): Extension) -> JsonResult { + if !FIRST_TIME_SETUP.load(std::sync::atomic::Ordering::Relaxed) { + return Ok(Json(false)); + } + let single_user = sqlx::query_scalar!("SELECT 1 FROM password LIMIT 2") + .fetch_all(&db) + .await + .ok() + .unwrap_or_default() + .len() + == 1; + if single_user { + let user_is_admin_and_password_changeme = sqlx::query_scalar!( + "SELECT 1 FROM password WHERE email = 'admin@windmill.dev' AND password_hash = '$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I'" + ).fetch_all(&db) + .await + .ok() + .unwrap_or_default() + .len() == 1; + if user_is_admin_and_password_changeme { + let base_url_is_not_set = + sqlx::query_scalar!("SELECT COUNT(*) FROM global_settings WHERE name = 'base_url'") + .fetch_optional(&db) + .await + .ok() + .flatten() + .flatten() + .unwrap_or(0) + == 0; + if base_url_is_not_set { + return Ok(Json(true)); + } + } + } + FIRST_TIME_SETUP.store(false, std::sync::atomic::Ordering::Relaxed); + Ok(Json(false)) +} + +#[derive(Deserialize)] +struct WorkspaceUsername { + pub username: String, +} + +async fn exists_username( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(WorkspaceUsername { username }): Json, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", + &w_id, + &username + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + tx.commit().await?; + Ok(Json(exists)) +} + +async fn list_users( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + require_admin(authed.is_admin, &authed.username)?; + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as!( + User, + " + SELECT * + FROM usr + WHERE workspace_id = $1 + ", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_user_usage( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + require_admin(authed.is_admin, &authed.username)?; + } + let mut tx = user_db.begin(&authed).await?; + let rows = tokio::time::timeout( + Duration::from_secs(300), + sqlx::query_as!( + UserWithUsage, + " + SELECT usr.email, usage.executions + FROM usr, LATERAL ( + SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.workspace_id = $1 + AND j.kind NOT IN ('flow', 'flowpreview', 'flownode') + AND j.permissioned_as_email = usr.email + AND now() - '1 week'::interval < j.created_at + ) usage + WHERE workspace_id = $1 + ", + w_id + ) + .fetch_all(&mut *tx), + ) + .await + .map_err(|e| Error::internal_err(format!("Timed out while fetching user usage: {e:#}")))??; + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct ActiveUsersOnly { + active_only: Option, +} + +async fn list_users_as_super_admin( + authed: ApiAuthed, + Extension(db): Extension, + Query(pagination): Query, + Query(ActiveUsersOnly { active_only }): Query, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let per_page = pagination.per_page.unwrap_or(10000).max(1); + let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; + + let rows = if active_only.is_some_and(|x| x) { + sqlx::query_as!( + GlobalUserInfo, + "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), + authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) + SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user + FROM password + WHERE email IN (SELECT email FROM active_users) + ORDER BY super_admin DESC, devops DESC + LIMIT $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&db) + .await? + } else { + sqlx::query_as!( + GlobalUserInfo, + "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \ + $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&db) + .await? + }; + + Ok(Json(rows)) +} + +#[derive(Serialize, Deserialize)] +struct Progress { + progress: u64, + skipped_all: bool, +} +async fn get_tutorial_progress( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult { + let row = sqlx::query!( + "SELECT progress::bigint as progress, skipped_all FROM tutorial_progress WHERE email = $1", + authed.email + ) + .fetch_optional(&db) + .await?; + + if let Some(row) = row { + Ok(Json(Progress { + progress: row.progress.unwrap_or_default() as u64, + skipped_all: row.skipped_all, + })) + } else { + Ok(Json(Progress { progress: 0, skipped_all: false })) + } +} + +async fn update_tutorial_progress( + authed: ApiAuthed, + Extension(db): Extension, + Json(progress): Json, +) -> Result { + sqlx::query!( + "INSERT INTO tutorial_progress (email, progress, skipped_all) VALUES ($2, $1::bigint::bit(64), $3) ON CONFLICT (email) DO UPDATE SET progress = EXCLUDED.progress, skipped_all = EXCLUDED.skipped_all", + progress.progress as i64, + authed.email, + progress.skipped_all + ) + .execute(&db) + .await?; + Ok("tutorial progress updated".to_string()) +} + +async fn list_usernames( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + return Ok(Json(vec![ + authed.username, + "other_usernames_redacted_in_demo_workspace".to_string(), + ])); + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_invites( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + let mut tx = db.begin().await?; + let rows = sqlx::query_as!( + WorkspaceInvite, + "SELECT + workspace_invite.workspace_id, + workspace_invite.email, + workspace_invite.is_admin, + workspace_invite.operator, + workspace.parent_workspace_id + FROM workspace_invite JOIN workspace ON workspace_invite.workspace_id = workspace.id WHERE email = $1", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +lazy_static::lazy_static! { + static ref INVALIDATE_ALL_SESSIONS_ON_LOGOUT: bool = std::env::var("INVALIDATE_ALL_SESSIONS_ON_LOGOUT") + .unwrap_or("false".to_string()) + .parse::() + .unwrap_or(false); +} + +#[derive(Deserialize)] +struct LogoutQuery { + rd: Option, +} +async fn logout( + Tokened { token }: Tokened, + cookies: Cookies, + Extension(db): Extension, + Query(LogoutQuery { rd }): Query, +) -> Result { + let mut cookie = Cookie::new(COOKIE_NAME, ""); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + cookies.remove(cookie); + let mut tx = db.begin().await?; + + let email = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT { + sqlx::query_scalar!( + "WITH email_lookup AS ( + SELECT email FROM token WHERE token = $1 + ) + DELETE FROM token + WHERE email = (SELECT email FROM email_lookup) AND label = 'session' + RETURNING email", + token + ) + .fetch_optional(&mut *tx) + .await? + } else { + sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token) + .fetch_optional(&mut *tx) + .await? + }; + + if let Some(email) = email { + let email = email.unwrap_or("noemail".to_string()); + let audit_message = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT { + "users.logout_all" + } else { + "users.logout" + }; + audit_log( + &mut *tx, + &AuditAuthor { + email: email.clone(), + username: email, + username_override: None, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }, + audit_message, + ActionKind::Delete, + "global", + Some(&truncate_token(&token)), + None, + ) + .await?; + } + tx.commit().await?; + if let Some(rd) = rd { + Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response()) + } else { + Ok((StatusCode::OK, "logged out successfully".to_string()).into_response()) + } +} + +async fn whoami( + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed, +) -> JsonResult { + let user = get_user(&w_id, &username, &db).await?; + if let Some(user) = user { + Ok(Json(user)) + } else { + Ok(Json(UserInfo { + workspace_id: w_id, + email: email.clone(), + username: email, + name: None, + is_admin, + is_super_admin: is_admin, + created_at: chrono::Utc::now(), + groups: groups, + operator: false, + disabled: false, + role: Some("superadmin".to_string()), + folders_read: folders.clone().into_iter().map(|x| x.0).collect(), + folders: folders + .clone() + .into_iter() + .filter_map(|x| if x.1 { Some(x.0) } else { None }) + .collect(), + folders_owners: folders + .into_iter() + .filter_map(|x| if x.2 { Some(x.0) } else { None }) + .collect(), + })) + } +} + +async fn global_whoami( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, + Tokened { token }: Tokened, +) -> JsonResult { + let user = sqlx::query_as!( + GlobalUserInfo, + "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user FROM password WHERE \ + email = $1", + email + ) + .fetch_one(&db) + .await + .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}"))); + + if let Ok(user) = user { + Ok(Json(user)) + } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) { + Ok(Json(GlobalUserInfo { + email: email.clone(), + login_type: Some("superadmin_secret".to_string()), + super_admin: true, + devops: false, + verified: true, + name: None, + company: None, + username: None, + operator_only: None, + first_time_user: false, + })) + } else { + Err(user.unwrap_err()) + } +} + +async fn exists_email(Extension(db): Extension, Path(email): Path) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +async fn get_email(ApiAuthed { email, .. }: ApiAuthed) -> Result { + Ok(email) +} + +async fn get_usage( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, +) -> Result { + let usage = sqlx::query_scalar!( + " + SELECT usage.usage FROM usage + WHERE is_workspace = false + AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) + AND id = $1", + email + ) + .fetch_optional(&db) + .await? + .unwrap_or(0); + Ok(usage.to_string()) +} + +#[derive(FromRow, Serialize)] +pub struct User2 { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub created_at: chrono::DateTime, + pub operator: bool, + pub disabled: bool, + pub role: Option, + pub super_admin: bool, + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub added_via: Option, +} + +async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { + let user = sqlx::query_as!( + User2, + "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2 + ", + username, + w_id + ) + .fetch_optional(db) + .await?; + let groups = get_groups_for_user( + &w_id, + username, + &user + .as_ref() + .map(|x| x.email.to_string()) + .unwrap_or_else(|| "".to_string()), + db, + ) + .await?; + let folders = get_folders_for_user(&w_id, username, &groups, db).await?; + + Ok(user.map(|usr| UserInfo { + groups, + workspace_id: usr.workspace_id, + email: usr.email, + username: usr.username, + name: usr.name, + is_admin: usr.is_admin, + is_super_admin: usr.super_admin, + created_at: usr.created_at, + operator: usr.operator, + disabled: usr.disabled, + role: usr.role, + folders_read: folders.clone().into_iter().map(|x| x.0).collect(), + folders: folders + .clone() + .into_iter() + .filter_map(|x| if x.1 { Some(x.0) } else { None }) + .collect(), + folders_owners: folders + .into_iter() + .filter_map(|x| if x.2 { Some(x.0) } else { None }) + .collect(), + })) +} + +pub async fn is_owner_of_path( + authed: ApiAuthed, + Path((_w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + if authed.is_admin { + Ok(Json(true)) + } else { + Ok(Json(require_owner_of_path(&authed, path).is_ok())) + } +} + +pub use windmill_api_auth::require_owner_of_path; + +// get_perm_in_extra_perms_for_authed and require_is_writer are re-exported from windmill-api-auth above +async fn whois( + Extension(db): Extension, + Path((w_id, username)): Path<(String, String)>, +) -> JsonResult { + let user_o = get_user(&w_id, &username, &db).await?; + let user = not_found_if_none(user_o, "User", username)?; + Ok(Json(user)) +} + +// async fn create_invite_code( +// ApiAuthed { email, .. }: ApiAuthed, +// Extension(db): Extension, +// Json(nu): Json, +// ) -> Result<(StatusCode, String)> { + +// let mut tx = db.begin().await?; +// require_super_admin(&mut *tx, email).await?; + +// sqlx::query!( +// "INSERT INTO invite_code +// (code, seats_left) +// VALUES ($1, $2)", +// nu.code, +// nu.seats +// ) +// .execute(&mut *tx) +// .await?; + +// tx.commit().await?; + +// Ok(( +// StatusCode::CREATED, +// format!("new invite code {}", nu.code), +// )) +// } + +async fn decline_invite( + authed: ApiAuthed, + Extension(db): Extension, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let is_admin = sqlx::query_scalar!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin", + nu.workspace_id, + authed.email, + ) + .fetch_optional(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.decline_invite", + ActionKind::Delete, + &nu.workspace_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + if is_admin.is_some() { + Ok(( + StatusCode::OK, + format!( + "user {} declined invite to workspace {}", + &authed.email, nu.workspace_id + ), + )) + } else { + Err(Error::NotFound(format!( + "invite for {} not found", + authed.email + ))) + } +} + +pub use windmill_common::usernames::VALID_USERNAME; + +async fn accept_invite( + authed: ApiAuthed, + Extension(webhook): Extension, + Extension(db): Extension, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let r = sqlx::query!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin, operator", + nu.workspace_id, + authed.email, + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(r) = r { + let already_in_workspace = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &nu.workspace_id, + &authed.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_in_workspace { + tx.commit().await?; + return Ok(( + StatusCode::CREATED, + format!( + "user {} accepted invite to workspace {}", + &authed.email, nu.workspace_id + ), + )); + } + let username; + (tx, username) = join_workspace( + &nu.workspace_id, + &authed, + nu.username, + r.is_admin, + r.operator, + tx, + ) + .await?; + + audit_log( + &mut *tx, + &ApiAuthed { username: username.clone(), ..authed.clone() }, + "users.accept_invite", + ActionKind::Create, + &nu.workspace_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &username, + &db, + &nu.workspace_id, + windmill_git_sync::DeployedObject::User { email: authed.email.clone() }, + Some(format!("User '{}' accepted invite", &authed.email)), + true, + None, + ) + .await?; + webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace { + email: authed.email.clone(), + workspace: nu.workspace_id.clone(), + username: username, + }); + Ok(( + StatusCode::CREATED, + format!( + "user {} accepted invite to workspace {}", + &authed.email, nu.workspace_id + ), + )) + } else { + Err(Error::NotFound(format!( + "invite for {} not found", + authed.email + ))) + } +} + +async fn join_workspace<'c>( + w_id: &str, + authed: &ApiAuthed, + username: Option, + is_admin: bool, + operator: bool, + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, +) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, String)> { + let automate_username_creation = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + AUTOMATE_USERNAME_CREATION_SETTING, + ) + .fetch_optional(&mut *tx) + .await? + .map(|v| v.as_bool()) + .flatten() + .unwrap_or(false); + + let username = if automate_username_creation { + if username.is_some() && username.unwrap().len() > 0 { + return Err(Error::BadRequest( + "username is not allowed when username creation is automated".to_string(), + )); + } + get_instance_username_or_create_pending(&mut tx, &authed.email).await? + } else { + let username = username.ok_or(Error::BadRequest("username is required".to_string()))?; + let already_exists_username = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", + &w_id, + username, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_username { + return Err(Error::BadRequest(format!( + "user with username {} already exists in workspace {}", + username, w_id + ))); + } + + if !VALID_USERNAME.is_match(&username) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Usermame can only contain alphanumeric characters and underscores and must start with a letter" + ))); + } + username.to_string() + }; + + let already_exists_email = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &w_id, + authed.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_email { + return Err(Error::BadRequest(format!( + "user with email {} already exists in workspace {}", + authed.email, w_id + ))); + } + + sqlx::query!( + "INSERT INTO usr + (workspace_id, email, username, is_admin, operator) + VALUES ($1, $2, $3, $4, $5)", + &w_id, + authed.email, + username, + is_admin, + operator + ) + .execute(&mut *tx) + .await?; + sqlx::query_as!( + Group, + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + username, + "all", + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &AuditAuthor { username: username.clone(), ..authed.into() }, + "users.add_to_workspace", + ActionKind::Create, + &w_id, + Some(&authed.email), + None, + ) + .await?; + Ok((tx, username)) +} + +async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Result { + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.leave", + ActionKind::Delete, + "global", + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Left instance",)) +} + +async fn get_workspace_user( + ApiAuthed { username, is_admin, .. }: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_update)): Path<(String, String)>, +) -> Result> { + require_admin(is_admin, &username)?; + + let user = sqlx::query_as!( + User, + "SELECT * FROM usr WHERE username = $1 AND workspace_id = $2", + &username_to_update, + &w_id + ) + .fetch_optional(&db) + .await?; + + let user = not_found_if_none(user, "User", username_to_update)?; + + Ok(Json(user)) +} + +async fn update_workspace_user( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_update)): Path<(String, String)>, + Json(eu): Json, +) -> Result { + let mut tx = db.begin().await?; + + require_admin(authed.is_admin, &authed.username)?; + + update_workspace_user_internal( + &w_id, + &username_to_update, + eu.is_admin, + eu.operator, + eu.disabled, + &mut tx, + Some(&authed), + ) + .await?; + + let user_email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + &username_to_update, + &w_id + ) + .fetch_one(&mut *tx) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: user_email.clone() }, + Some(format!("Updated user '{}'", &user_email)), + true, + None, + ) + .await?; + + Ok(format!("user {} updated", user_email)) +} + +async fn convert_user_to_group( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_convert)): Path<(String, String)>, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let mut tx = db.begin().await?; + + // Get user email and current status + let user_info = sqlx::query!( + "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2", + username_to_convert, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + let user_info = not_found_if_none(user_info, "User", &username_to_convert)?; + + // Check if user is already a group user + if let Some(added_via) = &user_info.added_via { + if added_via.get("source").and_then(|v| v.as_str()) == Some("instance_group") { + return Err(Error::BadRequest( + "User is already a group user".to_string(), + )); + } + } + + // Find which instance groups this user belongs to that are configured for auto-add in this workspace + let eligible_groups = sqlx::query!( + r#" + SELECT + eig.igroup as group_name, + ws.auto_invite->'instance_groups_roles' as instance_groups_roles + FROM email_to_igroup eig + INNER JOIN workspace_settings ws ON ws.workspace_id = $1 + WHERE eig.email = $2 + AND ws.auto_invite->'instance_groups' ? eig.igroup + "#, + &w_id, + &user_info.email + ) + .fetch_all(&mut *tx) + .await?; + + if eligible_groups.is_empty() { + return Err(Error::BadRequest( + "User is not a member of any instance groups configured for auto-add in this workspace" + .to_string(), + )); + } + + // Determine the group with highest precedence (same logic as process_instance_group_auto_adds) + let roles: std::collections::HashMap = + if let Some(roles_json) = &eligible_groups[0].instance_groups_roles { + serde_json::from_value(roles_json.clone()).unwrap_or_default() + } else { + std::collections::HashMap::new() + }; + + let mut best_group = &eligible_groups[0].group_name; + let mut best_precedence = 0u8; + + for group in &eligible_groups { + let default_role = "developer".to_string(); + let role = roles.get(&group.group_name).unwrap_or(&default_role); + + let precedence = match role.as_str() { + "admin" => 3, + "developer" => 2, + "operator" => 1, + _ => 2, + }; + + if precedence > best_precedence { + best_precedence = precedence; + best_group = &group.group_name; + } + } + + let primary_group_name = best_group; + + // Determine role from group configuration using the selected primary group + let default_role = "developer".to_string(); + let role = roles + .get(primary_group_name) + .unwrap_or(&default_role) + .as_str(); + + let (is_admin, is_operator) = match role { + "admin" => (true, false), + "operator" => (false, true), + _ => (false, false), + }; + + // Update user with instance group information + let instance_group_source = serde_json::json!({ + "source": "instance_group", + "group": primary_group_name + }); + + sqlx::query!( + "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5", + instance_group_source, + is_admin, + is_operator, + username_to_convert, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.convert_to_group", + ActionKind::Update, + &w_id, + Some(&username_to_convert), + Some([("group", primary_group_name.as_str()), ("role", role)].into()), + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: user_info.email.clone() }, + Some(format!( + "Converted user '{}' to group user (group: {}, role: {})", + &user_info.email, primary_group_name, role + )), + true, + None, + ) + .await?; + + Ok(format!( + "User {} converted to group user (group: {}, role: {})", + username_to_convert, primary_group_name, role + )) +} + +async fn update_user( + authed: ApiAuthed, + Path(email_to_update): Path, + Extension(db): Extension, + Json(eu): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + let mut new_super_admin: Option = None; + if let Some(sa) = eu.is_super_admin { + sqlx::query_scalar!( + "UPDATE password SET super_admin = $1 WHERE email = $2", + sa, + &email_to_update + ) + .execute(&mut *tx) + .await?; + new_super_admin = Some(sa); + } + + if let Some(dv) = eu.is_devops { + sqlx::query_scalar!( + "UPDATE password SET devops = $1 WHERE email = $2", + dv, + &email_to_update + ) + .execute(&mut *tx) + .await?; + // If super_admin wasn't explicitly set, we still need to refresh tokens + if new_super_admin.is_none() { + new_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + &email_to_update + ) + .fetch_optional(&mut *tx) + .await?; + } + } + + if let Some(sa) = new_super_admin { + // Delete session tokens to force re-login with new privileges + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session'", + &email_to_update + ) + .execute(&mut *tx) + .await?; + // Update super_admin flag on non-session tokens (webhooks, API tokens, etc.) + sqlx::query!( + "UPDATE token SET super_admin = $1 WHERE email = $2 AND label != 'session'", + sa, + &email_to_update + ) + .execute(&mut *tx) + .await?; + } + + if let Some(n) = eu.name { + sqlx::query_scalar!( + "UPDATE password SET name = $1 WHERE email = $2", + n, + &email_to_update + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.update", + ActionKind::Update, + "global", + Some(&email_to_update), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("email {} updated", &email_to_update)) +} + +async fn delete_user( + authed: ApiAuthed, + Path(email_to_delete): Path, + Extension(db): Extension, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + + let usernames = sqlx::query_scalar!( + "DELETE FROM usr WHERE email = $1 RETURNING username", + &email_to_delete + ) + .fetch_all(&mut *tx) + .await?; + + for username in usernames { + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM usr_to_group WHERE usr = $1", &username) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM workspace_invite WHERE email = $1", + &email_to_delete + ) + .execute(&mut *tx) + .await?; + } + + // Remove user from all instance groups email_to_igroup + sqlx::query!( + "DELETE FROM email_to_igroup WHERE email = $1", + &email_to_delete + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.delete", + ActionKind::Delete, + "global", + Some(&email_to_delete), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("email {} deleted", &email_to_delete)) +} + +lazy_static::lazy_static! { + pub static ref NEW_USER_WEBHOOK: Option = std::env::var("NEW_USER_WEBHOOK").ok(); + +} + +/// Internal helper for updating workspace user permissions - used by both API and system operations +pub async fn update_workspace_user_internal( + w_id: &str, + username_to_update: &str, + is_admin: Option, + operator: Option, + disabled: Option, + tx: &mut Transaction<'_, Postgres>, + authed: Option<&ApiAuthed>, // None for system operations +) -> Result<()> { + if let Some(a) = is_admin { + sqlx::query_scalar!( + "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3", + a, + username_to_update, + w_id + ) + .execute(&mut **tx) + .await?; + } + if let Some(a) = operator { + sqlx::query_scalar!( + "UPDATE usr SET operator = $1 WHERE username = $2 AND workspace_id = $3", + a, + username_to_update, + w_id + ) + .execute(&mut **tx) + .await?; + } + if let Some(a) = disabled { + sqlx::query_scalar!( + "UPDATE usr SET disabled = $1 WHERE username = $2 AND workspace_id = $3", + a, + username_to_update, + w_id + ) + .execute(&mut **tx) + .await?; + } + + // Only audit if we have an authenticated user (API calls) + if let Some(auth) = authed { + audit_log( + &mut **tx, + auth, + "users.update", + ActionKind::Update, + w_id, + Some(username_to_update), + None, + ) + .await?; + } + + Ok(()) +} + +/// Internal helper for deleting workspace users - used by both API and system operations +pub async fn delete_workspace_user_internal( + w_id: &str, + username_to_delete: &str, + email_to_delete: &str, + tx: &mut Transaction<'_, Postgres>, + authed: Option<&ApiAuthed>, // None for system operations +) -> Result<()> { + sqlx::query_scalar!( + "DELETE FROM usr WHERE email = $1 AND workspace_id = $2", + email_to_delete, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "DELETE FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", + username_to_delete, + w_id + ) + .execute(&mut **tx) + .await?; + + // Only audit if we have an authenticated user (API calls) + if let Some(auth) = authed { + audit_log( + &mut **tx, + auth, + "users.delete", + ActionKind::Delete, + w_id, + Some(username_to_delete), + None, + ) + .await?; + } + + Ok(()) +} + +async fn delete_workspace_user( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_delete)): Path<(String, String)>, +) -> Result { + let mut tx = db.begin().await?; + + require_admin(authed.is_admin, &authed.username)?; + + let email_to_delete_o = sqlx::query_scalar!( + "SELECT email FROM usr where username = $1 AND workspace_id = $2", + username_to_delete, + &w_id, + ) + .fetch_optional(&db) + .await?; + + let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; + + delete_workspace_user_internal( + &w_id, + &username_to_delete, + &email_to_delete, + &mut tx, + Some(&authed), + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: email_to_delete.clone() }, + Some(format!( + "Removed user '{}' from workspace", + &email_to_delete + )), + true, + None, + ) + .await?; + + Ok(format!("username {} deleted", username_to_delete)) +} + +async fn set_login_type( + Extension(db): Extension, + Path(email): Path, + authed: ApiAuthed, + Json(et): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE password SET login_type = $1 WHERE email = $2", + et.login_type, + email + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.set_login_type", + ActionKind::Update, + "global", + Some(&email), + None, + ) + .await?; + + tx.commit().await?; + Ok(format!( + "login type of {} updated to {}", + email, et.login_type + )) +} + +#[allow(unreachable_code, unused_variables)] +async fn login( + cookies: Cookies, + Extension(db): Extension, + Extension(argon2): Extension>>, + Json(Login { email, password }): Json, +) -> Result { + #[cfg(feature = "no_auth")] + { + return Ok("no_auth".to_string()); + } + + let mut tx = db.begin().await?; + let email = email.to_lowercase(); + let audit_author = AuditAuthor { + email: email.clone(), + username: email.clone(), + username_override: None, + token_prefix: None, + }; + let email_w_h: Option<(String, String, bool)> = sqlx::query_as( + "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \ + 'password'", + ) + .bind(&email) + .fetch_optional(&mut *tx) + .await?; + + if let Some((email, hash, super_admin)) = email_w_h { + let parsed_hash = + PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; + if argon2 + .verify_password(password.as_bytes(), &parsed_hash) + .is_err() + { + audit_log( + &mut *tx, + &audit_author, + "users.login_failure", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + Err(Error::BadRequest("Invalid login".to_string())) + } else { + let token = create_session_token(&email, super_admin, &mut tx, cookies).await?; + + let audit_author = AuditAuthor { + email: email.clone(), + username: email.clone(), + username_override: None, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }; + + audit_log( + &mut *tx, + &audit_author, + "users.login", + ActionKind::Create, + "global", + Some(&truncate_token(&token)), + None, + ) + .await?; + + tx.commit().await?; + Ok(token) + } + } else { + audit_log( + &mut *tx, + &audit_author, + "users.login_failure", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + Err(Error::BadRequest("Invalid login".to_string())) + } +} + +#[derive(Deserialize)] +struct RefreshTokenQuery { + if_expiring_in_less_than_s: Option, +} +async fn refresh_token( + Extension(db): Extension, + Query(query): Query, + Tokened { token }: Tokened, + authed: ApiAuthed, + cookies: Cookies, +) -> Result { + let mut tx = db.begin().await?; + + if let Some(thresh_s) = query.if_expiring_in_less_than_s { + let not_expired = sqlx::query_scalar!("SELECT true FROM token WHERE token = $1 and expiration IS NOT NULL and expiration > now() + $2::int * '1 sec'::interval", &token, thresh_s) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(false); + if not_expired { + return Ok("token expiry is far enough".to_string()); + } + } + + let super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + &authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + + let new_token = create_session_token(&authed.email, super_admin, &mut tx, cookies).await?; + + audit_log( + &mut *tx, + &AuditAuthor { + email: authed.email.to_string(), + username: authed.email.to_string(), + username_override: None, + token_prefix: authed.token_prefix, + }, + "users.token.refresh", + ActionKind::Create, + &"global", + Some(&truncate_token(&new_token)), + None, + ) + .await?; + + tx.commit().await?; + Ok("token refreshed".to_string()) +} + +lazy_static::lazy_static! { + static ref MAX_SESSION_VALIDITY_SECONDS: i64 = std::env::var("MAX_SESSION_VALIDITY_SECONDS").ok().unwrap_or_else(|| String::new()).parse::().unwrap_or(3 * 24 * 60 * 60); + static ref INVALIDATE_OLD_SESSIONS: bool = std::env::var("INVALIDATE_OLD_SESSIONS").ok().unwrap_or_else(|| String::new()).parse::().unwrap_or(false); +} + +pub async fn create_session_token<'c>( + email: &str, + super_admin: bool, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + cookies: Cookies, +) -> Result { + let token = rd_string(32); + + if *INVALIDATE_OLD_SESSIONS { + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session'", + email + ) + .execute(&mut **tx) + .await?; + + audit_log( + &mut **tx, + &AuditAuthor { + email: email.to_string(), + username: email.to_string(), + username_override: None, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }, + "users.token.invalidate_old_sessions", + ActionKind::Delete, + &"global", + None, + None, + ) + .instrument(tracing::info_span!("token", email)) + .await?; + } + + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin) + VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5)", + token, + email, + "session", + &MAX_SESSION_VALIDITY_SECONDS.to_string(), + super_admin + ) + .execute(&mut **tx) + .await?; + + let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); + cookie.set_secure(IS_SECURE.read().await.clone()); + cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); + cookie.set_http_only(true); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + + let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); + expire += time::Duration::seconds(*MAX_SESSION_VALIDITY_SECONDS); + cookie.set_expires(expire); + cookies.add(cookie); + Ok(token) +} + +// create_token_internal is re-exported from windmill-api-auth above + +async fn create_token( + Extension(db): Extension, + authed: ApiAuthed, + Json(token_config): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?; + + tx.commit().await?; + Ok((StatusCode::CREATED, token)) +} + +async fn impersonate( + Extension(db): Extension, + authed: ApiAuthed, + Json(new_token): Json, +) -> Result<(StatusCode, String)> { + let token = rd_string(32); + require_super_admin(&db, &authed.email).await?; + + if new_token.impersonate_email.is_none() { + return Err(Error::BadRequest( + "impersonate_username is required".to_string(), + )); + } + + let impersonated = new_token.impersonate_email.unwrap(); + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + impersonated + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + let mut tx = db.begin().await?; + + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin) + VALUES ($1, $2, $3, $4, $5)", + token, + impersonated, + new_token.label, + new_token.expiration, + is_super_admin + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.impersonate", + ActionKind::Delete, + &"global", + Some(&token[0..10]), + Some([("impersonated", &format!("{impersonated}")[..])].into()), + ) + .instrument(tracing::info_span!("token", email = &impersonated)) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, token)) +} + +#[derive(Deserialize)] +struct ListTokenQuery { + exclude_ephemeral: Option, +} + +async fn list_tokens( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, + Query(query): Query, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let rows = if query.exclude_ephemeral.unwrap_or(false) { + sqlx::query_as!( + TruncatedToken, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ + last_used_at, scopes FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL) + ORDER BY created_at DESC LIMIT $2 OFFSET $3", + email, + per_page as i64, + offset as i64, + ) + .fetch_all(&db) + .await? + } else { + sqlx::query_as!( + TruncatedToken, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ + last_used_at, scopes FROM token WHERE email = $1 + ORDER BY created_at DESC LIMIT $2 OFFSET $3", + email, + per_page as i64, + offset as i64, + ) + .fetch_all(&db) + .await? + }; + Ok(Json(rows)) +} + +async fn delete_token( + Extension(db): Extension, + authed: ApiAuthed, + Path(token_prefix): Path, +) -> Result { + let mut tx = db.begin().await?; + + let tokens_deleted: Vec = sqlx::query_scalar( + "DELETE FROM token + WHERE email = $1 + AND token LIKE concat($2::text, '%') + RETURNING concat(substring(token for 10), '*****')", + ) + .bind(&authed.email) + .bind(&token_prefix) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.token.delete", + ActionKind::Delete, + &"global", + Some(&token_prefix), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!( + "deleted {} tokens {:?} with prefix {}", + tokens_deleted.len(), + tokens_deleted, + token_prefix + )) +} + +async fn leave_workspace( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + let mut tx = db.begin().await?; + sqlx::query!( + "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", + &w_id, + authed.username + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.leave_workspace", + ActionKind::Delete, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("left workspace {w_id}")) +} + +#[derive(Serialize)] +struct Runnable { + workspace: String, + endpoint_async: String, + endpoint_sync: String, + summary: String, + description: String, + schema: Option, + kind: String, + path: String, +} + +async fn get_all_runnables( + Extension(db): Extension, + authed: ApiAuthed, + Tokened { token }: Tokened, + Extension(cache): Extension>, +) -> JsonResult> { + let mut tx = db.clone().begin(&authed).await?; + let mut runnables = Vec::new(); + let workspaces = sqlx::query_scalar!( + "SELECT workspace.id as id FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \ + usr.email = $1 AND deleted = false", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + for workspace in workspaces { + let nauthed = cache + .get_authed(Some(workspace.clone()), &token) + .await + .ok_or_else(|| { + Error::BadRequest(format!("not authorized to access workspace: {workspace}")) + })?; + let mut tx = db.clone().begin(&nauthed).await?; + let flows = sqlx::query!( + "SELECT flow.workspace_id as workspace, flow.path, summary, description, flow_version.schema + FROM flow + LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.workspace_id = $1", + workspace + ) + .fetch_all(&mut *tx) + .await?; + runnables.extend( + flows + .into_iter() + .map(|f| Runnable { + workspace: f.workspace.clone(), + endpoint_async: format!("/w/{}/jobs/run/f/{}", &f.workspace, &f.path), + endpoint_sync: format!( + "/w/{}/jobs/run_wait_result/f/{}", + &f.workspace, &f.path + ), + summary: f.summary, + description: f.description, + schema: f.schema, + kind: "flow".to_string(), + path: f.path, + }) + .collect::>(), + ); + let scripts = sqlx::query!( + "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o + WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) + AND workspace_id = $1 and archived = false", workspace + ) + .fetch_all(&mut *tx) + .await?; + runnables.extend( + scripts + .into_iter() + .map(|s| Runnable { + workspace: s.workspace.clone(), + endpoint_async: format!("/w/{}/jobs/run/p/{}", &s.workspace, &s.path), + endpoint_sync: format!( + "/w/{}/jobs/run_wait_result/p/{}", + &s.workspace, &s.path + ), + summary: s.summary, + description: s.description, + schema: s.schema, + kind: "script".to_string(), + path: s.path, + }) + .collect::>(), + ); + tx.commit().await?; + } + Ok(Json(runnables)) +} + +//used by oauth +#[allow(dead_code)] +#[derive(Deserialize, Debug, Clone)] +pub struct LoginUserInfo { + pub email: Option, + pub name: Option, + pub company: Option, + pub preferred_username: Option, + pub displayName: Option, +} + +#[derive(Serialize)] +struct InstanceUsernameInfo { + username: String, + workspace_usernames: Vec, +} + +#[derive(Serialize)] +struct WorkspaceUsernameInfo { + workspace_id: String, + username: String, +} +async fn get_instance_username_info( + ApiAuthed { email, .. }: ApiAuthed, + Path(user_email): Path, + Extension(db): Extension, +) -> JsonResult { + require_super_admin(&db, &email).await?; + let mut tx = db.begin().await?; + let instance_username = match sqlx::query_scalar!( + "SELECT username FROM password WHERE email = $1", + &user_email + ) + .fetch_one(&mut *tx) + .await? + { + Some(username) => username, + None => generate_instance_wide_unique_username(&mut tx, &user_email).await?, + }; + + let workspace_usernames = sqlx::query_as!( + WorkspaceUsernameInfo, + "SELECT workspace_id, username FROM usr WHERE email = $1", + &user_email + ) + .fetch_all(&mut *tx) + .await?; + + Ok(Json(InstanceUsernameInfo { + username: instance_username, + workspace_usernames: workspace_usernames, + })) +} + +async fn username_to_email( + Path((w_id, username)): Path<(String, String)>, + Extension(db): Extension, +) -> Result { + let email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + &username, + &w_id + ) + .fetch_optional(&db) + .await?; + + let email = not_found_if_none(email, "user", username)?; + + Ok(email) +} + +#[cfg(feature = "enterprise")] +#[derive(Serialize, Deserialize)] +struct ExportedGlobalUser { + email: String, + password_hash: Option, + login_type: String, + super_admin: bool, + verified: bool, + name: Option, + company: Option, + first_time_user: bool, + username: Option, +} + +#[cfg(feature = "enterprise")] +async fn export_global_users( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + let users = sqlx::query_as!( + ExportedGlobalUser, + "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password" + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.export_export", + ActionKind::Execute, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(users)) +} + +#[cfg(not(feature = "enterprise"))] +async fn export_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[cfg(feature = "enterprise")] +async fn overwrite_global_users( + Extension(db): Extension, + authed: ApiAuthed, + Json(users): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password") + .execute(&mut *tx) + .await?; + for user in users { + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + user.email, + user.password_hash, + user.login_type, + user.super_admin, + user.verified, + user.name, + user.company, + user.first_time_user, + user.username + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.import_global", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + tx.commit().await?; + Ok("loaded global users".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +async fn overwrite_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +// NOTE: rename_user, update_username_in_workpsace, and RenameUser +// are in windmill-api (depend on EE secret_backend_ext) + +#[derive(Deserialize)] +pub struct RequestPasswordReset { + pub email: String, +} + +#[derive(Deserialize)] +pub struct ResetPassword { + pub token: String, + pub new_password: String, +} + +#[derive(Serialize)] +pub struct PasswordResetResponse { + pub message: String, +} + +// Password Reset Functions + +/// Check if SMTP is configured +async fn is_smtp_configured(Extension(db): Extension) -> JsonResult { + let smtp = windmill_common::server::load_smtp_config(&db).await?; + Ok(Json(smtp.is_some())) +} + +/// Request a password reset email +async fn request_password_reset( + Extension(db): Extension, + Json(req): Json, +) -> Result> { + let email = req.email.to_lowercase(); + + // Check if SMTP is configured + let smtp = windmill_common::server::load_smtp_config(&db).await?; + let smtp = smtp.ok_or_else(|| { + Error::BadRequest("SMTP is not configured. Password reset is not available.".to_string()) + })?; + + // Check if user exists with password login type + let user_exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1 AND login_type = 'password')", + &email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + // Always return success to prevent email enumeration + // But only send email if user exists + if user_exists { + // Generate a secure token + let token = rd_string(32); + + // Delete any existing tokens for this email + sqlx::query!("DELETE FROM magic_link WHERE email = $1", &email) + .execute(&db) + .await?; + + // Insert new token with 1 hour expiration + sqlx::query!( + "INSERT INTO magic_link (email, token, expiration) VALUES ($1, $2, NOW() + INTERVAL '1 hour')", + &email, + &token + ) + .execute(&db) + .await?; + + // Get the base URL for the reset link + let base_url = BASE_URL.read().await.clone(); + let base_url = if base_url.is_empty() { + std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()) + } else { + base_url + }; + + let reset_link = format!("{}/user/reset-password?token={}", base_url, token); + + // Send the email + let subject = "Windmill Password Reset"; + let content = format!( + "You have requested a password reset for your Windmill account.\n\n\ + Click the link below to reset your password:\n\ + {}\n\n\ + This link will expire in 1 hour.\n\n\ + If you did not request this password reset, you can safely ignore this email.", + reset_link + ); + + // Send the email - don't fail the request if email fails + if let Err(e) = windmill_common::email_oss::send_email_plain_text( + subject, + &content, + vec![email.clone()], + smtp, + Some(Duration::from_secs(10)), + ) + .await + { + tracing::error!("Failed to send password reset email to {}: {:?}", email, e); + } + } + + // Always return success to prevent email enumeration + Ok(Json(PasswordResetResponse { + message: "If an account with that email exists, a password reset link has been sent." + .to_string(), + })) +} + +// NOTE: reset_password is in windmill-api (depends on users_oss::hash_password EE dispatch) diff --git a/backend/windmill-api-users/src/users_oss.rs b/backend/windmill-api-users/src/users_oss.rs new file mode 100644 index 0000000000..afbb729b79 --- /dev/null +++ b/backend/windmill-api-users/src/users_oss.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; + +use windmill_api_auth::ApiAuthed; + +use crate::users::{EditPassword, NewUser}; + +use windmill_common::webhook::WebhookShared; +use windmill_common::DB; + +use argon2::Argon2; + +use axum::{extract::Extension, Json}; + +use http::StatusCode; + +use serde::Deserialize; + +use windmill_common::error::{Error, Result}; + +pub async fn create_user( + _authed: ApiAuthed, + _db: DB, + _webhook: WebhookShared, + _argon2: Arc>, + mut _nu: NewUser, +) -> Result<(StatusCode, String)> { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +pub async fn set_password( + _db: DB, + _argon2: Arc>, + _authed: ApiAuthed, + _user_email: &str, + _ep: EditPassword, +) -> Result { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +pub fn hash_password(_argon2: Arc>, _password: String) -> Result { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +#[derive(Deserialize, Debug)] +#[allow(dead_code)] +pub struct OnboardingData { + pub touch_point: String, + pub use_case: String, +} + +pub async fn submit_onboarding_data( + _authed: ApiAuthed, + Extension(_db): Extension, + Json(_data): Json, +) -> Result { + Err(Error::internal_err( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml new file mode 100644 index 0000000000..fd07e893be --- /dev/null +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "windmill-api-workspaces" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_workspaces" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +private = ["windmill-common/private"] +cloud = ["windmill-common/cloud"] +no_auth = ["windmill-api-auth/no_auth"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-api-users.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +windmill-queue.workspace = true +windmill-api-jobs.workspace = true +windmill-dep-map.workspace = true + +axum.workspace = true +chrono.workspace = true +hex.workspace = true +http.workspace = true +hyper.workspace = true +lazy_static.workspace = true +regex.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +uuid.workspace = true diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs new file mode 100644 index 0000000000..7c12ebd3a0 --- /dev/null +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -0,0 +1,6 @@ +pub mod workspaces; +pub mod workspaces_extra; +pub mod workspaces_oss; + +#[cfg(feature = "private")] +pub mod workspaces_ee; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs new file mode 100644 index 0000000000..b7fb588b5c --- /dev/null +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -0,0 +1,5040 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_api_auth::{require_super_admin, ApiAuthed}; +use windmill_api_users::users::WorkspaceInvite; +use windmill_common::email_oss::send_email_if_possible; +use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; +use windmill_common::webhook::WebhookShared; +use windmill_common::{BASE_URL, DB}; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::Utc; + +use regex::Regex; + +use hex; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; +use uuid::Uuid; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::db::UserDB; +use windmill_common::s3_helpers::LargeFileStorage; +use windmill_common::users::username_to_permissioned_as; +use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; +use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; +#[cfg(feature = "enterprise")] +use windmill_common::workspaces::GitRepositorySettings; +#[cfg(feature = "enterprise")] +use windmill_common::workspaces::WorkspaceDeploymentUISettings; +use windmill_common::workspaces::{ + get_datatable_resource_from_db_unchecked, DataTable, DataTableCatalogResourceType, + WorkspaceGitSyncSettings, +}; +use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; +use windmill_common::PgDatabase; +use windmill_common::{ + error::{Error, JsonResult, Result}, + global_settings::AUTOMATE_USERNAME_CREATION_SETTING, + oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, + utils::{paginate, rd_string, require_admin, Pagination}, +}; +use windmill_dep_map::scoped_dependency_map::{ + DependencyDependent, DependencyMap, ScopedDependencyMap, +}; +use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject}; + +use hyper::StatusCode; +use serde::{Deserialize, Serialize}; +use sqlx::{FromRow, Postgres, Transaction}; +use windmill_common::oauth2::InstanceEvent; +use windmill_common::utils::not_found_if_none; + +lazy_static::lazy_static! { + static ref WORKSPACE_KEY_REGEXP: Regex = Regex::new("^[a-zA-Z0-9]{64}$").unwrap(); +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/get_as_superadmin", get(get_workspace_as_superadmin)) + .route("/list_pending_invites", get(list_pending_invites)) + .route("/update", post(edit_workspace)) + .route("/archive", post(archive_workspace)) + .route("/invite_user", post(invite_user)) + .route("/add_user", post(add_user)) + .route("/delete_invite", post(delete_invite)) + .route("/rebuild_dependency_map", post(rebuild_dependency_map)) + .route("/get_dependency_map", get(get_dependency_map)) + .route("/get_dependents/*imported_path", get(get_dependents)) + .route("/get_dependents_amounts", post(get_dependents_amounts)) + .route("/get_settings", get(get_settings)) + .route("/get_deploy_to", get(get_deploy_to)) + .route("/edit_slack_command", post(edit_slack_command)) + .route( + "/run_slack_message_test_job", + post(run_slack_message_test_job), + ) + .route("/slack_oauth_config", get(get_slack_oauth_config)) + .route("/slack_oauth_config", post(set_slack_oauth_config)) + .route("/slack_oauth_config", delete(delete_slack_oauth_config)) + .route("/edit_webhook", post(edit_webhook)) + .route("/edit_auto_invite", post(edit_auto_invite)) + .route("/edit_instance_groups", post(edit_instance_groups)) + .route("/edit_deploy_to", post(edit_deploy_to)) + .route( + "/get_secondary_storage_names", + get(get_secondary_storage_names), + ) + .route("/is_premium", get(is_premium)) + .route("/edit_error_handler", post(edit_error_handler)) + .route("/edit_success_handler", post(edit_success_handler)) + .route( + "/edit_large_file_storage_config", + post(edit_large_file_storage_config), + ) + .route("/edit_ducklake_config", post(edit_ducklake_config)) + .route("/list_ducklakes", get(list_ducklakes)) + .route("/list_datatables", get(list_datatables)) + .route("/list_datatable_schemas", get(list_datatable_schemas)) + .route("/edit_datatable_config", post(edit_datatable_config)) + .route("/edit_git_sync_config", post(edit_git_sync_config)) + .route("/edit_git_sync_repository", post(edit_git_sync_repository)) + .route( + "/delete_git_sync_repository", + delete(delete_git_sync_repository), + ) + .route("/edit_deploy_ui_config", post(edit_deploy_ui_config)) + .route("/edit_default_app", post(edit_default_app)) + .route("/default_app", get(get_default_app)) + .route( + "/default_scripts", + post(edit_default_scripts).get(get_default_scripts), + ) + .route("/set_environment_variable", post(set_environment_variable)) + .route( + "/encryption_key", + get(get_encryption_key).post(set_encryption_key), + ) + .route("/leave", post(leave_workspace)) + .route("/get_workspace_name", get(get_workspace_name)) + .route("/create_fork", post(create_workspace_fork)) + .route("/change_workspace_name", post(change_workspace_name)) + .route("/change_workspace_color", post(change_workspace_color)) + .route( + "/change_workspace_id", + post(crate::workspaces_extra::change_workspace_id), + ) + .route("/usage", get(get_usage)) + .route("/used_triggers", get(get_used_triggers)) + .route("/public_app_rate_limit", post(edit_public_app_rate_limit)) + .route("/operator_settings", post(update_operator_settings)) + .route( + "/create_workspace_fork_branch", + post(create_workspace_fork_branch), + ) + .route( + "/reset_diff_tally/:fork_workspace_id", + post(reset_workspace_diffs), + ) + .route("/compare/:target_workspace_id", get(compare_workspaces)) +} +pub fn global_service() -> Router { + Router::new() + .route("/list_as_superadmin", get(list_workspaces_as_super_admin)) + .route("/list", get(list_workspaces)) + .route("/users", get(user_workspaces)) + .route("/create", post(create_workspace)) + .route("/create_fork", post(deprecated_create_workspace_fork)) + .route("/exists", post(exists_workspace)) + .route("/exists_username", post(exists_username)) + .route("/allowed_domain_auto_invite", get(is_allowed_auto_domain)) + .route("/unarchive/:workspace", post(unarchive_workspace)) + .route( + "/delete/:workspace", + delete(crate::workspaces_extra::delete_workspace), + ) + .route( + "/create_workspace_require_superadmin", + get(create_workspace_require_superadmin), + ) +} + +#[derive(FromRow, Serialize)] +struct Workspace { + id: String, + name: String, + owner: String, + deleted: bool, + premium: bool, + color: Option, + parent_workspace_id: Option, +} + +#[derive(FromRow, Serialize, Debug)] +pub struct WorkspaceSettings { + pub workspace_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub teams_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub teams_team_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub teams_team_guid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_command_script: Option, + pub teams_command_script: Option, + pub slack_email: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_oauth_client_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub slack_oauth_client_secret: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub customer_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub plan: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub webhook: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub deploy_to: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ai_config: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub large_file_storage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ducklake: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub datatable: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub git_sync: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub deploy_ui: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_app: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_scripts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mute_critical_alerts: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub git_app_installations: Option, + // Grouped config fields + #[serde(skip_serializing_if = "Option::is_none")] + pub auto_invite: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub success_handler: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_app_execution_limit_per_minute: Option, +} + +/// #[derive(sqlx::Type, Serialize, Deserialize, Debug)] +// #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] +// pub enum WorkspaceKeyKind { +// Cloud, +// } + +#[derive(Deserialize)] +struct EditCommandScript { + slack_command_script: Option, +} + +#[derive(Deserialize)] +struct RunSlackMessageTestJobRequest { + hub_script_path: String, + channel: String, + test_msg: String, +} + +#[derive(Serialize)] +struct RunSlackMessageTestJobResponse { + job_uuid: String, +} + +#[cfg(feature = "enterprise")] +#[derive(Deserialize)] +struct EditDeployTo { + deploy_to: Option, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct EditAutoInvite { + pub operator: Option, + pub invite_all: Option, + pub auto_add: Option, +} + +#[derive(Deserialize)] +struct EditWebhook { + webhook: Option, +} + +#[derive(Deserialize, Serialize, Debug)] +struct LargeFileStorageWithSecondary { + #[serde(flatten)] + large_file_storage: LargeFileStorage, + #[serde(default)] + secondary_storage: HashMap, +} +#[derive(Deserialize, Debug)] +struct EditLargeFileStorageConfig { + large_file_storage: Option, +} + +#[derive(Deserialize, Debug)] +struct EditDucklakeConfig { + settings: DucklakeSettings, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct DucklakeSettings { + pub ducklakes: HashMap, +} + +#[derive(Deserialize, Debug)] +struct EditDataTableConfig { + settings: DataTableSettings, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct DataTableSettings { + pub datatables: HashMap, +} + +#[derive(Deserialize)] +struct CreateWorkspace { + id: String, + name: String, + username: Option, + color: Option, +} + +#[derive(Deserialize)] +struct CreateWorkspaceFork { + id: String, + name: String, + color: Option, +} + +#[derive(Deserialize)] +struct EditWorkspace { + name: String, + owner: String, +} + +#[derive(Serialize)] +struct WorkspaceList { + pub email: String, + pub workspaces: Vec, +} + +#[derive(Serialize)] +struct UserWorkspace { + pub id: String, + pub name: String, + pub username: String, + pub color: Option, + pub operator_settings: Option>, + pub parent_workspace_id: Option, + pub disabled: bool, +} + +#[derive(Deserialize)] +struct WorkspaceId { + pub id: String, +} + +#[derive(Deserialize)] +struct ValidateUsername { + pub id: String, + pub username: String, +} + +#[derive(Deserialize)] +pub struct NewWorkspaceInvite { + pub email: String, + pub is_admin: bool, + pub operator: bool, +} + +#[derive(Deserialize)] +pub struct NewWorkspaceUser { + pub email: String, + pub username: Option, + pub is_admin: bool, + pub operator: bool, +} + +// New format for error handler (grouped) +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EditErrorHandlerNew { + pub path: Option, + pub extra_args: Option, + #[serde(default)] + pub muted_on_cancel: bool, + #[serde(default)] + pub muted_on_user_path: bool, +} + +// Legacy format for error handler (flat fields from old CLI) +#[derive(Deserialize)] +pub struct EditErrorHandlerLegacy { + pub error_handler: Option, + pub error_handler_extra_args: Option, + #[serde(default)] + pub error_handler_muted_on_cancel: bool, +} + +// Accepts both old and new formats +#[derive(Deserialize)] +#[serde(untagged)] +pub enum EditErrorHandler { + New(EditErrorHandlerNew), + Legacy(EditErrorHandlerLegacy), +} + +impl EditErrorHandler { + pub fn into_normalized(self) -> EditErrorHandlerNew { + match self { + EditErrorHandler::New(new) => new, + EditErrorHandler::Legacy(legacy) => EditErrorHandlerNew { + path: legacy.error_handler, + extra_args: legacy.error_handler_extra_args, + muted_on_cancel: legacy.error_handler_muted_on_cancel, + muted_on_user_path: false, // Old format doesn't have this field + }, + } + } +} + +// New format for success handler (grouped) +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EditSuccessHandlerNew { + pub path: Option, + pub extra_args: Option, +} + +// Legacy format for success handler (flat fields from old CLI) +#[derive(Deserialize)] +pub struct EditSuccessHandlerLegacy { + pub success_handler: Option, + pub success_handler_extra_args: Option, +} + +// Accepts both old and new formats +#[derive(Deserialize)] +#[serde(untagged)] +pub enum EditSuccessHandler { + New(EditSuccessHandlerNew), + Legacy(EditSuccessHandlerLegacy), +} + +impl EditSuccessHandler { + pub fn into_normalized(self) -> EditSuccessHandlerNew { + match self { + EditSuccessHandler::New(new) => new, + EditSuccessHandler::Legacy(legacy) => EditSuccessHandlerNew { + path: legacy.success_handler, + extra_args: legacy.success_handler_extra_args, + }, + } + } +} + +lazy_static::lazy_static! { + pub static ref EMAIL_REGEXP: Regex = Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$").unwrap(); +} + +async fn list_pending_invites( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as!( + WorkspaceInvite, + "SELECT + workspace_invite.workspace_id, + workspace_invite.email, + workspace_invite.is_admin, + workspace_invite.operator, + workspace.parent_workspace_id + FROM workspace_invite JOIN workspace ON workspace_invite.workspace_id = workspace.id + WHERE workspace_id = $1", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn is_premium( + authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + #[cfg(feature = "cloud")] + let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) + .await? + .premium; + #[cfg(not(feature = "cloud"))] + let premium = false; + Ok(Json(premium)) +} + +async fn exists_workspace( + authed: ApiAuthed, + Extension(user_db): Extension, + Json(WorkspaceId { id }): Json, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)", + id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + tx.commit().await?; + Ok(Json(exists)) +} + +async fn list_workspaces( + authed: ApiAuthed, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let workspaces = sqlx::query_as!( + Workspace, + "SELECT workspace.id, workspace.name, workspace.owner, workspace.deleted, workspace.premium, workspace_settings.color, workspace.parent_workspace_id + FROM workspace + LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id + JOIN usr ON usr.workspace_id = workspace.id + WHERE usr.email = $1 AND workspace.deleted = false", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(workspaces)) +} + +async fn get_settings( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let settings = sqlx::query_as!( + WorkspaceSettings, + r#" + SELECT + workspace_id, + slack_team_id, + teams_team_id, + teams_team_name, + teams_team_guid, + slack_name, + slack_command_script, + teams_command_script, + slack_email, + slack_oauth_client_id, + slack_oauth_client_secret, + customer_id, + plan, + webhook, + deploy_to, + ai_config, + large_file_storage, + datatable, + ducklake, + git_sync, + deploy_ui, + default_app, + default_scripts, + mute_critical_alerts, + color, + operator_settings, + git_app_installations, + auto_invite, + error_handler, + success_handler, + public_app_execution_limit_per_minute + FROM + workspace_settings + WHERE + workspace_id = $1 + "#, + &w_id + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("getting settings: {e:#}")))?; + + tx.commit().await?; + + let settings = not_found_if_none(settings, "workspace settings", &w_id)?; + Ok(Json(settings)) +} + +#[derive(Serialize)] +struct DeployTo { + deploy_to: Option, +} +async fn get_deploy_to( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let settings = sqlx::query_as!( + DeployTo, + "SELECT deploy_to FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("getting deploy_to: {e:#}")))?; + + tx.commit().await?; + Ok(Json(settings)) +} + +async fn edit_slack_command( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(es): Json, +) -> Result { + require_admin(is_admin, &username)?; + let mut tx = db.begin().await?; + + if es.slack_command_script.is_some() { + let exists_slack_command_with_team_id = sqlx::query_scalar!( + r#" + SELECT EXISTS (SELECT 1 + FROM workspace_settings + WHERE workspace_id <> $1 + AND slack_command_script IS NOT NULL + AND slack_team_id IS NOT NULL + AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1)) + "#, + &w_id + ) + .fetch_one(&mut *tx) + .await?.unwrap_or(false); + + if exists_slack_command_with_team_id { + return Err(Error::BadRequest( + "A workspace connected to the same slack team already has a command script. Please remove it first." + .to_string(), + )); + } + } + + sqlx::query!( + "UPDATE workspace_settings SET slack_command_script = $1 WHERE workspace_id = $2", + es.slack_command_script, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_command_script", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some( + [( + "script", + es.slack_command_script + .unwrap_or("NO_SCRIPT".to_string()) + .as_str(), + )] + .into(), + ), + ) + .await?; + tx.commit().await?; + + Ok(format!("Edit command script {}", &w_id)) +} + +async fn run_slack_message_test_job( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + let mut fake_result = HashMap::new(); + fake_result.insert("error".to_string(), to_raw_value(&req.test_msg)); + fake_result.insert("success_result".to_string(), to_raw_value(&req.test_msg)); + + let mut extra_args = HashMap::new(); + extra_args.insert("channel".to_string(), to_raw_value(&req.channel)); + extra_args.insert( + "slack".to_string(), + to_raw_value(&format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}")), + ); + + let uuid = windmill_queue::push_error_handler( + &db, + Uuid::parse_str("00000000-0000-0000-0000-000000000000")?, + None, + Some("slack_message_test".to_string()), + false, + w_id.as_str(), + &format!("script/{}", req.hub_script_path.as_str()), + sqlx::types::Json(&fake_result), + None, + Some(Utc::now()), + Some(sqlx::types::Json(to_raw_value(&extra_args))), + authed.email.as_str(), + false, + false, + None, // Note: we could mark it as high priority to return result quickly to the user + ) + .await?; + + Ok(Json(RunSlackMessageTestJobResponse { + job_uuid: uuid.to_string(), + })) +} + +#[derive(Deserialize)] +struct SetSlackOAuthConfigRequest { + slack_oauth_client_id: String, + slack_oauth_client_secret: String, +} + +#[derive(Serialize)] +struct GetSlackOAuthConfigResponse { + slack_oauth_client_id: Option, + slack_oauth_client_secret: Option, +} + +async fn get_slack_oauth_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + + let settings = sqlx::query!( + "SELECT slack_oauth_client_id, slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + + // Mask the secret if it exists + let masked_secret = settings + .slack_oauth_client_secret + .map(|_| "***".to_string()); + + Ok(Json(GetSlackOAuthConfigResponse { + slack_oauth_client_id: settings.slack_oauth_client_id, + slack_oauth_client_secret: masked_secret, + })) +} + +async fn set_slack_oauth_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + if req.slack_oauth_client_id.is_empty() || req.slack_oauth_client_secret.is_empty() { + return Err(Error::BadRequest( + "Both client ID and client secret are required".to_string(), + )); + } + + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE workspace_settings + SET slack_oauth_client_id = $1, slack_oauth_client_secret = $2 + WHERE workspace_id = $3", + &req.slack_oauth_client_id, + &req.slack_oauth_client_secret, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.set_slack_oauth_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("client_id", req.slack_oauth_client_id.as_str())].into()), + ) + .await?; + + tx.commit().await?; + + Ok(format!("Slack OAuth config set for workspace {}", &w_id)) +} + +async fn delete_slack_oauth_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE workspace_settings + SET slack_oauth_client_id = NULL, slack_oauth_client_secret = NULL + WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.delete_slack_oauth_config", + ActionKind::Delete, + &w_id, + Some(&authed.email), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "Slack OAuth config deleted for workspace {}", + &w_id + )) +} + +#[derive(Deserialize)] +struct GetSecondaryStorageNamesQuery { + #[serde(default)] + include_default: bool, +} + +async fn get_secondary_storage_names( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut result: Vec = sqlx::query_scalar!( + "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') AS \"secondary_storage_name!: _\" + FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&db) + .await?; + + // If include_default is true, check if primary storage is set and add "_default_" + if query.include_default { + let has_primary_storage: Option = sqlx::query_scalar!( + "SELECT (large_file_storage IS NOT NULL + AND large_file_storage != 'null'::jsonb + AND jsonb_typeof(large_file_storage) = 'object') AS \"has_primary!\" + FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&db) + .await?; + + if has_primary_storage.unwrap_or(false) { + result.insert(0, "_default_".to_string()); + } + } + + Ok(Json(result)) +} + +#[cfg(feature = "enterprise")] +async fn edit_deploy_to( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(es): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", + es.deploy_to, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_deploy_to", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some( + [( + "script", + es.deploy_to.unwrap_or("NO_DEPLOY_TO".to_string()).as_str(), + )] + .into(), + ), + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Settings { setting_type: "deploy_to".to_string() }, + None, + false, + None, + ) + .await?; + + Ok(format!("Edit deploy to for {}", &w_id)) +} + +#[cfg(not(feature = "enterprise"))] +async fn edit_deploy_to() -> Result { + return Err(Error::BadRequest( + "Deploy to is only available on enterprise".to_string(), + )); +} + +pub const BANNED_DOMAINS: &str = include_str!("../../windmill-api/banned_domains.txt"); +pub const WM_FORK_PREFIX: &str = "wm-fork-"; +pub const MAX_CUSTOM_PROMPT_LENGTH: usize = 5000; + +async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult { + let domain = email.split('@').last().unwrap(); + return Ok(Json(!BANNED_DOMAINS.contains(domain))); +} + +async fn edit_auto_invite( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(ea): Json, +) -> Result { + crate::workspaces_oss::edit_auto_invite(authed, db, w_id, ea).await +} + +#[cfg(feature = "private")] +async fn edit_instance_groups( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(config): Json, +) -> Result { + crate::workspaces_ee::edit_instance_groups(authed, db, w_id, config).await +} + +#[cfg(not(feature = "private"))] +async fn edit_instance_groups( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, + Json(_config): Json, +) -> Result { + Err(Error::BadRequest( + "Instance groups are only available on Windmill Enterprise Edition".to_string(), + )) +} + +async fn edit_webhook( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(ew): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + if let Some(webhook) = &ew.webhook { + sqlx::query!( + "UPDATE workspace_settings SET webhook = $1 WHERE workspace_id = $2", + webhook, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET webhook = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "workspaces.edit_webhook", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("webhook", &format!("{:?}", ew.webhook)[..])].into()), + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Settings { setting_type: "webhook".to_string() }, + None, + false, + None, + ) + .await?; + + Ok(format!("Edit webhook for workspace {}", &w_id)) +} + +async fn edit_large_file_storage_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.large_file_storage); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_large_file_storage_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("large_file_storage", args_for_audit.as_str())].into()), + ) + .await?; + + if let Some(lfs_config) = new_config.large_file_storage { + let serialized_lfs_config = + serde_json::to_value::(lfs_config) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + serialized_lfs_config, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + // Trigger git sync for large file storage changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { + setting_type: "large_file_storage".to_string(), + }, + Some("Large file storage configuration updated".to_string()), + false, + None, + ) + .await?; + + Ok(format!( + "Edit large file storage config for workspace {}", + &w_id + )) +} + +async fn list_ducklakes( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let ducklakes = sqlx::query_scalar!( + r#" + SELECT jsonb_object_keys(ws.ducklake->'ducklakes') AS ducklake_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_all(&db) + .await? + .into_iter() + .filter_map(|s| s) + .collect(); + + Ok(Json(ducklakes)) +} + +async fn list_datatables( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let datatables = sqlx::query_scalar!( + r#" + SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_all(&db) + .await? + .into_iter() + .filter_map(|s| s) + .collect(); + + Ok(Json(datatables)) +} + +/// Compact column representation: "type" or "type?" for nullable, with "=default" suffix if has default +type CompactColumn = String; + +/// Columns mapped by name to their compact type +type ColumnMap = HashMap; + +/// Tables mapped by name to their columns +type TableMap = HashMap; + +/// Schemas mapped by name to their tables +type SchemaMap = HashMap; + +#[derive(Serialize, Debug)] +struct DataTableSchema { + datatable_name: String, + /// Hierarchical schema: schema_name -> table_name -> column_name -> "type[?][=default]" + schemas: SchemaMap, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +async fn list_datatable_schemas( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + // Get all datatable names for this workspace + let datatable_names: Vec = sqlx::query_scalar!( + r#" + SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_all(&db) + .await? + .into_iter() + .filter_map(|s| s) + .collect(); + + let mut results = Vec::new(); + + for datatable_name in datatable_names { + let schema = match get_datatable_schema(&db, &w_id, &datatable_name).await { + Ok(schemas) => DataTableSchema { datatable_name, schemas, error: None }, + Err(e) => DataTableSchema { + datatable_name, + schemas: HashMap::new(), + error: Some(e.to_string()), + }, + }; + results.push(schema); + } + + Ok(Json(results)) +} + +async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Result { + // Get the datatable resource (connection credentials) + let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + + // Parse the resource as PgDatabase + let pg_db: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + + // Connect to the datatable database + let (client, connection) = pg_db.connect().await?; + + // Spawn the connection handler + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!("Datatable connection error: {}", e); + } + }); + + // First, get all non-system schemas (including empty ones) + let schema_rows = client + .query( + r#" + SELECT nspname::text AS schema_name + FROM pg_namespace + WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') + AND nspname NOT LIKE 'pg_%' + ORDER BY nspname + "#, + &[], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to query schemas: {}", e)))?; + + // Build hierarchical structure: schema -> table -> column -> compact_type + let mut schema_map: SchemaMap = HashMap::new(); + + // Collect schema names and initialize map + let schema_names: Vec = schema_rows + .iter() + .map(|row| { + let name: String = row.get(0); + schema_map.entry(name.clone()).or_default(); + name + }) + .collect(); + + // Query column information only for the schemas we found + let rows = client + .query( + r#" + SELECT + table_schema::text, + table_name::text, + column_name::text, + udt_name::text, + is_nullable::text, + column_default::text + FROM information_schema.columns + WHERE table_schema = ANY($1) + AND table_name IS NOT NULL + ORDER BY table_schema, table_name, ordinal_position + "#, + &[&schema_names], + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to query columns: {}", e)))?; + + for row in rows { + let table_schema: String = row.get(0); + let table_name: String = row.get(1); + let column_name: String = row.get(2); + let udt_name: String = row.get(3); + let is_nullable: String = row.get(4); + let column_default: Option = row.get(5); + + // Build compact type representation: "type[?][=default]" + let mut compact = udt_name; + if is_nullable == "YES" { + compact.push('?'); + } + if let Some(default) = column_default { + // Truncate long defaults for compactness + let short_default = if default.len() > 30 { + format!("{}...", &default[..27]) + } else { + default + }; + compact.push('='); + compact.push_str(&short_default); + } + + schema_map + .entry(table_schema) + .or_default() + .entry(table_name) + .or_default() + .insert(column_name, compact); + } + + Ok(schema_map) +} + +async fn edit_ducklake_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, email, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + let is_superadmin = require_super_admin(&db, &email).await.is_ok(); + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.settings); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_ducklake_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("ducklake", args_for_audit.as_str())].into()), + ) + .await?; + + // Check that non-superadmins are not abusing Instance databases + if !is_superadmin { + let old_ducklakes = sqlx::query_scalar!( + r#" + SELECT ws.ducklake->'ducklakes' AS ducklake_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null); + let old_ducklakes: HashMap = + serde_json::from_value(old_ducklakes).unwrap_or_default(); + for (name, dl) in new_config.settings.ducklakes.iter() { + if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { + let old_dl = old_ducklakes.get(name); + if old_dl.is_none() + || old_dl.unwrap().catalog.resource_type + != DucklakeCatalogResourceType::Instance + || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path + { + return Err(Error::BadRequest( + "Only superadmins can create or modify ducklakes with Instance databases" + .to_string(), + )); + } + } + } + } + + let config: serde_json::Value = serde_json::to_value(new_config.settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET ducklake = $1 WHERE workspace_id = $2", + config, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(format!("Edit ducklake config for workspace {}", &w_id)) +} + +async fn edit_datatable_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, email, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + let is_superadmin = require_super_admin(&db, &email).await.is_ok(); + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.settings); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_datatable_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("datatable", args_for_audit.as_str())].into()), + ) + .await?; + + // Check that non-superadmins are not abusing Instance databases + if !is_superadmin { + let old_datatables = sqlx::query_scalar!( + r#" + SELECT ws.datatable->'datatables' AS datatable_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(serde_json::Value::Null); + let old_datatables: HashMap = + serde_json::from_value(old_datatables).unwrap_or_default(); + for (name, dt) in new_config.settings.datatables.iter() { + if dt.database.resource_type == DataTableCatalogResourceType::Instance { + let old_dt = old_datatables.get(name); + if old_dt.is_none() + || old_dt.unwrap().database.resource_type + != DataTableCatalogResourceType::Instance + || old_dt.unwrap().database.resource_path != dt.database.resource_path + { + return Err(Error::BadRequest( + "Only superadmins can create or modify data tables with Instance databases" + .to_string(), + )); + } + } + } + } + + let config: serde_json::Value = serde_json::to_value(new_config.settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", + config, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(format!("Edit datatable config for workspace {}", &w_id)) +} + +#[derive(Deserialize)] +pub struct EditGitSyncConfig { + #[cfg(feature = "enterprise")] + pub git_sync_settings: Option, +} + +#[cfg(feature = "enterprise")] +#[derive(Deserialize, Debug)] +pub struct EditGitSyncRepository { + pub git_repo_resource_path: String, + pub repository: GitRepositorySettings, +} + +#[cfg(feature = "enterprise")] +#[derive(Deserialize, Debug)] +pub struct DeleteGitSyncRepositoryRequest { + pub git_repo_resource_path: String, +} + +#[cfg(feature = "enterprise")] +fn validate_git_repo_resource_path(path: &str) -> Result<()> { + // Resource paths should follow the pattern: $res:f// or $res:u// + if path.is_empty() { + return Err(Error::BadRequest( + "Resource path cannot be empty".to_string(), + )); + } + + // Must start with $res: prefix + if !path.starts_with("$res:") { + return Err(Error::BadRequest( + "Resource path must start with '$res:'".to_string(), + )); + } + + // Extract the actual path after $res: + let actual_path = &path[5..]; // Remove "$res:" prefix + + // Basic validation: must start with f/ or u/ and contain at least one slash + if !actual_path.starts_with("f/") && !actual_path.starts_with("u/") { + return Err(Error::BadRequest( + "Resource path must start with '$res:f/' or '$res:u/'".to_string(), + )); + } + + // Must have at least 3 parts (type, folder/user, name) + let parts: Vec<&str> = actual_path.split('/').collect(); + if parts.len() < 3 || parts.iter().any(|part| part.is_empty()) { + return Err(Error::BadRequest( + "Invalid resource path format".to_string(), + )); + } + + // Resource name validation (last part) + let resource_name = parts.last().unwrap(); + if !resource_name + .chars() + .all(|c| c.is_alphanumeric() || c == '_' || c == '-') + { + return Err(Error::BadRequest( + "Resource name can only contain alphanumeric characters, underscores, and hyphens" + .to_string(), + )); + } + + Ok(()) +} + +#[cfg(feature = "enterprise")] +fn cleanup_legacy_git_sync_settings_in_memory( + git_sync_settings: &mut windmill_common::workspaces::WorkspaceGitSyncSettings, + workspace_id: &str, +) { + // Check if all repositories are in new format (have settings field) + let all_repos_migrated = git_sync_settings + .repositories + .iter() + .all(|repo| repo.settings.is_some()); + + // If all repos are migrated and we still have legacy workspace-level settings + if all_repos_migrated + && (git_sync_settings.include_path.is_some() || git_sync_settings.include_type.is_some()) + { + tracing::info!( + workspace_id = workspace_id, + "All git sync repositories migrated to new format, cleaning up legacy workspace-level settings" + ); + + // Remove workspace-level legacy fields + git_sync_settings.include_path = None; + git_sync_settings.include_type = None; + } +} + +#[cfg(not(feature = "enterprise"))] +async fn edit_git_sync_config( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, + Json(_new_config): Json, +) -> Result { + return Err(Error::BadRequest( + "Git sync is only available on Windmill Enterprise Edition".to_string(), + )); +} + +#[cfg(feature = "enterprise")] +async fn edit_git_sync_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.git_sync_settings); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_git_sync_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("git_sync_settings", args_for_audit.as_str())].into()), + ) + .await?; + + if let Some(mut git_sync_settings) = new_config.git_sync_settings { + // Clean up legacy workspace-level settings if all repos are migrated + cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); + + let serialized_config = serde_json::to_value::(git_sync_settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + serialized_config, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET git_sync = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + + tx.commit().await?; + + // Trigger git sync for git sync settings changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() }, + Some("Git sync configuration updated".to_string()), + false, + None, + ) + .await?; + + Ok(format!("Edit git sync config for workspace {}", &w_id)) +} + +#[cfg(not(feature = "enterprise"))] +async fn edit_git_sync_repository( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, + Json(_new_config): Json, +) -> Result { + return Err(Error::BadRequest( + "Git sync is only available on Windmill Enterprise Edition".to_string(), + )); +} + +#[cfg(feature = "enterprise")] +async fn edit_git_sync_repository( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + + // Validate the resource path format + validate_git_repo_resource_path(&new_config.git_repo_resource_path)?; + + let mut tx = db.begin().await?; + + // First, get the current git sync settings + let current_settings = sqlx::query!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + let mut git_sync_settings = if let Some(row) = current_settings { + if let Some(git_sync) = row.git_sync { + serde_json::from_value::(git_sync) + .map_err(|err| Error::internal_err(err.to_string()))? + } else { + WorkspaceGitSyncSettings::default() + } + } else { + WorkspaceGitSyncSettings::default() + }; + + // Audit log before we move the repository + audit_log( + &mut *tx, + &authed, + "workspaces.edit_git_sync_repository", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some( + [ + ( + "repository_path", + new_config.git_repo_resource_path.as_str(), + ), + ("repository_data", &format!("{:?}", new_config.repository)), + ] + .into(), + ), + ) + .await?; + + // Check if repository exists before modifying + let repo_exists = git_sync_settings + .repositories + .iter() + .any(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path); + + // Find and update the specific repository, or add it if it doesn't exist + let repo_found = git_sync_settings + .repositories + .iter_mut() + .find(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path); + + if let Some(existing_repo) = repo_found { + // Update existing repository + *existing_repo = new_config.repository; + } else { + // Repository doesn't exist, add it as a new repository + git_sync_settings.repositories.push(new_config.repository); + } + + // Clean up legacy workspace-level settings if all repos are migrated + cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); + + // Save the updated configuration + let serialized_config = serde_json::to_value::(git_sync_settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + serialized_config, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + // Trigger git sync for individual repository update/add + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() }, + Some(format!( + "Git sync repository '{}' {}", + new_config.git_repo_resource_path, + if repo_exists { "updated" } else { "added" } + )), + false, + None, + ) + .await?; + + Ok(format!( + "{} git sync repository '{}' for workspace {}", + if repo_exists { "Updated" } else { "Added" }, + new_config.git_repo_resource_path, + &w_id + )) +} + +#[cfg(not(feature = "enterprise"))] +async fn delete_git_sync_repository( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, + Json(_request): Json, +) -> Result { + return Err(Error::BadRequest( + "Git sync is only available on Windmill Enterprise Edition".to_string(), + )); +} + +#[cfg(feature = "enterprise")] +async fn delete_git_sync_repository( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(request): Json, +) -> Result { + require_admin(is_admin, &username)?; + + // For deletion, only validate that path is not empty to allow cleanup of malformed entries + if request.git_repo_resource_path.is_empty() { + return Err(Error::BadRequest( + "Resource path cannot be empty".to_string(), + )); + } + + let mut tx = db.begin().await?; + + // First, get the current git sync settings + let current_settings = sqlx::query!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + let mut git_sync_settings = if let Some(row) = current_settings { + if let Some(git_sync) = row.git_sync { + serde_json::from_value::(git_sync) + .map_err(|err| Error::internal_err(err.to_string()))? + } else { + WorkspaceGitSyncSettings::default() + } + } else { + WorkspaceGitSyncSettings::default() + }; + + // Check if repository exists and remove it + let original_count = git_sync_settings.repositories.len(); + git_sync_settings + .repositories + .retain(|repo| repo.git_repo_resource_path != request.git_repo_resource_path); + + if git_sync_settings.repositories.len() == original_count { + return Err(Error::BadRequest(format!( + "Repository with path '{}' not found in git sync configuration", + request.git_repo_resource_path + ))); + } + + // Audit log + audit_log( + &mut *tx, + &authed, + "workspaces.delete_git_sync_repository", + ActionKind::Delete, + &w_id, + Some(&authed.email), + Some([("repository_path", request.git_repo_resource_path.as_str())].into()), + ) + .await?; + + // Clean up legacy workspace-level settings if all repos are migrated + cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); + + // Save the updated configuration + let serialized_config = serde_json::to_value::(git_sync_settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + serialized_config, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + // Trigger git sync for repository deletion + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() }, + Some(format!( + "Git sync repository '{}' deleted", + request.git_repo_resource_path + )), + false, + None, + ) + .await?; + + Ok(format!( + "Deleted git sync repository '{}' from workspace {}", + request.git_repo_resource_path, &w_id + )) +} + +#[cfg(feature = "enterprise")] +#[derive(Debug, Deserialize)] +struct EditDeployUIConfig { + deploy_ui_settings: Option, +} + +#[cfg(not(feature = "enterprise"))] +async fn edit_deploy_ui_config( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, +) -> Result { + return Err(Error::BadRequest( + "Deployment UI is only available on Windmill Enterprise Edition".to_string(), + )); +} + +#[cfg(feature = "enterprise")] +async fn edit_deploy_ui_config( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + let args_for_audit = format!("{:?}", new_config.deploy_ui_settings); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_deploy_ui_config", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("deployment_ui_settings", args_for_audit.as_str())].into()), + ) + .await?; + + if let Some(deploy_ui_settings) = new_config.deploy_ui_settings { + let serialized_config = + serde_json::to_value::(deploy_ui_settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET deploy_ui = $1 WHERE workspace_id = $2", + serialized_config, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET deploy_ui = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + Ok(format!("Edit deployment UI config for workspace {}", &w_id)) +} + +#[derive(Deserialize)] +pub struct EditDefaultApp { + #[cfg(feature = "enterprise")] + pub default_app_path: Option, +} + +#[cfg(not(feature = "enterprise"))] +async fn edit_default_app( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, + Json(_new_config): Json, +) -> Result { + return Err(Error::BadRequest( + "Setting a workspace default app is only available on Windmill Enterprise Edition" + .to_string(), + )); +} + +async fn edit_default_scripts( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json>, +) -> Result { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_default_scripts", + ActionKind::Update, + &w_id, + Some(&authed.email), + None, + ) + .await?; + + if let Some(config) = new_config { + sqlx::query!( + "UPDATE workspace_settings SET default_scripts = $1 WHERE workspace_id = $2", + config, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET default_scripts = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + // Trigger git sync for default scripts changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "default_scripts".to_string() }, + Some("Default scripts configuration updated".to_string()), + false, + None, + ) + .await?; + + Ok(format!("Edit default scripts for workspace {}", &w_id)) +} + +async fn get_default_scripts( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let mut tx = db.begin().await?; + let default_scripts = sqlx::query_scalar!( + "SELECT default_scripts FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await + .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; + tx.commit().await?; + + Ok(Json(default_scripts.flatten())) +} + +#[cfg(feature = "enterprise")] +async fn edit_default_app( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(new_config): Json, +) -> Result { + #[cfg(not(feature = "enterprise"))] + { + return Err(Error::BadRequest( + "Setting a workspace default app is only available on Windmill Enterprise Edition" + .to_string(), + )); + } + + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + let args_for_audit = format!("{:?}", new_config.default_app_path); + audit_log( + &mut *tx, + &authed, + "workspaces.edit_default_app", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("default_app", args_for_audit.as_str())].into()), + ) + .await?; + + if let Some(default_app_path) = new_config.default_app_path { + sqlx::query!( + "UPDATE workspace_settings SET default_app = $1 WHERE workspace_id = $2", + default_app_path, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET default_app = NULL WHERE workspace_id = $1", + &w_id, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + // Trigger git sync for default app changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "default_app".to_string() }, + Some("Default app configuration updated".to_string()), + false, + None, + ) + .await?; + + Ok(format!("Edit default app for workspace {}", &w_id)) +} + +#[derive(Serialize)] +struct WorkspaceDefaultApp { + pub default_app_path: Option, +} +async fn get_default_app( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + let mut tx = db.begin().await?; + let default_app_path = sqlx::query_scalar!( + "SELECT default_app FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&mut *tx) + .await + .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; + tx.commit().await?; + + Ok(Json(WorkspaceDefaultApp { default_app_path })) +} + +async fn edit_error_handler( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(ee): Json, +) -> Result { + require_admin(is_admin, &username)?; + + // Normalize to new format (handles both old CLI and new CLI requests) + let ee = ee.into_normalized(); + + let mut tx = db.begin().await?; + + sqlx::query_as!( + Group, + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", + w_id, + "error_handler", + "The group the error handler acts on behalf of", + serde_json::json!({username_to_permissioned_as(&authed.username): true}) + ) + .execute(&mut *tx) + .await?; + + if let Some(path) = &ee.path { + match ee.extra_args.as_ref() { + Some(extra_args) if extra_args.is_object() => { + let Ok(email_recipients) = serde_json::from_value::>>( + extra_args["email_recipients"].to_owned(), + ) else { + return Err(Error::BadRequest( + "Field `email_recipients` expected to be JSON array".to_string(), + )); + }; + + if let Some(email_recipients) = email_recipients { + for email in email_recipients { + if !EMAIL_REGEXP.is_match(&email) { + return Err(Error::BadRequest(format!( + "Invalid email format: {}", + email + ))); + } + } + } + } + None => {} + _ => { + return Err(Error::BadRequest( + "Field `extra_args` expected to be JSON object".to_string(), + )) + } + } + + let mut error_handler = serde_json::json!({ + "path": path, + }); + if let Some(extra_args) = &ee.extra_args { + error_handler["extra_args"] = extra_args.clone(); + } + if ee.muted_on_cancel { + error_handler["muted_on_cancel"] = serde_json::json!(true); + } + if ee.muted_on_user_path { + error_handler["muted_on_user_path"] = serde_json::json!(true); + } + + sqlx::query!( + "UPDATE workspace_settings SET error_handler = $1 WHERE workspace_id = $2", + error_handler, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET error_handler = NULL WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_error_handler", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("error_handler", &format!("{:?}", ee.path)[..])].into()), + ) + .await?; + tx.commit().await?; + + // Trigger git sync for error handler changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "error_handler".to_string() }, + Some("Error handler configuration updated".to_string()), + false, + None, + ) + .await?; + + Ok(format!("Edit error_handler for workspace {}", &w_id)) +} + +async fn edit_success_handler( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(es): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + // Normalize to new format (handles both old CLI and new CLI requests) + let es = es.into_normalized(); + + let mut tx = db.begin().await?; + + sqlx::query_as!( + Group, + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", + w_id, + "success_handler", + "The group the success handler acts on behalf of", + serde_json::json!({username_to_permissioned_as(&authed.username): true}) + ) + .execute(&mut *tx) + .await?; + + if let Some(path) = &es.path { + let mut success_handler = serde_json::json!({ + "path": path, + }); + if let Some(extra_args) = &es.extra_args { + success_handler["extra_args"] = extra_args.clone(); + } + + sqlx::query!( + "UPDATE workspace_settings SET success_handler = $1 WHERE workspace_id = $2", + success_handler, + &w_id + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + "UPDATE workspace_settings SET success_handler = NULL WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "workspaces.edit_success_handler", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some([("success_handler", &format!("{:?}", es.path)[..])].into()), + ) + .await?; + tx.commit().await?; + + // Trigger git sync for success handler changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "success_handler".to_string() }, + Some("Success handler configuration updated".to_string()), + false, + None, + ) + .await?; + + Ok(format!("Edit success_handler for workspace {}", &w_id)) +} + +#[derive(Deserialize)] +struct NewEnvironmentVariable { + name: String, + value: Option, +} + +async fn set_environment_variable( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(NewEnvironmentVariable { value, name }): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + match value { + Some(value) => { + sqlx::query!( + "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = EXCLUDED.value", + &w_id, + name, + value + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspace.set_environment_variable", + ActionKind::Create, + &w_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Set environment variable {}", name)) + } + None => { + sqlx::query!( + "DELETE FROM workspace_env WHERE workspace_id = $1 AND name = $2", + &w_id, + name + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspace.delete_environment_variable", + ActionKind::Delete, + &w_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Deleted environment variable {}", name)) + } + } +} + +#[derive(Serialize)] +pub struct GetEncryptionKeyResponse { + key: String, +} + +async fn get_encryption_key( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_admin(authed.is_admin, &authed.username)?; + + let encryption_key_opt = sqlx::query_scalar!( + "SELECT key FROM workspace_key WHERE workspace_id = $1", + w_id + ) + .fetch_optional(&db) + .await?; + + let encryption_key = not_found_if_none(encryption_key_opt, "workspace_encryption_key", w_id)?; + return Ok(Json(GetEncryptionKeyResponse { key: encryption_key })); +} + +#[derive(Deserialize)] +struct SetEncryptionKeyRequest { + new_key: String, + skip_reencrypt: Option, +} + +async fn set_encryption_key( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> Result<()> { + require_super_admin(&db, &authed.email).await?; + + if !WORKSPACE_KEY_REGEXP.is_match(request.new_key.as_str()) { + return Err(Error::BadRequest( + "Encryption key should be an alphanumeric string of 64 characters".to_string(), + )); + } + + let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; + + sqlx::query!( + "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", + request.new_key.clone(), + w_id + ) + .execute(&db) + .await?; + + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + + if !request.skip_reencrypt.unwrap_or(false) { + let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; + + let mut truncated_new_key = request.new_key.clone(); + truncated_new_key.truncate(8); + tracing::warn!( + "Re-encrypting all secrets for workspace {}. New key is {}***", + w_id, + truncated_new_key + ); + + let all_variables = sqlx::query!( + "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", + w_id + ) + .fetch_all(&db) + .await?; + + for variable in all_variables { + if !variable.is_secret { + continue; + } + let decrypted_value = + decrypt(&previous_encryption_key, variable.value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })?; + let new_encrypted_value = encrypt(&new_encryption_key, decrypted_value.as_str()); + sqlx::query!( + "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3", + new_encrypted_value, + w_id, + variable.path + ) + .execute(&db) + .await?; + } + } + + // Trigger git sync for encryption key changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Key { key_type: "encryption_key".to_string() }, + Some("Encryption key updated".to_string()), + false, + None, + ) + .await?; + + return Ok(()); +} + +#[derive(Serialize)] +struct UsedTriggers { + pub websocket_used: bool, + pub http_routes_used: bool, + pub kafka_used: bool, + pub nats_used: bool, + pub postgres_used: bool, + pub mqtt_used: bool, + pub sqs_used: bool, + pub gcp_used: bool, + pub email_used: bool, + pub nextcloud_used: bool, +} + +async fn get_used_triggers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let triggers_used = sqlx::query_as!( + UsedTriggers, + r#" + SELECT + EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS "websocket_used!", + EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS "http_routes_used!", + EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as "kafka_used!", + EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!", + EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!", + EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS "mqtt_used!", + EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!", + EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS "gcp_used!", + EXISTS(SELECT 1 FROM email_trigger WHERE workspace_id = $1) AS "email_used!", + EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS "nextcloud_used!" + "#, + w_id + ) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(triggers_used)) +} + +async fn get_workspace_as_superadmin( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + require_super_admin(&db, &authed.email).await?; + let workspace = sqlx::query_as!( + Workspace, + "SELECT + workspace.id AS \"id!\", + workspace.name AS \"name!\", + workspace.owner AS \"owner!\", + workspace.deleted AS \"deleted!\", + workspace.premium AS \"premium!\", + workspace_settings.color AS \"color\", + workspace.parent_workspace_id AS \"parent_workspace_id\" + FROM workspace + LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id + WHERE workspace.id = $1", + w_id + ) + .fetch_optional(&db) + .await?; + + let workspace = not_found_if_none(workspace, "workspace", w_id)?; + + Ok(Json(workspace)) +} + +async fn list_workspaces_as_super_admin( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Query(pagination): Query, + ApiAuthed { email, .. }: ApiAuthed, +) -> JsonResult> { + require_super_admin(&db, &email).await?; + let (per_page, offset) = paginate(pagination); + + let mut tx = user_db.begin(&authed).await?; + let workspaces = sqlx::query_as!( + Workspace, + "SELECT + workspace.id AS \"id!\", + workspace.name AS \"name!\", + workspace.owner AS \"owner!\", + workspace.deleted AS \"deleted!\", + workspace.premium AS \"premium!\", + workspace_settings.color AS \"color\", + workspace.parent_workspace_id AS \"parent_workspace_id\" + FROM workspace + LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id + LIMIT $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(workspaces)) +} + +async fn user_workspaces( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, +) -> JsonResult { + let mut tx = db.begin().await?; + let workspaces = sqlx::query_as!( + UserWorkspace, + "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, + CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, + usr.disabled + FROM workspace + JOIN usr ON usr.workspace_id = workspace.id + JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id + WHERE usr.email = $1 AND workspace.deleted = false", + email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(WorkspaceList { email, workspaces })) +} + +pub async fn check_w_id_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> { + if w_id == "global" { + return Err(windmill_common::error::Error::BadRequest( + "'global' is not allowed as a workspace ID".to_string(), + )); + } + let exists = sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)", w_id) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(windmill_common::error::Error::BadRequest(format!( + "Workspace {} already exists", + w_id + ))); + } + return Ok(()); +} + +lazy_static::lazy_static! { + + pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = { + match std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN") { + Ok(val) => val == "true", + Err(_) => true, + } + }; + + pub static ref DISABLE_WORKSPACE_FORK: bool = { + match std::env::var("DISABLE_WORKSPACE_FORK") { + Ok(val) => val == "true", + Err(_) => false, + } + }; + +} + +async fn create_workspace_require_superadmin() -> String { + format!("{}", *CREATE_WORKSPACE_REQUIRE_SUPERADMIN) +} + +async fn _check_nb_of_workspaces(db: &DB) -> Result<()> { + let nb_workspaces = sqlx::query_scalar!( + "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false", + ) + .fetch_one(db) + .await?; + if nb_workspaces.unwrap_or(0) >= 2 { + return Err(Error::BadRequest( + "You have reached the maximum number of workspaces (2 outside of default workspace 'admins') without an enterprise license. Archive/delete another workspace to create a new one" + .to_string(), + )); + } + return Ok(()); +} + +async fn create_workspace( + authed: ApiAuthed, + Extension(db): Extension, + Json(nw): Json, +) -> Result { + if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN { + require_super_admin(&db, &authed.email).await?; + } + + #[cfg(not(feature = "enterprise"))] + _check_nb_of_workspaces(&db).await?; + + if *CLOUD_HOSTED { + let nb_workspaces = sqlx::query_scalar!( + "SELECT COUNT(*) FROM workspace WHERE owner = $1", + authed.email + ) + .fetch_one(&db) + .await?; + if nb_workspaces.unwrap_or(0) >= 10 { + return Err(Error::BadRequest( + "You have reached the maximum number of workspaces (10) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + + check_w_id_conflict(&mut tx, &nw.id).await?; + sqlx::query!( + "INSERT INTO workspace + (id, name, owner) + VALUES ($1, $2, $3)", + nw.id, + nw.name, + authed.email, + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "INSERT INTO workspace_settings + (workspace_id, color) + VALUES ($1, $2)", + nw.id, + nw.color, + ) + .execute(&mut *tx) + .await?; + let key = rd_string(64); + sqlx::query!( + "INSERT INTO workspace_key + (workspace_id, kind, key) + VALUES ($1, 'cloud', $2)", + nw.id, + &key + ) + .execute(&mut *tx) + .await?; + + // let mc = magic_crypt::new_magic_crypt!(key, 256); + // sqlx::query!( + // "INSERT INTO variable + // (workspace_id, path, value, is_secret, description) + // VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'), + // ($3, 'g/all/not_secret', $4, false, 'This item is not secret')", + // nw.id, + // crate::variables::encrypt(&mc, "pretty secret value"), + // nw.id, + // "finland does not actually exist", + // ) + // .execute(&mut *tx) + // .await?; + + let automate_username_creation = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + AUTOMATE_USERNAME_CREATION_SETTING, + ) + .fetch_optional(&mut *tx) + .await? + .map(|v| v.as_bool()) + .flatten() + .unwrap_or(false); + + let username = if automate_username_creation { + if nw.username.is_some() && nw.username.unwrap().len() > 0 { + return Err(Error::BadRequest( + "username is not allowed when username creation is automated".to_string(), + )); + } + get_instance_username_or_create_pending(&mut tx, &authed.email).await? + } else { + nw.username + .ok_or(Error::BadRequest("username is required".to_string()))? + }; + + sqlx::query!( + "INSERT INTO usr + (workspace_id, email, username, is_admin) + VALUES ($1, $2, $3, true)", + nw.id, + authed.email, + username, + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "INSERT INTO group_ + VALUES ($1, 'all', 'The group that always contains all users of this workspace')", + nw.id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "INSERT INTO usr_to_group + VALUES ($1, 'all', $2)", + nw.id, + username + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.create", + ActionKind::Create, + &nw.id, + Some(nw.name.as_str()), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("Created workspace {}", &nw.id)) +} + +async fn clone_workspace_data( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // Clone workspace settings (merge with existing basic settings) + update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; + + // Clone workspace environment variables + clone_workspace_env(tx, source_workspace_id, target_workspace_id).await?; + + // Clone folders + clone_folders(tx, source_workspace_id, target_workspace_id).await?; + + // Clone groups + clone_groups(tx, source_workspace_id, target_workspace_id).await?; + + // Clone resource types + clone_resource_types(tx, source_workspace_id, target_workspace_id).await?; + + // Clone resources + clone_resources(tx, source_workspace_id, target_workspace_id).await?; + + // Clone variables with re-encryption + clone_variables(tx, source_workspace_id, target_workspace_id).await?; + + // Clone scripts with new hashes + clone_scripts(tx, source_workspace_id, target_workspace_id).await?; + + // Clone flows with new versions + clone_flows(tx, source_workspace_id, target_workspace_id).await?; + + // Clone flow nodes + clone_flow_nodes(tx, source_workspace_id, target_workspace_id).await?; + + // Clone apps with new IDs and app scripts + let _app_id_mapping = clone_apps(tx, source_workspace_id, target_workspace_id).await?; + + // Clone raw apps + clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?; + + // Clone workspace runnable dependencies and dependency map + clone_workspace_runnable_dependencies(tx, source_workspace_id, target_workspace_id).await?; + + // TODO: Enable when git sync is implemented for workspace dependencies. + // // Clone workspace dependencies + // clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?; + Ok(()) +} + +async fn update_workspace_settings( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO workspace_key (workspace_id, kind, key) + SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#" + UPDATE workspace_settings + SET + deploy_to = $1, + ai_config = source_ws.ai_config, + large_file_storage = source_ws.large_file_storage, + ducklake = source_ws.ducklake, + datatable = source_ws.datatable, + git_app_installations = source_ws.git_app_installations + FROM workspace_settings source_ws + WHERE source_ws.workspace_id = $1 + AND workspace_settings.workspace_id = $2 + "#, + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + let current_git_sync_settings = sqlx::query!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + source_workspace_id + ) + .fetch_optional(&mut **tx) + .await?; + + let mut git_sync_settings = if let Some(row) = current_git_sync_settings { + if let Some(git_sync) = row.git_sync { + serde_json::from_value::(git_sync) + .map_err(|err| Error::internal_err(err.to_string()))? + } else { + WorkspaceGitSyncSettings::default() + } + } else { + WorkspaceGitSyncSettings::default() + }; + + // We only keep the first git sync repo that is sync mode (use_individual_branch = false), since it is considered the main one + // Context: see WIN-1559 + git_sync_settings.repositories = git_sync_settings + .repositories + .into_iter() + .filter(|r| !r.use_individual_branch.unwrap_or(false)) + .take(1) + .collect(); + + let serialized_config = serde_json::to_value::(git_sync_settings) + .map_err(|err| Error::internal_err(err.to_string()))?; + + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + serialized_config, + target_workspace_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_workspace_env( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO workspace_env (workspace_id, name, value) + SELECT $2, name, value + FROM workspace_env + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_folders( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by) + SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, created_by + FROM folder + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_groups( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO group_ (workspace_id, name, summary, extra_perms) + SELECT $2, name, summary, extra_perms + FROM group_ + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "INSERT INTO usr_to_group (workspace_id, group_, usr) + SELECT $2, group_, usr + FROM usr_to_group + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_resource_types( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension) + SELECT $2, name, schema, description, edited_at, created_by, format_extension + FROM resource_type + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_resources( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, edited_at, created_by) + SELECT $2, path, value, description, resource_type, extra_perms, edited_at, created_by + FROM resource + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_variables( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) + SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at + FROM variable + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_scripts( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // Clone all scripts directly with a single query + sqlx::query!( + r#"INSERT INTO script ( + workspace_id, hash, path, parent_hashes, summary, description, content, + created_by, created_at, archived, schema, deleted, is_template, + extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, + envs, concurrent_limit, concurrency_time_window_s, cache_ttl, + dedicated_worker, ws_error_handler_muted, priority, timeout, + delete_after_use, restart_unless_cancelled, concurrency_key, + visible_to_runner_only, no_main_func, codebase, has_preprocessor, + on_behalf_of_email, assets + ) + SELECT + $1, hash, path, parent_hashes, summary, description, content, + created_by, created_at, archived, schema, deleted, is_template, + extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, + envs, concurrent_limit, concurrency_time_window_s, cache_ttl, + dedicated_worker, ws_error_handler_muted, priority, timeout, + delete_after_use, restart_unless_cancelled, concurrency_key, + visible_to_runner_only, no_main_func, codebase, has_preprocessor, + on_behalf_of_email, assets + FROM script + WHERE workspace_id = $2"#, + target_workspace_id, + source_workspace_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_flows( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // First, clone flows without versions + sqlx::query!( + "INSERT INTO flow ( + workspace_id, path, summary, description, value, edited_by, edited_at, + archived, schema, extra_perms, dependency_job, draft_only, tag, + ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, + concurrency_key, versions, on_behalf_of_email, lock_error_logs + ) + SELECT $2, path, summary, description, value, edited_by, edited_at, + archived, schema, extra_perms, NULL, draft_only, tag, + ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, + concurrency_key, ARRAY[]::bigint[], on_behalf_of_email, lock_error_logs + FROM flow + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + // Then clone flow versions + let flow_versions = sqlx::query!( + "SELECT id, workspace_id, path, value, schema, created_by, created_at + FROM flow_version + WHERE workspace_id = $1 + ORDER BY path, created_at", + source_workspace_id + ) + .fetch_all(&mut **tx) + .await?; + + for version in flow_versions { + let new_version_id = sqlx::query_scalar!( + "INSERT INTO flow_version (workspace_id, path, value, schema, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id", + target_workspace_id, + version.path, + version.value, + version.schema, + version.created_by, + version.created_at, + ) + .fetch_one(&mut **tx) + .await?; + + // Update flow to include this version + sqlx::query!( + "UPDATE flow + SET versions = array_append(versions, $1) + WHERE workspace_id = $2 AND path = $3", + new_version_id, + target_workspace_id, + version.path, + ) + .execute(&mut **tx) + .await?; + } + + Ok(()) +} + +async fn clone_flow_nodes( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO flow_node (workspace_id, hash, path, lock, code, flow, hash_v2) + SELECT $2, + (SELECT COALESCE(MAX(hash), 0) FROM flow_node) + row_number() OVER () AS new_hash, + source_fn.path, source_fn.lock, source_fn.code, source_fn.flow, source_fn.hash_v2 + FROM flow_node source_fn + WHERE source_fn.workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_apps( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result> { + // Get all apps from source workspace + let apps = sqlx::query!( + "SELECT id, workspace_id, path, summary, policy, versions, extra_perms, draft_only, custom_path + FROM app + WHERE workspace_id = $1", + source_workspace_id + ) + .fetch_all(&mut **tx) + .await?; + + let mut app_id_mapping: HashMap = HashMap::new(); + + // Clone apps with new IDs + for app in apps { + let new_app_id = sqlx::query_scalar!( + "INSERT INTO app (workspace_id, path, summary, policy, versions, extra_perms, draft_only, custom_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + RETURNING id", + target_workspace_id, + app.path, + app.summary, + app.policy, + &Vec::::new(), // Start with empty versions array + app.extra_perms, + app.draft_only, + app.custom_path, + ) + .fetch_one(&mut **tx) + .await?; + + app_id_mapping.insert(app.id, new_app_id); + } + + { + // Clone app versions + let app_versions = sqlx::query!( + "SELECT app_id, value, created_by, created_at, raw_app + FROM app_version + WHERE app_id = ANY(SELECT id FROM app WHERE workspace_id = $1) + ORDER BY app_id, created_at", + source_workspace_id + ) + .fetch_all(&mut **tx) + .await?; + + for version in app_versions { + if let Some(&new_app_id) = app_id_mapping.get(&version.app_id) { + sqlx::query!( + "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) + VALUES ($1, $2, $3, $4, $5)", + new_app_id, + version.value, + version.created_by, + version.created_at, + version.raw_app, + ) + .execute(&mut **tx) + .await?; + } + } + } + + // Update app versions arrays + sqlx::query!( + "UPDATE app SET versions = ( + SELECT array_agg(av.id ORDER BY av.created_at) + FROM app_version av + WHERE av.app_id = app.id + ) WHERE workspace_id = $1", + target_workspace_id + ) + .execute(&mut **tx) + .await?; + + // Clone app scripts with recomputed hashes + let app_scripts = sqlx::query!( + "SELECT app, hash, lock, code, code_sha256 + FROM app_script + WHERE app = ANY(SELECT id FROM app WHERE workspace_id = $1)", + source_workspace_id + ) + .fetch_all(&mut **tx) + .await?; + + for app_script in app_scripts { + if let Some(&new_app_id) = app_id_mapping.get(&app_script.app) { + // Recompute hash using app_id, code_sha256, and lock + let mut hasher = Sha256::new(); + hasher.update(new_app_id.to_be_bytes()); + hasher.update(hex::decode(&app_script.code_sha256)?); + if let Some(lock) = &app_script.lock { + hasher.update(lock.as_bytes()); + } + let new_hash = hex::encode(hasher.finalize()); + + sqlx::query!( + "INSERT INTO app_script (app, hash, lock, code, code_sha256) + VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", + new_app_id, + new_hash, + app_script.lock, + app_script.code, + app_script.code_sha256, + ) + .execute(&mut **tx) + .await?; + } + } + + Ok(app_id_mapping) +} + +async fn clone_raw_apps( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO raw_app (path, version, workspace_id, summary, edited_at, data, extra_perms) + SELECT path, version, $2, summary, edited_at, data, extra_perms + FROM raw_app + WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn clone_workspace_runnable_dependencies( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // Clone workspace_runnable_dependencies + sqlx::query!( + "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) + SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path + FROM workspace_runnable_dependencies + WHERE workspace_id = $2", + target_workspace_id, + source_workspace_id + ) + .execute(&mut **tx) + .await?; + + // Clone dependency_map to preserve import relationships + sqlx::query!( + "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) + SELECT $1, importer_path, importer_kind, imported_path, importer_node_id + FROM dependency_map + WHERE workspace_id = $2", + target_workspace_id, + source_workspace_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +#[allow(dead_code)] +async fn clone_workspace_dependencies( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + // Clone workspace_runnable_dependencies + sqlx::query!( + "INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at) + SELECT $1, language, name, description, content, archived, created_at + FROM workspace_dependencies + WHERE workspace_id = $2", + target_workspace_id, + source_workspace_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} + +async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result { + return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string())); +} + +/// Return the uuids of the git sync jobs to create the branch before creating the fork +async fn create_workspace_fork_branch( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(nw): Json, +) -> JsonResult> { + if *CLOUD_HOSTED { + return Err(Error::BadRequest(format!( + "Forking workspaces is not available on app.windmill.dev" + ))); + } + + if *DISABLE_WORKSPACE_FORK { + require_super_admin(&db, &authed.email).await?; + } + + Ok(Json( + handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, + )) +} + +async fn create_workspace_fork( + authed: ApiAuthed, + Extension(db): Extension, + Path(parent_workspace_id): Path, + Json(nw): Json, +) -> Result { + if *CLOUD_HOSTED { + return Err(Error::BadRequest(format!( + "Forking workspaces is not available on app.windmill.dev" + ))); + } + + if *DISABLE_WORKSPACE_FORK { + require_super_admin(&db, &authed.email).await?; + } + + let mut tx: Transaction<'_, Postgres> = db.begin().await?; + + // Generate unique forked workspace ID with wm-fork prefix + if !nw.id.starts_with(WM_FORK_PREFIX) { + return Err(Error::BadRequest(format!( + "The id `{}` is invalid for a forked workspace. It should be prefixed by {}", + nw.id, WM_FORK_PREFIX + ))); + } + + let forked_id = nw.id; + + sqlx::query!( + "INSERT INTO workspace + (id, name, owner, parent_workspace_id) + VALUES ($1, $2, $3, $4)", + forked_id, + nw.name, + authed.email, + parent_workspace_id, + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "INSERT INTO workspace_settings + (workspace_id, color) + VALUES ($1, $2)", + forked_id, + nw.color, + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "INSERT INTO usr + (workspace_id, email, username, is_admin) + SELECT $1, email, username, is_admin FROM usr + WHERE workspace_id = $3 AND email = $2 + ", + forked_id, + authed.email, + parent_workspace_id, + ) + .execute(&mut *tx) + .await?; + + // Clone all data from the parent workspace using Rust implementation + clone_workspace_data(&mut tx, &parent_workspace_id, &forked_id).await?; + + sqlx::query!( + "INSERT INTO workspace_invite (workspace_id, email, is_admin, operator) + SELECT $1, email, is_admin, operator + FROM usr + WHERE workspace_id = $2", + &forked_id, + &parent_workspace_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.create_fork", + ActionKind::Create, + &forked_id, + Some(nw.name.as_str()), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("Created forked workspace {}", &forked_id)) +} + +async fn edit_workspace( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { is_admin, username, .. }: ApiAuthed, + Json(ew): Json, +) -> Result { + require_admin(is_admin, &username)?; + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace SET name = $1, owner = $2 WHERE id = $3", + ew.name, + ew.owner, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.update", + ActionKind::Update, + &w_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Updated workspace {}", &w_id)) +} + +/// Archive a workspace: disable schedules, cancel jobs, and mark as deleted. +/// Returns (schedules_disabled_count, jobs_canceled_count). +pub(crate) async fn archive_workspace_impl( + db: &DB, + w_id: &str, + username: &str, +) -> Result<(usize, usize)> { + // Step 1: Disable all schedules and clear their queued jobs + let mut tx = db.begin().await?; + let disabled_schedules = sqlx::query_scalar!( + "UPDATE schedule SET enabled = false WHERE workspace_id = $1 AND enabled = true RETURNING path", + w_id + ) + .fetch_all(&mut *tx) + .await?; + + let schedules_count = disabled_schedules.len(); + tracing::info!( + "Disabled {} schedules in workspace {}", + schedules_count, + w_id + ); + + // Clear all schedule-related jobs using the existing clear_schedule function + for schedule_path in &disabled_schedules { + windmill_queue::schedule::clear_schedule(&mut tx, schedule_path, w_id).await?; + } + + // Mark workspace as archived + sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", w_id) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + // Step 2: Get all remaining queued jobs for this workspace (non-schedule jobs) + let jobs_to_cancel = + sqlx::query_scalar!("SELECT id FROM v2_job_queue WHERE workspace_id = $1", w_id) + .fetch_all(db) + .await?; + + let jobs_count = jobs_to_cancel.len(); + tracing::info!( + "Found {} remaining jobs to cancel in workspace {}", + jobs_count, + w_id + ); + + // Step 3: Cancel all remaining jobs using the existing cancel_jobs function + let canceled_count = if !jobs_to_cancel.is_empty() { + let axum::Json(canceled_jobs) = windmill_api_jobs::cancel_jobs( + jobs_to_cancel, + db, + username, + w_id, + false, // force_cancel + ) + .await?; + + let count = canceled_jobs.len(); + tracing::info!("Canceled {} jobs in workspace {}", count, w_id); + count + } else { + 0 + }; + + Ok((schedules_count, canceled_count)) +} + +async fn archive_workspace( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let (schedules_count, canceled_count) = + archive_workspace_impl(&db, &w_id, &authed.username).await?; + + // Audit log + let mut tx = db.begin().await?; + let mut audit_params = HashMap::new(); + audit_params.insert("disabled_schedules", schedules_count.to_string()); + audit_params.insert("canceled_jobs", canceled_count.to_string()); + let audit_params_refs: HashMap<&str, &str> = + audit_params.iter().map(|(k, v)| (*k, v.as_str())).collect(); + + audit_log( + &mut *tx, + &authed, + "workspaces.archive", + ActionKind::Update, + &w_id, + Some(&authed.email), + Some(audit_params_refs), + ) + .await?; + tx.commit().await?; + + Ok(format!( + "Archived workspace {}, disabled {} schedules and canceled {} jobs", + &w_id, schedules_count, canceled_count + )) +} + +async fn leave_workspace( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + let mut tx = db.begin().await?; + sqlx::query!( + "DELETE FROM usr WHERE workspace_id = $1 AND email = $2", + &w_id, + &authed.email + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.leave", + ActionKind::Delete, + &w_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Left workspace {}", &w_id)) +} + +async fn unarchive_workspace( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let mut tx = db.begin().await?; + sqlx::query!("UPDATE workspace SET deleted = false WHERE id = $1", &w_id) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.unarchive", + ActionKind::Update, + &w_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Unarchived workspace {}", &w_id)) +} + +async fn invite_user( + ApiAuthed { username, is_admin, .. }: ApiAuthed, + Extension(db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(mut nu): Json, +) -> Result<(StatusCode, String)> { + require_admin(is_admin, &username)?; + + nu.email = nu.email.to_lowercase(); + + let mut tx = db.begin().await?; + + let already_in_workspace = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &w_id, + nu.email + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_in_workspace { + return Err(Error::BadRequest(format!( + "user with email {} already exists in workspace {}", + nu.email, w_id + ))); + } + + sqlx::query!( + "INSERT INTO workspace_invite + (workspace_id, email, is_admin, operator) + VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, email) + DO UPDATE SET is_admin = EXCLUDED.is_admin, operator = EXCLUDED.operator", + &w_id, + nu.email, + nu.is_admin, + nu.operator + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + send_email_if_possible( + &format!("Invited to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} + +If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", + BASE_URL.read().await.clone() + ), + &nu.email, + ); + + webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace { + email: nu.email.clone(), + workspace: w_id, + }); + + Ok(( + StatusCode::CREATED, + format!("user with email {} invited", nu.email), + )) +} + +async fn add_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(mut nu): Json, +) -> Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + nu.email = nu.email.to_lowercase(); + + let mut tx = db.begin().await?; + + let already_exists_email = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &w_id, + nu.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_email { + return Err(Error::BadRequest(format!( + "user with email {} already exists in workspace {}", + nu.email, w_id + ))); + } + + let automate_username_creation = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + AUTOMATE_USERNAME_CREATION_SETTING, + ) + .fetch_optional(&mut *tx) + .await? + .map(|v| v.as_bool()) + .flatten() + .unwrap_or(false); + + let username = if automate_username_creation { + if nu.username.is_some() && nu.username.unwrap().len() > 0 { + return Err(Error::BadRequest( + "username is not allowed when username creation is automated".to_string(), + )); + } + get_instance_username_or_create_pending(&mut tx, &nu.email).await? + } else { + let username = nu + .username + .ok_or(Error::BadRequest("username is required".to_string()))?; + + if !VALID_USERNAME.is_match(&username) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Usermame can only contain alphanumeric characters and underscores and must start with a letter" + ))); + } + + username + }; + + sqlx::query!( + "INSERT INTO usr + (workspace_id, email, username, is_admin, operator) + VALUES ($1, $2, $3, $4, $5)", + &w_id, + nu.email, + username, + nu.is_admin, + nu.operator + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2", + &w_id, + nu.email + ) + .execute(&mut *tx) + .await?; + + sqlx::query_as!( + Group, + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + username, + "all", + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.add_to_workspace", + ActionKind::Create, + &w_id, + Some(&nu.email), + None, + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: nu.email.clone() }, + Some(format!("Added user '{}' to workspace", &nu.email)), + true, + None, + ) + .await?; + + send_email_if_possible( + &format!("Added to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} by {} + +If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", + authed.email, + BASE_URL.read().await.clone() + ), + &nu.email, + ); + + webhook.send_instance_event(InstanceEvent::UserAddedWorkspace { + workspace: w_id.clone(), + email: nu.email.clone(), + }); + + Ok(( + StatusCode::CREATED, + format!("user with email {} added", nu.email), + )) +} + +async fn delete_invite( + ApiAuthed { username, is_admin, .. }: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + require_admin(is_admin, &username)?; + + let mut tx = db.begin().await?; + + sqlx::query!( + "DELETE FROM workspace_invite WHERE + workspace_id = $1 AND email = $2 AND is_admin = $3 AND operator = $4", + &w_id, + nu.email, + nu.is_admin, + nu.operator + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(( + StatusCode::CREATED, + format!("invite to email {} deleted", nu.email), + )) +} + +async fn exists_username( + Extension(db): Extension, + Json(vu): Json, +) -> Result { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)", + vu.username, + vu.id + ) + .fetch_one(&db) + .await? + .unwrap_or(true); + + if exists { + return Err(Error::BadRequest("username already taken".to_string())); + } + + Ok("valid username".to_string()) +} + +async fn get_workspace_name( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + let workspace = sqlx::query_scalar!("SELECT name FROM workspace WHERE id = $1", &w_id) + .fetch_one(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(workspace) +} + +async fn get_dependency_map( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + let dmap = sqlx::query_as!( + DependencyMap, + " + SELECT workspace_id, importer_path, importer_kind::text, imported_path, importer_node_id + FROM dependency_map WHERE workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(dmap)) +} + +#[axum::debug_handler] +async fn rebuild_dependency_map( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest("Disabled on Cloud".into())); + } + ScopedDependencyMap::rebuild_map(&w_id, &db).await +} + +#[axum::debug_handler] +async fn get_dependents( + Extension(db): Extension, + Path((w_id, imported_path)): Path<(String, String)>, + _authed: ApiAuthed, +) -> JsonResult> { + tracing::debug!( + workspace_id = %w_id, + imported_path = %imported_path, + "API: Getting dependents for imported path" + ); + + let dependents = ScopedDependencyMap::get_dependents(&imported_path, &w_id, &db).await?; + + tracing::debug!( + workspace_id = %w_id, + imported_path = %imported_path, + dependents_count = dependents.len(), + "API: Found dependents: {:?}", + dependents + ); + + Ok(Json(dependents)) +} + +#[derive(Serialize, Debug)] +struct DependentsAmount { + imported_path: String, + count: i64, +} + +#[axum::debug_handler] +async fn get_dependents_amounts( + Extension(db): Extension, + Path(w_id): Path, + Json(imported_paths): Json>, +) -> JsonResult> { + tracing::debug!( + workspace_id = %w_id, + imported_paths = ?imported_paths, + "API: Getting dependents amounts for imported paths" + ); + + let results = sqlx::query_as!( + DependentsAmount, + r#" + SELECT + imported_path, + COUNT(DISTINCT importer_path) as "count!" + FROM dependency_map + WHERE workspace_id = $1 AND imported_path = ANY($2) + GROUP BY imported_path + "#, + w_id, + &imported_paths + ) + .fetch_all(&db) + .await?; + + tracing::debug!( + workspace_id = %w_id, + results_count = results.len(), + "API: Found dependents amounts: {:?}", + results + ); + + Ok(Json(results)) +} + +#[derive(Deserialize)] +struct ChangeWorkspaceName { + new_name: String, +} + +#[derive(Deserialize)] +struct ChangeWorkspaceColor { + color: Option, +} + +async fn change_workspace_name( + authed: ApiAuthed, + Path(w_id): Path, + Extension(db): Extension, + Json(rw): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE workspace SET name = $1 WHERE id = $2", + &rw.new_name, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspace.change_workspace_name", + ActionKind::Update, + &w_id, + Some(&authed.email), + None, + ) + .await?; + + tx.commit().await?; + + // Trigger git sync for workspace name changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { setting_type: "workspace_name".to_string() }, + Some(format!("Workspace name updated to {}", &rw.new_name)), + false, + None, + ) + .await?; + + Ok(format!("updated workspace name to {}", &rw.new_name)) +} + +async fn change_workspace_color( + authed: ApiAuthed, + Path(w_id): Path, + Extension(db): Extension, + Json(rw): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + sqlx::query!( + "UPDATE workspace_settings SET color = $1 WHERE workspace_id = $2", + rw.color, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Settings { setting_type: "workspace_color".to_string() }, + None, + false, + None, + ) + .await?; + + Ok(format!( + "updated workspace color to {}", + rw.color.as_deref().unwrap_or("no color") + )) +} + +async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Result { + let usage = sqlx::query_scalar!( + " + SELECT usage.usage FROM usage + WHERE is_workspace = true + AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) + AND id = $1", + w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(0); + Ok(usage.to_string()) +} + +#[derive(Deserialize)] +pub struct EditPublicAppRateLimitRequest { + pub public_app_execution_limit_per_minute: Option, +} + +async fn edit_public_app_rate_limit( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + sqlx::query!( + "UPDATE workspace_settings SET public_app_execution_limit_per_minute = $1 WHERE workspace_id = $2", + req.public_app_execution_limit_per_minute, + &w_id + ) + .execute(&db) + .await?; + + // Cache is invalidated via DB trigger -> notify_event -> polling in main.rs + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Settings { setting_type: "public_app_rate_limit".to_string() }, + None, + false, + None, + ) + .await?; + + Ok(format!( + "Updated public app rate limit for workspace: {}", + &w_id + )) +} + +// 5 minutes fallback TTL (in addition to event-based invalidation) +const PUBLIC_APP_RATE_LIMIT_CACHE_TTL_SECS: i64 = 300; + +pub async fn get_public_app_rate_limit(db: &DB, w_id: &str) -> Result> { + use windmill_common::workspaces::PUBLIC_APP_RATE_LIMIT_CACHE; + + let now = Utc::now().timestamp(); + + if let Some((rate_limit, cached_at)) = PUBLIC_APP_RATE_LIMIT_CACHE.get(w_id) { + if now - cached_at < PUBLIC_APP_RATE_LIMIT_CACHE_TTL_SECS { + return Ok(rate_limit); + } + } + + let result: Option> = sqlx::query_scalar( + "SELECT public_app_execution_limit_per_minute FROM workspace_settings WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_optional(db) + .await?; + let rate_limit = result.flatten(); + PUBLIC_APP_RATE_LIMIT_CACHE.insert(w_id.to_string(), (rate_limit, now)); + Ok(rate_limit) +} + +#[derive(Deserialize, Serialize)] +struct ChangeOperatorSettings { + #[serde(default)] + runs: bool, + #[serde(default)] + schedules: bool, + #[serde(default)] + resources: bool, + #[serde(default)] + variables: bool, + #[serde(default)] + assets: bool, + #[serde(default)] + triggers: bool, + #[serde(default)] + audit_logs: bool, + #[serde(default)] + groups: bool, + #[serde(default)] + folders: bool, + #[serde(default)] + workers: bool, +} + +async fn update_operator_settings( + authed: ApiAuthed, + Path(w_id): Path, + Extension(db): Extension, + Json(settings): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + let settings_json = serde_json::json!(settings); + + sqlx::query!( + "UPDATE workspace_settings SET operator_settings = $1 WHERE workspace_id = $2", + settings_json, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + // Trigger git sync for operator settings changes + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::Settings { + setting_type: "operator_settings".to_string(), + }, + Some("Operator settings updated".to_string()), + false, + None, + ) + .await?; + + Ok("Operator settings updated successfully".to_string()) +} + +#[derive(Serialize)] +pub struct WorkspaceComparison { + pub all_ahead_items_visible: bool, + pub all_behind_items_visible: bool, + pub skipped_comparison: bool, + pub diffs: Vec, + pub summary: CompareSummary, +} + +#[derive(Serialize, Default)] +pub struct CompareSummary { + pub total_diffs: usize, + pub total_ahead: usize, + pub total_behind: usize, + pub scripts_changed: usize, + pub flows_changed: usize, + pub apps_changed: usize, + pub resources_changed: usize, + pub variables_changed: usize, + pub resource_types_changed: usize, + pub folders_changed: usize, + pub conflicts: usize, // Items that are both ahead and behind +} + +async fn reset_workspace_diffs( + authed: ApiAuthed, + Path((w_id, target_workspace_id)): Path<(String, String)>, + Extension(db): Extension, + Extension(user_db): Extension, +) -> JsonResult<()> { + // Needed to compute the has_changes: Option. Otherwise it will be None, and the query will not hit the items + let _ = compare_workspaces( + authed, + Path((w_id.clone(), target_workspace_id.clone())), + Extension(db.clone()), + Extension(user_db), + ) + .await?; + + sqlx::query!( + "DELETE FROM workspace_diff WHERE has_changes = false AND ( + (source_workspace_id = $1 AND fork_workspace_id = $2) + OR (source_workspace_id = $2 AND fork_workspace_id =$1) + )", + target_workspace_id, + w_id, + ) + .execute(&db) + .await?; + + Ok(Json(())) +} + +#[derive(Serialize, Debug, Clone, Default)] +pub struct WorkspaceDiffRow { + kind: String, + path: String, + ahead: i32, + behind: i32, + has_changes: Option, + exists_in_source: Option, + exists_in_fork: Option, +} + +async fn compare_workspaces( + authed: ApiAuthed, + Path((source_workspace_id, fork_workspace_id)): Path<(String, String)>, + Extension(db): Extension, + Extension(user_db): Extension, +) -> JsonResult { + // require_admin(authed.is_admin, &authed.username)?; + + let skipped_comparison: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 FROM skip_workspace_diff_tally + WHERE workspace_id = $1 + )", + ) + .bind(&fork_workspace_id) + .fetch_one(&db) + .await?; + + if skipped_comparison { + return Ok(Json(WorkspaceComparison { + all_ahead_items_visible: true, + all_behind_items_visible: true, + skipped_comparison, + diffs: vec![], + summary: Default::default(), + })); + } + + let diff_items = sqlx::query_as!( + WorkspaceDiffRow, + "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff + WHERE source_workspace_id = $1 AND fork_workspace_id = $2", + source_workspace_id, + fork_workspace_id, + ) + .fetch_all(&db) + .await?; + + let mut confirmed_diffs = vec![]; + for item in diff_items { + if let Some(has_changes) = item.has_changes { + if has_changes { + confirmed_diffs.push(item); + } + continue; + } + + let item_comparison = match item.kind.as_str() { + "script" => Some( + compare_two_scripts(&db, &source_workspace_id, &fork_workspace_id, &item.path) + .await?, + ), + "flow" => Some( + compare_two_flows(&db, &source_workspace_id, &fork_workspace_id, &item.path) + .await?, + ), + "app" => Some( + compare_two_apps(&db, &source_workspace_id, &fork_workspace_id, &item.path).await?, + ), + "resource" => Some( + compare_two_resources(&db, &source_workspace_id, &fork_workspace_id, &item.path) + .await?, + ), + "variable" => Some( + compare_two_variables(&db, &source_workspace_id, &fork_workspace_id, &item.path) + .await?, + ), + "resource_type" => Some( + compare_two_resource_types( + &db, + &source_workspace_id, + &fork_workspace_id, + &item.path, + ) + .await?, + ), + "folder" => Some( + compare_two_folders(&db, &source_workspace_id, &fork_workspace_id, &item.path) + .await?, + ), + k => { + tracing::error!("Received unrecognized item kind `{k}` with path: `{}` while computing diff of {fork_workspace_id} and {source_workspace_id} workspaces. Skipping this item", item.path); + None + // Some(ItemComparison { + // has_changes: true, + // exists_in_source: true, + // exists_in_fork: true, + // }) + } + }; + + if let Some(item_comparison) = item_comparison { + if item_comparison.has_changes { + sqlx::query!( + "UPDATE workspace_diff SET has_changes = true, exists_in_source = $5, exists_in_fork = $6 + WHERE path = $3 AND kind = $4 AND ( + (source_workspace_id = $1 AND fork_workspace_id = $2) + OR (source_workspace_id = $2 AND fork_workspace_id =$1) + )", + source_workspace_id, + fork_workspace_id, + item.path, + item.kind, + item_comparison.exists_in_source, + item_comparison.exists_in_fork, + ) + .execute(&db) + .await?; + confirmed_diffs.push(WorkspaceDiffRow { + has_changes: Some(item_comparison.has_changes), + exists_in_source: Some(item_comparison.exists_in_source), + exists_in_fork: Some(item_comparison.exists_in_fork), + ..item + }); + } else { + sqlx::query!( + "DELETE FROM workspace_diff WHERE path = $3 AND kind = $4 AND ( + (source_workspace_id = $1 AND fork_workspace_id = $2) + OR (source_workspace_id = $2 AND fork_workspace_id =$1) + )", + source_workspace_id, + fork_workspace_id, + item.path, + item.kind, + ) + .execute(&db) + .await?; + } + } + } + + let visible_diffs = filter_visible_diffs( + &confirmed_diffs, + &source_workspace_id, + &fork_workspace_id, + user_db.begin(&authed).await?, + ) + .await?; + + let summary = CompareSummary { + total_diffs: visible_diffs.len(), + total_ahead: visible_diffs + .iter() + .map(|s| s.ahead) + .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)), + total_behind: visible_diffs + .iter() + .map(|s| s.behind) + .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)), + scripts_changed: visible_diffs.iter().filter(|s| s.kind == "script").count(), + flows_changed: visible_diffs.iter().filter(|s| s.kind == "flow").count(), + apps_changed: visible_diffs.iter().filter(|s| s.kind == "app").count(), + resources_changed: visible_diffs + .iter() + .filter(|s| s.kind == "resource") + .count(), + variables_changed: visible_diffs + .iter() + .filter(|s| s.kind == "variable") + .count(), + resource_types_changed: visible_diffs + .iter() + .filter(|s| s.kind == "resource_type") + .count(), + folders_changed: visible_diffs.iter().filter(|s| s.kind == "folder").count(), + conflicts: visible_diffs + .iter() + .filter(|s| s.ahead > 0 && s.behind > 0) + .count(), + }; + + let all_ahead_items_visible = summary.total_ahead + == confirmed_diffs + .iter() + .map(|s| s.ahead) + .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)); + let all_behind_items_visible = summary.total_behind + == confirmed_diffs + .iter() + .map(|s| s.behind) + .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)); + + return Ok(Json(WorkspaceComparison { + all_ahead_items_visible, + all_behind_items_visible, + skipped_comparison: false, + diffs: visible_diffs, + summary, + })); +} + +async fn filter_visible_diffs( + confirmed_diffs: &[WorkspaceDiffRow], + source_workspace_id: &str, + fork_workspace_id: &str, + mut tx: Transaction<'static, Postgres>, +) -> Result> { + // Step 1: Group paths by (workspace, kind) + let mut source_items: HashMap<&str, Vec<&str>> = HashMap::new(); + let mut fork_items: HashMap<&str, Vec<&str>> = HashMap::new(); + + for diff in confirmed_diffs { + if diff.exists_in_source.unwrap_or(false) { + source_items.entry(&diff.kind).or_default().push(&diff.path); + } + if diff.exists_in_fork.unwrap_or(false) { + fork_items.entry(&diff.kind).or_default().push(&diff.path); + } + } + + // Step 2: Batch query for each (workspace, kind) combination + let source_visible = query_visible_items(&mut tx, source_workspace_id, &source_items).await?; + let fork_visible = query_visible_items(&mut tx, fork_workspace_id, &fork_items).await?; + + // Step 3: Filter diffs based on visibility + let visible_diffs: Vec = confirmed_diffs + .iter() + .filter(|diff| { + let v = (diff.kind.to_string(), diff.path.to_string()); + let source_ok = !diff.exists_in_source.unwrap_or(false) || source_visible.contains(&v); + let fork_ok = !diff.exists_in_fork.unwrap_or(false) || fork_visible.contains(&v); + source_ok && fork_ok + }) + .cloned() + .collect(); + + Ok(visible_diffs) +} + +async fn query_visible_items<'c>( + tx: &mut Transaction<'c, Postgres>, + workspace_id: &str, + items_by_kind: &HashMap<&str, Vec<&str>>, +) -> Result> { + let mut visible = HashSet::new(); + + for (kind, paths) in items_by_kind { + let paths_vec: Vec = paths.iter().map(|s| s.to_string()).collect(); + + let results = match *kind { + "script" => { + sqlx::query_scalar!( + "SELECT path FROM script + WHERE workspace_id = $1 AND path = ANY($2) AND archived = false", + workspace_id, + &paths_vec + ) + .fetch_all(&mut **tx) + .await? + } + "flow" => { + sqlx::query_scalar!( + "SELECT path FROM flow + WHERE workspace_id = $1 AND path = ANY($2) AND archived = false", + workspace_id, + &paths_vec + ) + .fetch_all(&mut **tx) + .await? + } + "app" => { + sqlx::query_scalar!( + "SELECT path FROM app + WHERE workspace_id = $1 AND path = ANY($2)", + workspace_id, + &paths_vec + ) + .fetch_all(&mut **tx) + .await? + } + "resource" => { + sqlx::query_scalar!( + "SELECT path FROM resource + WHERE workspace_id = $1 AND path = ANY($2)", + workspace_id, + &paths_vec + ) + .fetch_all(&mut **tx) + .await? + } + "variable" => { + sqlx::query_scalar!( + "SELECT path FROM variable + WHERE workspace_id = $1 AND path = ANY($2)", + workspace_id, + &paths_vec + ) + .fetch_all(&mut **tx) + .await? + } + "folder" => { + let a: Vec = paths_vec + .iter() + .map(|p| p.strip_prefix("f/").unwrap_or(p.as_str()).to_string()) + .collect(); + sqlx::query_scalar!( + "SELECT name FROM folder + WHERE workspace_id = $1 AND name = ANY($2)", + workspace_id, + &a, + ) + .fetch_all(&mut **tx) + .await? + .into_iter() + .map(|p| format!("f/{p}")) + .collect() + } + "resource_type" => { + sqlx::query_scalar!( + "SELECT name FROM resource_type + WHERE workspace_id = $1 AND name = ANY($2)", + workspace_id, + &paths_vec + ) + .fetch_all(&mut **tx) + .await? + } + _ => vec![], // Unknown kind + }; + + for path in results { + visible.insert((kind.to_string(), path)); + } + } + + Ok(visible) +} + +#[derive(Debug)] +struct ItemComparison { + has_changes: bool, + exists_in_source: bool, + exists_in_fork: bool, +} + +async fn compare_two_scripts( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + // Get latest script from each workspace + let source_script = sqlx::query!( + "SELECT hash, created_at, content, summary, description, lock, schema + FROM script + WHERE workspace_id = $1 AND path = $2 AND archived = false + ORDER BY created_at DESC + LIMIT 1", + source_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let target_script = sqlx::query!( + "SELECT hash, created_at, content, summary, description, lock, schema + FROM script + WHERE workspace_id = $1 AND path = $2 AND archived = false + ORDER BY created_at DESC + LIMIT 1", + fork_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata differences + if let (Some(source), Some(target)) = (&source_script, &target_script) { + if source.content != target.content + || source.summary != target.summary + || source.description != target.description + || source.lock != target.lock + || source.schema != target.schema + { + has_changes = true; + } + } else if source_script.is_some() || target_script.is_some() { + // The script exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_script.is_some(), + exists_in_fork: target_script.is_some(), + }); +} + +async fn compare_two_flows( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + // Get latest flow from each workspace + let source_flow = sqlx::query!( + "SELECT value, summary, description, schema + FROM flow + WHERE workspace_id = $1 AND path = $2 AND archived = false", + source_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let target_flow = sqlx::query!( + "SELECT value, summary, description, schema + FROM flow + WHERE workspace_id = $1 AND path = $2 AND archived = false", + fork_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata differences + if let (Some(source), Some(target)) = (&source_flow, &target_flow) { + if source.value != target.value + || source.summary != target.summary + || source.description != target.description + || source.schema != target.schema + { + has_changes = true; + } + } else if source_flow.is_some() || target_flow.is_some() { + // The flow exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_flow.is_some(), + exists_in_fork: target_flow.is_some(), + }); +} + +async fn compare_two_apps( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + // Get app with its latest version data from source workspace + let source_app = sqlx::query!( + "SELECT app.summary, app.policy, app_version.value + FROM app + JOIN app_version + ON app_version.id = app.versions[array_upper(app.versions, 1)] + WHERE app.workspace_id = $1 AND app.path = $2 AND COALESCE(app.draft_only, false) = false", + source_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let target_app = sqlx::query!( + "SELECT app.summary, app.policy, app_version.value + FROM app + JOIN app_version + ON app_version.id = app.versions[array_upper(app.versions, 1)] + WHERE app.workspace_id = $1 AND app.path = $2 AND COALESCE(app.draft_only, false) = false", + fork_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata and content differences + if let (Some(source), Some(target)) = (&source_app, &target_app) { + if source.summary != target.summary + || source.policy != target.policy + || source.value != target.value + { + has_changes = true; + } + } else if source_app.is_some() || target_app.is_some() { + // The app exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_app.is_some(), + exists_in_fork: target_app.is_some(), + }); +} + +async fn compare_two_resources( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + // Get resource from each workspace + let source_resource = sqlx::query!( + "SELECT value, description, resource_type + FROM resource + WHERE workspace_id = $1 AND path = $2", + source_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let target_resource = sqlx::query!( + "SELECT value, description, resource_type + FROM resource + WHERE workspace_id = $1 AND path = $2", + fork_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata differences + if let (Some(source), Some(target)) = (&source_resource, &target_resource) { + if source.value != target.value + || source.description != target.description + || source.resource_type != target.resource_type + { + has_changes = true; + } + } else if source_resource.is_some() || target_resource.is_some() { + // The resource exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_resource.is_some(), + exists_in_fork: target_resource.is_some(), + }); +} + +async fn compare_two_variables( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + path: &str, +) -> Result { + // Get variable from each workspace + let source_variable = sqlx::query!( + "SELECT value, is_secret, description + FROM variable + WHERE workspace_id = $1 AND path = $2", + source_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let target_variable = sqlx::query!( + "SELECT value, is_secret, description + FROM variable + WHERE workspace_id = $1 AND path = $2", + fork_workspace_id, + path + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata differences + if let (Some(source), Some(target)) = (&source_variable, &target_variable) { + if source.is_secret != target.is_secret + || source.value != target.value + || source.description != target.description + { + has_changes = true; + } + } else if source_variable.is_some() || target_variable.is_some() { + // The variable exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_variable.is_some(), + exists_in_fork: target_variable.is_some(), + }); +} + +async fn compare_two_resource_types( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + name: &str, +) -> Result { + // Get resource type from each workspace + let source_resource_type = sqlx::query!( + "SELECT schema, description, format_extension + FROM resource_type + WHERE workspace_id = $1 AND name = $2", + source_workspace_id, + name + ) + .fetch_optional(db) + .await?; + + let target_resource_type = sqlx::query!( + "SELECT schema, description, format_extension + FROM resource_type + WHERE workspace_id = $1 AND name = $2", + fork_workspace_id, + name + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata differences + if let (Some(source), Some(target)) = (&source_resource_type, &target_resource_type) { + if source.schema != target.schema + || source.description != target.description + || source.format_extension != target.format_extension + { + has_changes = true; + } + } else if source_resource_type.is_some() || target_resource_type.is_some() { + // The resource type exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_resource_type.is_some(), + exists_in_fork: target_resource_type.is_some(), + }); +} + +async fn compare_two_folders( + db: &DB, + source_workspace_id: &str, + fork_workspace_id: &str, + name: &str, +) -> Result { + // Get folder from each workspace + let source_folder = sqlx::query!( + "SELECT display_name, owners, extra_perms, summary + FROM folder + WHERE workspace_id = $1 AND name = $2", + source_workspace_id, + name.strip_prefix("f/"), + ) + .fetch_optional(db) + .await?; + + let target_folder = sqlx::query!( + "SELECT display_name, owners, extra_perms, summary + FROM folder + WHERE workspace_id = $1 AND name = $2", + fork_workspace_id, + name.strip_prefix("f/"), + ) + .fetch_optional(db) + .await?; + + let mut has_changes = false; + + // Check metadata differences + if let (Some(source), Some(target)) = (&source_folder, &target_folder) { + if source.display_name != target.display_name + || source.owners != target.owners + || source.extra_perms != target.extra_perms + || source.summary != target.summary + { + has_changes = true; + } + } else if source_folder.is_some() || target_folder.is_some() { + // The folder exists in one of source or target, but not the other, this is considered as a change + has_changes = true + } + + return Ok(ItemComparison { + has_changes, + exists_in_source: source_folder.is_some(), + exists_in_fork: target_folder.is_some(), + }); +} diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs similarity index 99% rename from backend/windmill-api/src/workspaces_extra.rs rename to backend/windmill-api-workspaces/src/workspaces_extra.rs index 79b9554ebf..b6b1cceae9 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; -use crate::db::ApiAuthed; +use windmill_api_auth::{require_super_admin, ApiAuthed}; +use windmill_common::DB; use crate::workspaces::{ archive_workspace_impl, check_w_id_conflict, CREATE_WORKSPACE_REQUIRE_SUPERADMIN, WM_FORK_PREFIX, }; -use crate::{db::DB, utils::require_super_admin}; use axum::extract::Query; use axum::{ @@ -487,7 +487,7 @@ pub(crate) async fn change_workspace_id( ); for schedule_path in &enabled_schedule_paths { - crate::schedule::clear_schedule(&mut tx, schedule_path, &old_id).await?; + windmill_queue::schedule::clear_schedule(&mut tx, schedule_path, &old_id).await?; } // Move queued jobs (not running) to new workspace using skip lock diff --git a/backend/windmill-api/src/workspaces_oss.rs b/backend/windmill-api-workspaces/src/workspaces_oss.rs similarity index 71% rename from backend/windmill-api/src/workspaces_oss.rs rename to backend/windmill-api-workspaces/src/workspaces_oss.rs index 4f55539da1..da46622c26 100644 --- a/backend/windmill-api/src/workspaces_oss.rs +++ b/backend/windmill-api-workspaces/src/workspaces_oss.rs @@ -3,10 +3,11 @@ pub use crate::workspaces_ee::*; #[cfg(not(feature = "private"))] -use crate::{ - db::{ApiAuthed, DB}, - workspaces::EditAutoInvite, -}; +use crate::workspaces::EditAutoInvite; +#[cfg(not(feature = "private"))] +use windmill_api_auth::ApiAuthed; +#[cfg(not(feature = "private"))] +use windmill_common::DB; #[cfg(not(feature = "private"))] pub async fn edit_auto_invite( diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index f0672666d0..bbf1492d84 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,51 +10,64 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] stripe = [] -agent_worker_server = [] +inline_preview = ["dep:windmill-worker"] +agent_worker_server = ["dep:windmill-worker"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] -embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] -parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"] -prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] -openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"] +embedding = ["windmill-api-embeddings/embedding"] +parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] +prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus"] +openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] tantivy = ["dep:windmill-indexer"] -kafka = ["dep:rdkafka", "dep:rdkafka-sys"] -nats = ["dep:async-nats", "dep:nkeys"] -websocket = ["dep:tokio-tungstenite"] -smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp"] +kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] +kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] +nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] +websocket = ["dep:windmill-trigger-websocket"] +smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email"] license = ["dep:rsa"] zip = ["dep:async_zip"] -oauth2 = ["dep:windmill-oauth"] -http_trigger = ["dep:matchit", "dep:thiserror", "dep:sha1", "dep:constant_time_eq"] +oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"] +http_trigger = ["dep:matchit", "dep:windmill-trigger-http", "windmill-store/http_trigger"] static_frontend = ["dep:rust-embed"] -postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"] -mqtt_trigger = ["dep:thiserror", "dep:rumqttc"] -native_trigger = ["dep:strum", "dep:backon", "oauth2"] -sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config"] -deno_core = ["dep:deno_core", "dep:deno_error"] -gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] -cloud = ["windmill-common/cloud"] -mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] -bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config"] -python = [] +postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-store/postgres_trigger"] +mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-store/mqtt_trigger"] +native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "dep:strum", "oauth2"] +sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-store/sqs_trigger"] +gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-store/gcp_trigger"] +cloud = ["windmill-common/cloud", "windmill-api-auth/cloud", "windmill-store/cloud", "windmill-api-workspaces/cloud"] +mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth", "windmill-api-auth/mcp", "windmill-store/mcp"] +bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"] +python = ["windmill-dep-map/python", "dep:windmill-parser-py", "dep:windmill-parser-py-imports", "windmill-api-scripts/python", "windmill-trigger/python", "windmill-common/python"] +no_auth = ["windmill-api-auth/no_auth", "windmill-store/no_auth", "windmill-api-users/no_auth"] +quickjs = ["windmill-jseval/quickjs"] [dependencies] windmill-mcp = { workspace = true, optional = true } +windmill-api-auth.workspace = true +windmill-api-scripts.workspace = true +windmill-api-users.workspace = true +windmill-api-workspaces.workspace = true +windmill-api-groups.workspace = true +windmill-api-sse.workspace = true +windmill-api-jobs.workspace = true +windmill-trigger.workspace = true +windmill-store.workspace = true windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true windmill-parser.workspace = true windmill-parser-sql.workspace = true windmill-parser-ts.workspace = true -windmill-parser-py.workspace = true -windmill-parser-py-imports.workspace = true +windmill-parser-py = { workspace = true, optional = true } +windmill-parser-py-imports = { workspace = true, optional = true } windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } -windmill-autoscaling.workspace = true -windmill-worker.workspace = true +windmill-autoscaling = { workspace = true, optional = true } +windmill-worker = { workspace = true, optional = true } +windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true anyhow.workspace = true @@ -102,8 +115,6 @@ postgres-native-tls.workspace = true hmac.workspace = true cookie.workspace = true sha2.workspace = true -sha1 = { workspace = true, optional = true } -constant_time_eq = { workspace = true, optional = true } urlencoding.workspace = true lazy_static.workspace = true prometheus = { workspace = true, optional = true } @@ -116,60 +127,42 @@ libxml = { workspace = true, optional = true } async-recursion.workspace = true rsa = { workspace = true, optional = true} uuid.workspace = true -tinyvector = { workspace = true, optional = true} -hf-hub = { workspace = true, optional = true} -tokenizers = { workspace = true, optional = true} -candle-core = { workspace = true, optional = true} -candle-transformers = { workspace = true, optional = true} -candle-nn = { workspace = true, optional = true} datafusion = { workspace = true, optional = true} object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} jsonwebtoken = { workspace = true } matchit = { workspace = true, optional = true } -tokio-tungstenite = { workspace = true, optional = true} -rdkafka = { workspace = true, optional = true } -rdkafka-sys = { workspace = true, optional = true } - -async-nats = { workspace = true, optional = true } -nkeys = { workspace = true, optional = true } +windmill-trigger-kafka = { workspace = true, optional = true } +windmill-trigger-postgres = { workspace = true, optional = true } +windmill-trigger-mqtt = { workspace = true, optional = true } +windmill-trigger-websocket = { workspace = true, optional = true } +windmill-trigger-email = { workspace = true, optional = true } +windmill-trigger-nats = { workspace = true, optional = true } +windmill-trigger-sqs = { workspace = true, optional = true } +windmill-trigger-gcp = { workspace = true, optional = true } +windmill-trigger-http = { workspace = true, optional = true } +windmill-native-triggers = { workspace = true, optional = true } +windmill-api-embeddings.workspace = true const_format.workspace = true pin-project.workspace = true http.workspace = true indexmap.workspace = true async-stream.workspace = true ulid.workspace = true -rust-postgres = { workspace = true, optional = true } -pg_escape = { workspace = true, optional = true } -byteorder = { workspace = true, optional = true } -thiserror = { workspace = true, optional = true } -rust_decimal = { workspace = true, optional = true } -rust-postgres-native-tls = { workspace = true, optional = true} -rumqttc = { workspace = true, optional = true } -aws-sdk-sqs = { workspace = true, optional = true } -aws-sdk-sso = { workspace = true, optional = true } -aws-sdk-ssooidc = { workspace = true, optional = true } -aws-sdk-sts = { workspace = true, optional = true } rustls = { workspace = true } -aws-sigv4.workspace = true -aws-sdk-config.workspace = true +aws-sigv4 = { workspace = true, optional = true } +aws-sdk-config = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } -aws-credential-types.workspace = true +aws-credential-types = { workspace = true, optional = true } aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } -aws-smithy-types.workspace = true +aws-smithy-types = { workspace = true, optional = true } async-trait.workspace = true -google-cloud-pubsub = { workspace = true, optional = true } -google-cloud-googleapis = { workspace = true , optional = true } -tonic = { workspace = true, optional = true } -deno_error = { workspace = true, optional = true } -deno_core = { workspace = true, optional = true } +windmill-jseval.workspace = true tar.workspace = true flate2.workspace = true -backon = {workspace = true, optional = true} strum = { workspace = true, optional = true } dashmap.workspace = true [build-dependencies] -deno_core = { workspace = true, optional = true } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 03dd11b36b..0047421126 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.625.0 + version: 1.629.1 title: Windmill API contact: @@ -1310,6 +1310,20 @@ paths: schema: type: string + /settings/get_stats: + get: + summary: get encrypted telemetry stats (EE only) + operationId: getStats + tags: + - setting + responses: + "200": + description: base64-encoded encrypted telemetry blob + content: + text/plain: + schema: + type: string + /settings/latest_key_renewal_attempt: get: summary: get latest key renewal attempt @@ -7769,23 +7783,6 @@ paths: items: $ref: "#/components/schemas/ListableRawApp" - /w/{workspace}/raw_apps/exists/{path}: - get: - summary: does an app exisst at path - operationId: existsRawApp - tags: - - raw_app - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/Path" - responses: - "200": - description: app exists - content: - application/json: - schema: - type: boolean - /w/{workspace}/apps/get_data/v/{secretWithExtension}: get: summary: get raw app data by @@ -8215,88 +8212,6 @@ paths: schema: $ref: "#/components/schemas/AppWithLastVersion" - /w/{workspace}/raw_apps/create: - post: - summary: create raw app - operationId: createRawApp - tags: - - raw_app - parameters: - - $ref: "#/components/parameters/WorkspaceId" - requestBody: - description: new raw app - required: true - content: - application/json: - schema: - type: object - properties: - path: - type: string - value: - type: string - summary: - type: string - required: - - path - - value - - summary - responses: - "201": - description: raw app created - content: - text/plain: - schema: - type: string - - /w/{workspace}/raw_apps/update/{path}: - post: - summary: update app - operationId: updateRawApp - tags: - - raw_app - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/ScriptPath" - requestBody: - description: updateraw app - required: true - content: - application/json: - schema: - type: object - properties: - path: - type: string - summary: - type: string - value: - type: string - responses: - "200": - description: app updated - content: - text/plain: - schema: - type: string - - /w/{workspace}/raw_apps/delete/{path}: - delete: - summary: delete raw app - operationId: deleteRawApp - tags: - - raw_app - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/Path" - responses: - "200": - description: app deleted - content: - text/plain: - schema: - type: string - /w/{workspace}/apps/delete/{path}: delete: summary: delete app @@ -17264,6 +17179,12 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/StaticTransform" JavascriptTransform: $ref: "../../openflow.openapi.yaml#/components/schemas/JavascriptTransform" + AiTransform: + $ref: "../../openflow.openapi.yaml#/components/schemas/AiTransform" + ProviderConfig: + $ref: "../../openflow.openapi.yaml#/components/schemas/ProviderConfig" + MemoryConfig: + $ref: "../../openflow.openapi.yaml#/components/schemas/MemoryConfig" FlowModuleValue: $ref: "../../openflow.openapi.yaml#/components/schemas/FlowModuleValue" RawScript: @@ -19145,6 +19066,7 @@ components: java, ruby, duckdb, + bunnative, # for related places search: ADD_NEW_LANG ] @@ -19347,70 +19269,93 @@ components: properties: path: type: string + description: The unique path identifier for this schedule edited_by: type: string + description: Username of the last person who edited this schedule edited_at: type: string format: date-time + description: Timestamp of the last edit schedule: type: string + description: Cron expression with 6 fields (seconds, minutes, hours, day of month, month, day of week). Example '0 0 12 * * *' for daily at noon timezone: type: string + description: IANA timezone for the schedule (e.g., 'UTC', 'Europe/Paris', 'America/New_York') enabled: type: boolean + description: Whether the schedule is currently active and will trigger jobs script_path: type: string + description: Path to the script or flow to execute when triggered is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script args: $ref: "#/components/schemas/ScriptArgs" extra_perms: type: object additionalProperties: type: boolean + description: Additional permissions for this schedule email: type: string + description: Email of the user who owns this schedule, used for permissioned_as error: type: string + description: Last error message if the schedule failed to trigger on_failure: - # a reference to a script path, flow path, or webhook (script/, flow/) type: string + description: Path to a script or flow to run when the scheduled job fails on_failure_times: type: number + description: Number of consecutive failures before the on_failure handler is triggered (default 1) on_failure_exact: type: boolean + description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N on_failure_extra_args: $ref: "#/components/schemas/ScriptArgs" on_recovery: type: string + description: Path to a script or flow to run when the schedule recovers after failures on_recovery_times: type: number + description: Number of consecutive successes before the on_recovery handler is triggered (default 1) on_recovery_extra_args: $ref: "#/components/schemas/ScriptArgs" on_success: type: string + description: Path to a script or flow to run after each successful execution on_success_extra_args: $ref: "#/components/schemas/ScriptArgs" ws_error_handler_muted: type: boolean + description: If true, the workspace-level error handler will not be triggered for this schedule's failures retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" summary: type: string + description: Short summary describing the purpose of this schedule description: type: string + description: Detailed description of what this schedule does no_flow_overlap: type: boolean + description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs) tag: type: string + description: Worker tag to route jobs to specific worker groups paused_until: type: string format: date-time + description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time cron_version: type: string + description: Cron parser version. Use 'v2' for extended syntax with additional features dynamic_skip: type: string - description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean. + description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false) required: - path - edited_by @@ -19457,81 +19402,75 @@ components: properties: path: type: string - description: The path where the schedule will be created + description: The unique path identifier for this schedule schedule: type: string - description: The cron schedule to trigger the script or flow. Should include seconds. + description: Cron expression with 6 fields (seconds, minutes, hours, day of month, month, day of week). Example '0 0 12 * * *' for daily at noon timezone: type: string - description: The timezone to use for the cron schedule + description: IANA timezone for the schedule (e.g., 'UTC', 'Europe/Paris', 'America/New_York') script_path: type: string - description: The path to the script or flow to trigger + description: Path to the script or flow to execute when triggered is_flow: type: boolean - description: Whether the schedule is for a flow + description: True if script_path points to a flow, false if it points to a script args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow enabled: type: boolean - description: Whether the schedule is enabled + description: Whether the schedule is currently active and will trigger jobs on_failure: - # a reference to a script path, flow path, or webhook (script/, flow/) type: string - description: The path to the script or flow to trigger on failure + description: Path to a script or flow to run when the scheduled job fails on_failure_times: type: number - description: The number of times to retry on failure + description: Number of consecutive failures before the on_failure handler is triggered (default 1) on_failure_exact: type: boolean - description: Whether the schedule should only run on the exact time + description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N on_failure_extra_args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on failure on_recovery: type: string - description: The path to the script or flow to trigger on recovery + description: Path to a script or flow to run when the schedule recovers after failures on_recovery_times: type: number - description: The number of times to retry on recovery + description: Number of consecutive successes before the on_recovery handler is triggered (default 1) on_recovery_extra_args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on recovery on_success: type: string - description: The path to the script or flow to trigger on success + description: Path to a script or flow to run after each successful execution on_success_extra_args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on success ws_error_handler_muted: type: boolean - description: Whether the WebSocket error handler is muted + description: If true, the workspace-level error handler will not be triggered for this schedule's failures retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" - description: The retry configuration for the schedule no_flow_overlap: type: boolean - description: Whether the schedule should not run if a flow is already running + description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs) summary: type: string - description: The summary of the schedule + description: Short summary describing the purpose of this schedule description: type: string - description: The description of the schedule + description: Detailed description of what this schedule does tag: type: string - description: The tag of the schedule + description: Worker tag to route jobs to specific worker groups paused_until: type: string - description: The date and time the schedule will be paused until format: date-time + description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time cron_version: type: string - description: The version of the cron schedule to use (last is v2) + description: Cron parser version. Use 'v2' for extended syntax with additional features dynamic_skip: type: string - description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean. + description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false) required: - path - schedule @@ -19545,69 +19484,63 @@ components: properties: schedule: type: string - description: The cron schedule to trigger the script or flow. Should include seconds. + description: Cron expression with 6 fields (seconds, minutes, hours, day of month, month, day of week). Example '0 0 12 * * *' for daily at noon timezone: type: string - description: The timezone to use for the cron schedule + description: IANA timezone for the schedule (e.g., 'UTC', 'Europe/Paris', 'America/New_York') args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on_failure: - # a reference to a script path, flow path, or webhook (script/, flow/) type: string - description: The path to the script or flow to trigger on failure + description: Path to a script or flow to run when the scheduled job fails on_failure_times: type: number - description: The number of times to retry on failure + description: Number of consecutive failures before the on_failure handler is triggered (default 1) on_failure_exact: type: boolean - description: Whether the schedule should only run on the exact time + description: If true, trigger on_failure handler only on exactly N failures, not on every failure after N on_failure_extra_args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on failure on_recovery: type: string - description: The path to the script or flow to trigger on recovery + description: Path to a script or flow to run when the schedule recovers after failures on_recovery_times: type: number - description: The number of times to retry on recovery + description: Number of consecutive successes before the on_recovery handler is triggered (default 1) on_recovery_extra_args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on recovery on_success: type: string - description: The path to the script or flow to trigger on success + description: Path to a script or flow to run after each successful execution on_success_extra_args: $ref: "#/components/schemas/ScriptArgs" - description: The arguments to pass to the script or flow on success ws_error_handler_muted: type: boolean - description: Whether the WebSocket error handler is muted + description: If true, the workspace-level error handler will not be triggered for this schedule's failures retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" - description: The retry configuration for the schedule no_flow_overlap: type: boolean - description: Whether the schedule should not run if a flow is already running + description: If true, skip this schedule's execution if the previous run is still in progress (prevents concurrent runs) summary: type: string - description: The summary of the schedule + description: Short summary describing the purpose of this schedule description: type: string - description: The description of the schedule + description: Detailed description of what this schedule does tag: type: string - description: The tag of the schedule + description: Worker tag to route jobs to specific worker groups paused_until: type: string - description: The date and time the schedule will be paused until format: date-time + description: ISO 8601 datetime until which the schedule is paused. Schedule resumes automatically after this time cron_version: type: string - description: The version of the cron schedule to use (last is v2) + description: Cron parser version. Use 'v2' for extended syntax with additional features dynamic_skip: type: string - description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean. + description: Path to a script that validates scheduled datetimes. Receives scheduled_for datetime and returns boolean to skip (true) or run (false) required: - schedule - timezone @@ -19643,25 +19576,34 @@ components: properties: path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when triggered email: type: string + description: Email of the user who owns this trigger, used for permissioned_as extra_perms: type: object + description: Additional permissions for this trigger additionalProperties: type: boolean workspace_id: type: string + description: The workspace this trigger belongs to edited_by: type: string + description: Username of the last person who edited this trigger edited_at: type: string format: date-time + description: Timestamp of the last edit is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" + description: Trigger mode (enabled/disabled) required: - path - script_path @@ -19806,43 +19748,61 @@ components: properties: route_path: type: string + description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /. static_asset_config: type: object + description: Configuration for serving static assets (s3 bucket, storage path, filename) properties: s3: type: string + description: S3 bucket path for static assets storage: type: string + description: Storage path for static assets filename: type: string + description: Filename for the static asset required: - s3 http_method: $ref: "#/components/schemas/HttpMethod" + description: HTTP method (get, post, put, delete, patch) that triggers this endpoint authentication_resource_path: type: string + description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods) summary: type: string + description: Short summary describing the purpose of this trigger description: type: string + description: Detailed description of what this trigger does request_type: $ref: "#/components/schemas/HttpRequestType" + description: "How the request is handled - 'sync' waits for result, 'async' returns job ID immediately, 'sync_sse' streams results via Server-Sent Events" authentication_method: $ref: "#/components/schemas/AuthenticationMethod" + description: "How requests are authenticated - 'none' (public), 'windmill' (Windmill token), 'api_key', 'basic_http', 'custom_script', 'signature'" is_static_website: type: boolean + description: If true, serves static files from S3/storage instead of running a script workspaced_route: type: boolean + description: If true, the route includes the workspace ID in the path wrap_body: type: boolean + description: If true, wraps the request body in a 'body' parameter raw_string: type: boolean + description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - route_path @@ -19859,54 +19819,75 @@ components: properties: path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when triggered route_path: type: string + description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /. workspaced_route: type: boolean + description: If true, the route includes the workspace ID in the path summary: type: string + description: Short summary describing the purpose of this trigger description: type: string + description: Detailed description of what this trigger does static_asset_config: type: object + description: Configuration for serving static assets (s3 bucket, storage path, filename) properties: s3: type: string + description: S3 bucket path for static assets storage: type: string + description: Storage path for static assets filename: type: string + description: Filename for the static asset required: - s3 is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script http_method: $ref: "#/components/schemas/HttpMethod" + description: HTTP method (get, post, put, delete, patch) that triggers this endpoint authentication_resource_path: type: string + description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods) is_async: type: boolean description: Deprecated, use request_type instead request_type: $ref: "#/components/schemas/HttpRequestType" + description: "How the request is handled - 'sync' waits for result, 'async' returns job ID immediately, 'sync_sse' streams results via Server-Sent Events" authentication_method: $ref: "#/components/schemas/AuthenticationMethod" + description: "How requests are authenticated - 'none' (public), 'windmill' (Windmill token), 'api_key', 'basic_http', 'custom_script', 'signature'" is_static_website: type: boolean + description: If true, serves static files from S3/storage instead of running a script wrap_body: type: boolean + description: If true, wraps the request body in a 'body' parameter mode: $ref: "#/components/schemas/TriggerMode" raw_string: type: boolean + description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path @@ -19922,52 +19903,73 @@ components: properties: path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when triggered route_path: type: string + description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). Must NOT start with a /. summary: type: string + description: Short summary describing the purpose of this trigger description: type: string + description: Detailed description of what this trigger does workspaced_route: type: boolean + description: If true, the route includes the workspace ID in the path static_asset_config: type: object + description: Configuration for serving static assets (s3 bucket, storage path, filename) properties: s3: type: string + description: S3 bucket path for static assets storage: type: string + description: Storage path for static assets filename: type: string + description: Filename for the static asset required: - s3 authentication_resource_path: type: string + description: Path to the resource containing authentication configuration (for api_key, basic_http, custom_script, signature methods) is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script http_method: $ref: "#/components/schemas/HttpMethod" + description: HTTP method (get, post, put, delete, patch) that triggers this endpoint is_async: type: boolean description: Deprecated, use request_type instead request_type: $ref: "#/components/schemas/HttpRequestType" + description: "How the request is handled - 'sync' waits for result, 'async' returns job ID immediately, 'sync_sse' streams results via Server-Sent Events" authentication_method: $ref: "#/components/schemas/AuthenticationMethod" + description: "How requests are authenticated - 'none' (public), 'windmill' (Windmill token), 'api_key', 'basic_http', 'custom_script', 'signature'" is_static_website: type: boolean + description: If true, serves static files from S3/storage instead of running a script wrap_body: type: boolean + description: If true, wraps the request body in a 'body' parameter raw_string: type: boolean + description: If true, passes the request body as a raw string instead of parsing as JSON error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path - script_path @@ -20019,15 +20021,20 @@ components: properties: url: type: string + description: The WebSocket URL to connect to (can be a static URL or computed by a runnable) server_id: type: string + description: ID of the server currently handling this trigger (internal) last_server_ping: type: string format: date-time + description: Timestamp of last server heartbeat (internal) error: type: string + description: Last error message if the trigger failed filters: type: array + description: Array of key-value filters to match incoming messages (only matching messages trigger the script) items: type: object properties: @@ -20039,19 +20046,26 @@ components: - value initial_messages: type: array + description: Messages to send immediately after connecting (can be raw strings or computed by runnables) items: $ref: "#/components/schemas/WebsocketTriggerInitialMessage" url_runnable_args: + description: Arguments to pass to the script/flow that computes the WebSocket URL $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean + description: If true, the script can return a message to send back through the WebSocket can_return_error_result: type: boolean + description: If true, error results are sent back through the WebSocket error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: + description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" retry: + description: Retry configuration for failed executions $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" required: @@ -20065,16 +20079,21 @@ components: properties: path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script url: type: string + description: The WebSocket URL to connect to (can be a static URL or computed by a runnable) mode: $ref: "#/components/schemas/TriggerMode" filters: type: array + description: Array of key-value filters to match incoming messages (only matching messages trigger the script) items: type: object properties: @@ -20086,19 +20105,26 @@ components: - value initial_messages: type: array + description: Messages to send immediately after connecting (can be raw strings or computed by runnables) items: $ref: "#/components/schemas/WebsocketTriggerInitialMessage" url_runnable_args: + description: Arguments to pass to the script/flow that computes the WebSocket URL $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean + description: If true, the script can return a message to send back through the WebSocket can_return_error_result: type: boolean + description: If true, error results are sent back through the WebSocket error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: + description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" retry: + description: Retry configuration for failed executions $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" required: @@ -20115,14 +20141,19 @@ components: properties: url: type: string + description: The WebSocket URL to connect to (can be a static URL or computed by a runnable) path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script filters: type: array + description: Array of key-value filters to match incoming messages (only matching messages trigger the script) items: type: object properties: @@ -20134,19 +20165,26 @@ components: - value initial_messages: type: array + description: Messages to send immediately after connecting (can be raw strings or computed by runnables) items: $ref: "#/components/schemas/WebsocketTriggerInitialMessage" url_runnable_args: + description: Arguments to pass to the script/flow that computes the WebSocket URL $ref: "#/components/schemas/ScriptArgs" can_return_message: type: boolean + description: If true, the script can return a message to send back through the WebSocket can_return_error_result: type: boolean + description: If true, error results are sent back through the WebSocket error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: + description: Arguments to pass to the error handler $ref: "#/components/schemas/ScriptArgs" retry: + description: Retry configuration for failed executions $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" required: @@ -20225,31 +20263,43 @@ components: properties: mqtt_resource_path: type: string + description: Path to the MQTT resource containing broker connection configuration subscribe_topics: type: array items: $ref: "#/components/schemas/MqttSubscribeTopic" + description: Array of MQTT topics to subscribe to, each with topic name and QoS level v3_config: $ref: "#/components/schemas/MqttV3Config" + description: MQTT v3 specific configuration (clean_session) v5_config: $ref: "#/components/schemas/MqttV5Config" + description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval) client_id: type: string + description: MQTT client ID for this connection client_version: $ref: "#/components/schemas/MqttClientVersion" + description: MQTT protocol version ('v3' or 'v5') server_id: type: string + description: ID of the server currently handling this trigger (internal) last_server_ping: type: string format: date-time + description: Timestamp of last server heartbeat (internal) error: type: string + description: Last error message if the trigger failed error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - subscribe_topics - mqtt_resource_path @@ -20259,32 +20309,44 @@ components: properties: mqtt_resource_path: type: string + description: Path to the MQTT resource containing broker connection configuration subscribe_topics: type: array items: $ref: "#/components/schemas/MqttSubscribeTopic" + description: Array of MQTT topics to subscribe to, each with topic name and QoS level client_id: type: string + description: MQTT client ID for this connection v3_config: $ref: "#/components/schemas/MqttV3Config" + description: MQTT v3 specific configuration (clean_session) v5_config: $ref: "#/components/schemas/MqttV5Config" + description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval) client_version: $ref: "#/components/schemas/MqttClientVersion" + description: MQTT protocol version ('v3' or 'v5') path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path - script_path @@ -20297,32 +20359,44 @@ components: properties: mqtt_resource_path: type: string + description: Path to the MQTT resource containing broker connection configuration subscribe_topics: type: array items: $ref: "#/components/schemas/MqttSubscribeTopic" + description: Array of MQTT topics to subscribe to, each with topic name and QoS level client_id: type: string + description: MQTT client ID for this connection v3_config: $ref: "#/components/schemas/MqttV3Config" + description: MQTT v3 specific configuration (clean_session) v5_config: $ref: "#/components/schemas/MqttV5Config" + description: MQTT v5 specific configuration (clean_start, topic_alias_maximum, session_expiry_interval) client_version: $ref: "#/components/schemas/MqttClientVersion" + description: MQTT protocol version ('v3' or 'v5') path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path - script_path @@ -20336,14 +20410,18 @@ components: enum: - push - pull + description: "Delivery mode for messages. 'push' for HTTP push delivery where messages are sent to a webhook endpoint, 'pull' for polling where the trigger actively fetches messages." PushConfig: type: object + description: "Configuration for push delivery mode." properties: audience: type: string + description: "The audience claim for OIDC tokens used in push authentication." authenticate: type: boolean + description: "If true, push messages will include OIDC authentication tokens." required: - authenticate - base_endpoint @@ -20352,15 +20430,20 @@ components: allOf: - $ref: "#/components/schemas/TriggerExtraProperty" type: object + description: "A Google Cloud Pub/Sub trigger that executes a script or flow when messages are received." properties: gcp_resource_path: type: string + description: "Path to the GCP resource containing service account credentials for authentication." topic_id: type: string + description: "Google Cloud Pub/Sub topic ID to subscribe to." subscription_id: type: string + description: "Google Cloud Pub/Sub subscription ID." server_id: type: string + description: "ID of the server currently handling this trigger (internal use)." delivery_type: $ref: "#/components/schemas/DeliveryType" delivery_config: @@ -20370,14 +20453,19 @@ components: last_server_ping: type: string format: date-time + description: "Timestamp of last server heartbeat (internal use)." error: type: string + description: "Last error message if the trigger failed." error_handler_path: type: string + description: "Path to a script or flow to run when the triggered job fails." error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: "Arguments to pass to the error handler." retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: "Retry configuration for failed executions." required: - gcp_resource_path - topic_id @@ -20394,31 +20482,40 @@ components: GcpTriggerData: type: object + description: "Data for creating or updating a Google Cloud Pub/Sub trigger." properties: gcp_resource_path: type: string + description: "Path to the GCP resource containing service account credentials for authentication." subscription_mode: $ref: "#/components/schemas/SubscriptionMode" topic_id: type: string + description: "Google Cloud Pub/Sub topic ID to subscribe to." subscription_id: type: string + description: "Google Cloud Pub/Sub subscription ID." base_endpoint: type: string + description: "Base URL for push delivery endpoint." delivery_type: $ref: "#/components/schemas/DeliveryType" delivery_config: $ref: "#/components/schemas/PushConfig" path: type: string + description: "The unique path identifier for this trigger." script_path: type: string + description: "Path to the script or flow to execute when a message is received." is_flow: type: boolean + description: "True if script_path points to a flow, false if it points to a script." mode: $ref: "#/components/schemas/TriggerMode" auto_acknowledge_msg: type: boolean + description: "If true, automatically acknowledge messages after processing." ack_deadline: type: integer format: int32 @@ -20427,10 +20524,13 @@ components: description: "Time in seconds within which the message must be acknowledged. If not provided, defaults to the subscription's acknowledgment deadline (600 seconds)." error_handler_path: type: string + description: "Path to a script or flow to run when the triggered job fails." error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: "Arguments to pass to the error handler." retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: "Retry configuration for failed executions." required: - path - script_path @@ -20468,27 +20568,37 @@ components: properties: queue_url: type: string + description: The full URL of the AWS SQS queue to poll for messages aws_auth_resource_type: $ref: "#/components/schemas/AwsAuthResourceType" + description: Authentication type - 'credentials' for access key/secret, 'oidc' for OpenID Connect aws_resource_path: type: string + description: Path to the AWS resource containing credentials or OIDC configuration message_attributes: type: array items: type: string + description: Array of SQS message attribute names to include with each message server_id: type: string + description: ID of the server currently handling this trigger (internal) last_server_ping: type: string format: date-time + description: Timestamp of last server heartbeat (internal) error: type: string + description: Last error message if the trigger failed error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - queue_url @@ -20549,28 +20659,38 @@ components: properties: queue_url: type: string + description: The full URL of the AWS SQS queue to poll for messages aws_auth_resource_type: $ref: "#/components/schemas/AwsAuthResourceType" + description: Authentication type - 'credentials' for access key/secret, 'oidc' for OpenID Connect aws_resource_path: type: string + description: Path to the AWS resource containing credentials or OIDC configuration message_attributes: type: array items: type: string + description: Array of SQS message attribute names to include with each message path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - queue_url - aws_resource_path @@ -20584,28 +20704,38 @@ components: properties: queue_url: type: string + description: The full URL of the AWS SQS queue to poll for messages aws_auth_resource_type: $ref: "#/components/schemas/AwsAuthResourceType" + description: Authentication type - 'credentials' for access key/secret, 'oidc' for OpenID Connect aws_resource_path: type: string + description: Path to the AWS resource containing credentials or OIDC configuration message_attributes: type: array items: type: string + description: Array of SQS message attribute names to include with each message path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - queue_url - aws_resource_path @@ -20698,23 +20828,32 @@ components: properties: postgres_resource_path: type: string + description: Path to the PostgreSQL resource containing connection configuration publication_name: type: string + description: Name of the PostgreSQL publication to subscribe to for change data capture server_id: type: string + description: ID of the server currently handling this trigger (internal) replication_slot_name: type: string + description: Name of the PostgreSQL logical replication slot to use error: type: string + description: Last error message if the trigger failed last_server_ping: type: string format: date-time + description: Timestamp of last server heartbeat (internal) error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - postgres_resource_path - replication_slot_name @@ -20725,26 +20864,36 @@ components: properties: replication_slot_name: type: string + description: Name of the PostgreSQL logical replication slot to use publication_name: type: string + description: Name of the PostgreSQL publication to subscribe to for change data capture path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when database changes are detected is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" postgres_resource_path: type: string + description: Path to the PostgreSQL resource containing connection configuration publication: $ref: "#/components/schemas/PublicationData" + description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path - script_path @@ -20757,26 +20906,36 @@ components: properties: replication_slot_name: type: string + description: Name of the PostgreSQL logical replication slot to use publication_name: type: string + description: Name of the PostgreSQL publication to subscribe to for change data capture path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when database changes are detected is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script mode: $ref: "#/components/schemas/TriggerMode" postgres_resource_path: type: string + description: Path to the PostgreSQL resource containing connection configuration publication: $ref: "#/components/schemas/PublicationData" + description: Configuration for creating/managing the publication (tables, operations) error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path - script_path @@ -20793,12 +20952,15 @@ components: properties: kafka_resource_path: type: string + description: Path to the Kafka resource containing connection configuration group_id: type: string + description: Kafka consumer group ID for this trigger topics: type: array items: type: string + description: Array of Kafka topic names to subscribe to filters: type: array items: @@ -20812,17 +20974,23 @@ components: - value server_id: type: string + description: ID of the server currently handling this trigger (internal) last_server_ping: type: string format: date-time + description: Timestamp of last server heartbeat (internal) error: type: string + description: Last error message if the trigger failed error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - kafka_resource_path @@ -20835,18 +21003,24 @@ components: properties: path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script kafka_resource_path: type: string + description: Path to the Kafka resource containing connection configuration group_id: type: string + description: Kafka consumer group ID for this trigger topics: type: array items: type: string + description: Array of Kafka topic names to subscribe to filters: type: array items: @@ -20862,10 +21036,13 @@ components: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path @@ -20881,12 +21058,15 @@ components: properties: kafka_resource_path: type: string + description: Path to the Kafka resource containing connection configuration group_id: type: string + description: Kafka consumer group ID for this trigger topics: type: array items: type: string + description: Array of Kafka topic names to subscribe to filters: type: array items: @@ -20900,16 +21080,22 @@ components: - value path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path @@ -20927,29 +21113,40 @@ components: properties: nats_resource_path: type: string + description: Path to the NATS resource containing connection configuration use_jetstream: type: boolean + description: If true, uses NATS JetStream for durable message delivery stream_name: type: string + description: JetStream stream name (required when use_jetstream is true) consumer_name: type: string + description: JetStream consumer name (required when use_jetstream is true) subjects: type: array items: type: string + description: Array of NATS subjects to subscribe to server_id: type: string + description: ID of the server currently handling this trigger (internal) last_server_ping: type: string format: date-time + description: Timestamp of last server heartbeat (internal) error: type: string + description: Last error message if the trigger failed error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - nats_resource_path @@ -20961,30 +21158,41 @@ components: properties: path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script nats_resource_path: type: string + description: Path to the NATS resource containing connection configuration use_jetstream: type: boolean + description: If true, uses NATS JetStream for durable message delivery stream_name: type: string + description: JetStream stream name (required when use_jetstream is true) consumer_name: type: string + description: JetStream consumer name (required when use_jetstream is true) subjects: type: array items: type: string + description: Array of NATS subjects to subscribe to mode: $ref: "#/components/schemas/TriggerMode" error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path @@ -20999,28 +21207,39 @@ components: properties: nats_resource_path: type: string + description: Path to the NATS resource containing connection configuration use_jetstream: type: boolean + description: If true, uses NATS JetStream for durable message delivery stream_name: type: string + description: JetStream stream name (required when use_jetstream is true) consumer_name: type: string + description: JetStream consumer name (required when use_jetstream is true) subjects: type: array items: type: string + description: Array of NATS subjects to subscribe to path: type: string + description: The unique path identifier for this trigger script_path: type: string + description: Path to the script or flow to execute when a message is received is_flow: type: boolean + description: True if script_path points to a flow, false if it points to a script error_handler_path: type: string + description: Path to a script or flow to run when the triggered job fails error_handler_args: $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions required: - path - script_path @@ -22685,10 +22904,8 @@ components: - job AssetUsageAccessType: type: string - enum: - - r - - w - - rw + enum: [r, w, rw] + nullable: true AssetKind: type: string enum: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 1422e9e52d..34e11e692a 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -4,19 +4,21 @@ use crate::db::{ApiAuthed, DB}; #[cfg(feature = "bedrock")] use axum::routing::get; -use axum::{ - body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router, -}; #[cfg(feature = "bedrock")] use axum::Json; +use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use futures::StreamExt; use http::{HeaderMap, Method}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::{json, value::RawValue}; use std::collections::HashMap; +use std::time::Duration; use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::ai_providers::{empty_string_as_none, AIProvider, ProviderConfig, ProviderModel}; +use windmill_common::ai_providers::{ + empty_string_as_none, AIProvider, ProviderConfig, ProviderModel, +}; use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::utils::configure_client; use windmill_common::variables::get_variable_or_self; @@ -27,6 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10; const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90; +const KEEPALIVE_INTERVAL_SECS: u64 = 15; lazy_static::lazy_static! { /// AI request timeout in seconds. @@ -151,9 +154,17 @@ struct AIStandardResource { organization_id: Option, #[serde(default, deserialize_with = "empty_string_as_none")] region: Option, - #[serde(alias = "awsAccessKeyId", default, deserialize_with = "empty_string_as_none")] + #[serde( + alias = "awsAccessKeyId", + default, + deserialize_with = "empty_string_as_none" + )] aws_access_key_id: Option, - #[serde(alias = "awsSecretAccessKey", default, deserialize_with = "empty_string_as_none")] + #[serde( + alias = "awsSecretAccessKey", + default, + deserialize_with = "empty_string_as_none" + )] aws_secret_access_key: Option, /// Platform for Anthropic API (standard or google_vertex_ai) #[serde(default)] @@ -213,9 +224,7 @@ impl AIRequestConfig { let base_url = if matches!(provider, AIProvider::AWSBedrock) { String::new() } else { - provider - .get_base_url(resource.base_url, db) - .await? + provider.get_base_url(resource.base_url, db).await? }; let api_key = if let Some(api_key) = resource.api_key { Some(get_variable_or_self(api_key, db, w_id).await?) @@ -338,7 +347,8 @@ impl AIRequestConfig { let is_azure = provider.is_azure_openai(base_url); let is_anthropic = matches!(provider, AIProvider::Anthropic); - let is_anthropic_vertex = is_anthropic && self.platform == AnthropicPlatform::GoogleVertexAi; + let is_anthropic_vertex = + is_anthropic && self.platform == AnthropicPlatform::GoogleVertexAi; let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let is_google_ai = matches!(provider, AIProvider::GoogleAI); @@ -477,7 +487,9 @@ fn transform_anthropic_for_vertex(body: &Bytes) -> Result<(String, Bytes)> { let model = json_body .remove("model") .and_then(|v| v.as_str().map(|s| s.to_string())) - .ok_or_else(|| Error::BadRequest("Missing 'model' field in Anthropic request".to_string()))?; + .ok_or_else(|| { + Error::BadRequest("Missing 'model' field in Anthropic request".to_string()) + })?; // Add anthropic_version to body (required for Vertex AI) json_body.insert( @@ -561,6 +573,40 @@ async fn check_bedrock_credentials( Ok(Json(response)) } +fn is_sse_response(headers: &HeaderMap) -> bool { + headers + .get(http::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .map(|ct| ct.contains("text/event-stream")) + .unwrap_or(false) +} + +fn inject_keepalives( + upstream: S, + interval: Duration, +) -> impl futures::Stream> +where + S: futures::Stream> + Unpin, +{ + async_stream::stream! { + tokio::pin!(upstream); + loop { + tokio::select! { + biased; + chunk = upstream.next() => { + match chunk { + Some(item) => yield item, + None => break, + } + } + _ = tokio::time::sleep(interval) => { + yield Ok(Bytes::from(": keepalive\n\n")); + } + } + } + } +} + async fn global_proxy( authed: ApiAuthed, Extension(db): Extension, @@ -625,7 +671,15 @@ async fn global_proxy( let status_code = response.status(); let headers = response.headers().clone(); let stream = response.bytes_stream(); - Ok((status_code, headers, axum::body::Body::from_stream(stream))) + let body = if is_sse_response(&headers) { + axum::body::Body::from_stream(inject_keepalives( + stream, + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + axum::body::Body::from_stream(stream) + }; + Ok((status_code, headers, body)) } async fn proxy( @@ -732,20 +786,21 @@ async fn proxy( #[cfg(feature = "bedrock")] { // Extract model and streaming flag for Bedrock transformation (only for POST requests) - let (model, is_streaming) = - if matches!(provider, AIProvider::AWSBedrock) && method == Method::POST { - #[derive(Deserialize, Debug)] - struct BedrockRequest { - model: String, - #[serde(default)] - stream: bool, - } - let parsed: BedrockRequest = serde_json::from_slice(&body) - .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; - (Some(parsed.model), parsed.stream) - } else { - (None, false) - }; + let (model, is_streaming) = if matches!(provider, AIProvider::AWSBedrock) + && method == Method::POST + { + #[derive(Deserialize, Debug)] + struct BedrockRequest { + model: String, + #[serde(default)] + stream: bool, + } + let parsed: BedrockRequest = serde_json::from_slice(&body) + .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; + (Some(parsed.model), parsed.stream) + } else { + (None, false) + }; // For Bedrock requests, use the SDK-based approach if matches!(provider, AIProvider::AWSBedrock) { @@ -850,5 +905,13 @@ async fn proxy( let status_code = response.status(); let headers = response.headers().clone(); let stream = response.bytes_stream(); - Ok((status_code, headers, axum::body::Body::from_stream(stream))) + let body = if is_sse_response(&headers) { + axum::body::Body::from_stream(inject_keepalives( + stream, + Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )) + } else { + axum::body::Body::from_stream(stream) + }; + Ok((status_code, headers, body)) } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 9d2d6d9db3..6e9352e8ee 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1411,7 +1411,7 @@ async fn delete_app( deployed_object, Some(format!("App '{}' deleted", path)), true, - None, + None, ) .await?; diff --git a/backend/windmill-api/src/assets.rs b/backend/windmill-api/src/assets.rs index 6a53c90ebd..8e8afc7036 100644 --- a/backend/windmill-api/src/assets.rs +++ b/backend/windmill-api/src/assets.rs @@ -184,7 +184,13 @@ async fn list_assets( FROM asset INNER JOIN asset_summary ON asset.path = asset_summary.path AND asset.kind = asset_summary.kind LEFT JOIN resource ON asset.kind = 'resource' - AND array_to_string((string_to_array(asset.path, '/'))[1:3], '/') = resource.path + AND ( + -- Extract base path before '?' for ?table= syntax + CASE + WHEN asset.path LIKE '%?%' THEN split_part(asset.path, '?', 1) + ELSE asset.path + END + ) = resource.path AND resource.workspace_id = $1 LEFT JOIN v2_job job ON asset.usage_kind = 'job' AND asset.usage_path = job.id::text diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 0d501cb308..144703ba5c 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,749 +1,4 @@ -#[cfg(feature = "enterprise")] -use crate::ee_oss::ExternalJwks; -use axum::{ - async_trait, - extract::{FromRequestParts, OriginalUri, Query}, - Extension, Json, +pub use windmill_api_auth::auth::{ + invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, AuthCache, + ExpiringAuthCache, OptTokened, Tokened, TruncatedTokenWithEmail, }; -use chrono::TimeZone; -use http::{request::Parts, StatusCode}; -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use tower_cookies::Cookies; -use tracing::Span; - -use crate::db::{ApiAuthed, OptJobAuthed, DB}; -use std::{ - str::FromStr, - sync::{ - atomic::{AtomicI64, AtomicU64, Ordering}, - Arc, - }, -}; -#[cfg(feature = "enterprise")] -use tokio::sync::RwLock; - -use windmill_common::{ - auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, - error::{Error, JsonResult}, - jwt, - users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, -}; - -lazy_static::lazy_static! { - // Global auth cache accessible from main.rs for direct invalidation - pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); - -} -// Global function to invalidate a specific token from cache -pub fn invalidate_token_from_cache(token: &str) { - // Remove all cache entries for this token (across all workspaces) - AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); - tracing::info!( - "Invalidated token from auth cache: {}...", - &token[..token.len().min(8)] - ); -} - -#[derive(Clone)] -pub struct ExpiringAuthCache { - pub authed: ApiAuthed, - pub expiry: chrono::DateTime, - pub job_id: Option, -} - -pub struct AuthCache { - db: DB, - superadmin_secret: Option, - #[cfg(feature = "enterprise")] - ext_jwks: Option>>, -} - -impl AuthCache { - pub fn new( - db: DB, - superadmin_secret: Option, - #[cfg(feature = "enterprise")] ext_jwks: Option>>, - ) -> Self { - AuthCache { - db, - superadmin_secret, - #[cfg(feature = "enterprise")] - ext_jwks, - } - } - - pub async fn invalidate(&self, w_id: &str, token: String) { - AUTH_CACHE.remove(&(w_id.to_string(), token)); - } - - pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { - Some(self.get_opt_job_authed(w_id, token).await?.authed) - } - - pub async fn get_opt_job_authed( - &self, - w_id: Option, - token: &str, - ) -> Option { - let key = ( - w_id.as_ref().unwrap_or(&"".to_string()).to_string(), - token.to_string(), - ); - let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); - match s { - Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => { - Some(OptJobAuthed { authed, job_id }) - } - #[cfg(feature = "enterprise")] - _ if token.starts_with("jwt_ext_") => { - let authed_and_exp = match crate::ee_oss::jwt_ext_auth( - w_id.as_ref(), - token.trim_start_matches("jwt_ext_"), - self.ext_jwks.clone(), - &self.db, - ) - .await - { - Ok(r) => Some(r), - Err(e) => { - tracing::error!("JWT_EXT auth error: {:?}", e); - None - } - }; - - if let Some((authed, exp, job_id)) = authed_and_exp.clone() { - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), - job_id, - }, - ); - - Some(OptJobAuthed { authed, job_id }) - } else { - None - } - } - _ if token.starts_with("jwt_") => { - let jwt_token = token.trim_start_matches("jwt_"); - - let jwt_result = jwt::decode_with_internal_secret::(jwt_token).await; - - match jwt_result { - Ok(claims) => { - if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { - tracing::error!("JWT auth error: workspace_id mismatch"); - return None; - } - let username_override = username_override_from_label(claims.label); - - let authed = crate::db::ApiAuthed { - email: claims.email, - username: claims.username, - is_admin: claims.is_admin, - is_operator: claims.is_operator, - groups: claims.groups, - folders: claims.folders, - scopes: None, - username_override, - token_prefix: claims.audit_span, - }; - let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok()); - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc - .timestamp_nanos(claims.exp as i64 * 1_000_000_000), - job_id, - }, - ); - - Some(OptJobAuthed { authed, job_id }) - } - Err(err) => { - tracing::error!("JWT auth error: {:?}", err); - None - } - } - } - _ => { - let user_o = sqlx::query!( - "UPDATE token SET last_used_at = now() WHERE - token = $1 - AND (expiration > NOW() OR expiration IS NULL) - AND (workspace_id IS NULL OR workspace_id = $2) - RETURNING owner, email, super_admin, scopes, label", - token, - w_id.as_ref(), - ) - .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) - .fetch_optional(&self.db) - .await - .ok() - .flatten(); - - if let Some(user) = user_o { - let authed_o = { - match user { - (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { - let username_override = username_override_from_label(label); - if let Some((prefix, name)) = owner.split_once('/') { - if prefix == "u" { - let (is_admin, is_operator) = if super_admin { - (true, false) - } else { - let r = sqlx::query!( - "SELECT is_admin, operator FROM usr where username = $1 AND \ - workspace_id = $2 AND disabled = false", - name, - &w_id.as_ref().unwrap() - ) - .fetch_one(&self.db) - .await - .ok(); - if let Some(r) = r { - (r.is_admin, r.operator) - } else { - (false, true) - } - }; - - let w_id = &w_id.unwrap(); - let groups = - get_groups_for_user(w_id, &name, &email, &self.db) - .await - .ok() - .unwrap_or_default(); - - let folders = - get_folders_for_user(w_id, &name, &groups, &self.db) - .await - .ok() - .unwrap_or_default(); - - Some(ApiAuthed { - email: email, - username: name.to_string(), - is_admin, - is_operator, - groups, - folders, - scopes: None, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } else { - let groups = vec![name.to_string()]; - let folders = get_folders_for_user( - &w_id.unwrap(), - "", - &groups, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - Some(ApiAuthed { - email: email, - username: format!("group-{name}"), - is_admin: false, - groups, - is_operator: false, - folders, - scopes: None, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } - } else { - let groups = vec![]; - let folders = vec![]; - Some(ApiAuthed { - email: email, - username: owner, - is_admin: super_admin, - is_operator: true, - groups, - folders, - scopes: None, - username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }) - } - } - (_, Some(email), super_admin, scopes, label) => { - let username_override = username_override_from_label(label); - if w_id.is_some() { - let row_o = sqlx::query!( - "SELECT username, is_admin, operator FROM usr WHERE - email = $1 AND workspace_id = $2 AND disabled = false", - &email, - w_id.as_ref().unwrap() - ) - .map(|x| (x.username, x.is_admin, x.operator)) - .fetch_optional(&self.db) - .await - .unwrap_or(Some(("error".to_string(), false, false))); - - match row_o { - Some((username, is_admin, is_operator)) => { - let groups = get_groups_for_user( - &w_id.as_ref().unwrap(), - &username, - &email, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - - let folders = get_folders_for_user( - &w_id.unwrap(), - &username, - &groups, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - Some(ApiAuthed { - email, - username, - is_admin: is_admin || super_admin, - is_operator, - groups, - folders, - scopes, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } - None if super_admin => Some(ApiAuthed { - email: email.clone(), - username: email, - is_admin: super_admin, - is_operator: false, - groups: vec![], - folders: vec![], - scopes, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }), - None => None, - } - } else { - Some(ApiAuthed { - email: email.to_string(), - username: email, - is_admin: super_admin, - is_operator: true, - groups: Vec::new(), - folders: Vec::new(), - scopes, - username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }) - } - } - _ => None, - } - }; - if let Some(authed) = authed_o.as_ref() { - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc::now() - + chrono::Duration::try_seconds(120).unwrap(), - job_id: None, - }, - ); - } - authed_o.map(|authed| OptJobAuthed { authed, job_id: None }) - } else if self - .superadmin_secret - .as_ref() - .map(|x| x == token) - .unwrap_or(false) - { - let authed = ApiAuthed { - email: SUPERADMIN_SECRET_EMAIL.to_string(), - username: "superadmin_secret".to_string(), - is_admin: true, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }; - Some(OptJobAuthed { authed, job_id: None }) - } else { - None - } - } - } - } -} - -async fn extract_token(parts: &mut Parts, state: &S) -> Option { - let auth_header = parts - .headers - .get(http::header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|s| s.strip_prefix("Bearer ")); - - let from_cookie = match auth_header { - Some(x) => Some(x.to_owned()), - None => Extension::::from_request_parts(parts, state) - .await - .ok() - .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), - }; - - #[derive(Deserialize)] - struct Token { - token: Option, - } - match from_cookie { - Some(token) => Some(token), - None => Query::::from_request_parts(parts, state) - .await - .ok() - .and_then(|token| token.token.clone()), - } -} - -#[derive(Clone, Debug)] -pub struct Tokened { - pub token: String, -} - -#[derive(Clone, Debug)] -pub struct OptTokened { - #[allow(dead_code)] - pub token: Option, -} - -struct BruteForceCounter { - counter: AtomicU64, - last_reset: AtomicI64, -} - -lazy_static::lazy_static! { - static ref BRUTE_FORCE_COUNTER: BruteForceCounter = - BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; -} - -impl BruteForceCounter { - async fn increment(&self) { - let now = time::OffsetDateTime::now_utc().unix_timestamp(); - if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { - tracing::error!( - "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" - ); - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - if now - self.last_reset.load(Ordering::Relaxed) > 60 { - self.counter.store(0, Ordering::Relaxed); - self.last_reset.store(now, Ordering::Relaxed); - } - } -} - -#[async_trait] -impl FromRequestParts for Tokened -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(Tokened { token: "".to_string() }); - }; - let already_tokened = parts.extensions.get::(); - if let Some(tokened) = already_tokened { - Ok(tokened.clone()) - } else { - let token_o = extract_token(parts, state).await; - if let Some(token) = token_o { - let tokened = Self { token }; - parts.extensions.insert(tokened.clone()); - Ok(tokened) - } else { - BRUTE_FORCE_COUNTER.increment().await; - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) - } - } - } -} - -#[async_trait] -impl FromRequestParts for OptTokened -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(OptTokened { token: None }); - }; - let already_tokened = parts.extensions.get::(); - if let Some(tokened) = already_tokened { - Ok(OptTokened { token: Some(tokened.token.clone()) }) - } else { - let token_o = extract_token(parts, state).await; - Ok(OptTokened { token: token_o }) - } - } -} - -pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { - if let Some(scopes) = scopes { - for scope in scopes.iter_mut() { - if scope.starts_with("run:") { - let (_, part_scope) = scope.split_once(":").unwrap(); - - if let Some((kind, path)) = part_scope.split_once("/") { - //appending a 's' as runnable kind is singular while new scope format expect it to be plural - *scope = format!("jobs:run:{}s:{}", kind, path); - } - } else if scope.starts_with("jobs:") { - // Map old jobs scopes to new format - let new_scope = match scope.as_str() { - "jobs:listjobs" => "jobs:read", - "jobs:runscript" => "jobs:run:scripts", - "jobs:runflow" => "jobs:run:flows", - "jobs:resumeflow" => "jobs:run:flows", - "jobs:deletejob" => "jobs:write", - _ => continue, - }; - - *scope = new_scope.to_string(); - } - } - } -} - -fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { - let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { - Some(path_vec[3].to_owned()) - } else if path_vec.len() >= 5 - && path_vec[0] == "" - && path_vec[1] == "api" - && path_vec[2] == "mcp" - && path_vec[3] == "w" - { - Some(path_vec[4].to_owned()) - } else { - if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" { - Some(path_vec[4].to_owned()) - } else { - None - } - }; - - workspace_id -} - -#[async_trait] -impl FromRequestParts for ApiAuthed -where - S: Send + Sync, -{ - type Rejection = Error; - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - let opt_job_authed = OptJobAuthed::from_request_parts(parts, state).await?; - Ok(opt_job_authed.authed) - } -} - -#[async_trait] -impl FromRequestParts for OptJobAuthed -where - S: Send + Sync, -{ - type Rejection = Error; - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(OptJobAuthed::default()); - }; - let already_authed = parts.extensions.get::(); - - if let Some(authed) = already_authed { - return Ok(authed.clone()); - } - - let already_tokened = parts.extensions.get::(); - let token_o = if let Some(token) = already_tokened { - Some(token.token.clone()) - } else { - extract_token(parts, state).await - }; - if let Some(token) = token_o { - if let Ok(Extension(cache)) = - Extension::>::from_request_parts(parts, state).await - { - let original_uri = OriginalUri::from_request_parts(parts, state) - .await - .ok() - .map(|x| x.0) - .unwrap_or_default(); - let path_vec: Vec<&str> = original_uri.path().split("/").collect(); - let workspace_id = maybe_get_workspace_id_from_path(&path_vec); - - if let Some(mut opt_job_authed) = - cache.get_opt_job_authed(workspace_id.clone(), &token).await - { - let authed = &mut opt_job_authed.authed; - if authed.scopes.is_some() { - transform_old_scope_to_new_scope(authed.scopes.as_mut()); - - let path = original_uri.path(); - let method = parts.method.as_str(); - - if let Err(err) = crate::scopes::check_scopes_for_route( - authed.scopes.as_deref(), - path, - method, - ) { - BRUTE_FORCE_COUNTER.increment().await; - return Err(err); - } - } - parts.extensions.insert(authed.clone()); - - Span::current().record("username", &authed.username.as_str()); - Span::current().record("email", &authed.email); - - if let Some(workspace_id) = workspace_id { - Span::current().record("workspace_id", &workspace_id); - } - return Ok(opt_job_authed); - } - } - } - BRUTE_FORCE_COUNTER.increment().await; - Err(Error::NotAuthorized("Unauthorized".to_string())) - } -} - -fn username_override_from_label(label: Option) -> Option { - match label { - Some(label) - if label.starts_with("webhook-") - || label.starts_with("http-") - || label.starts_with("email-") - || label.starts_with("ws-") => - { - Some(label) - } - Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( - label - .trim_start_matches("ephemeral-script-end-user-") - .to_string(), - ), - Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), - Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { - Some(format!("label-{label}")) - } - _ => None, - } -} - -#[derive(FromRow, Serialize)] -pub struct TruncatedTokenWithEmail { - pub label: Option, - pub token_prefix: Option, - pub expiration: Option>, - pub created_at: chrono::DateTime, - pub last_used_at: chrono::DateTime, - pub scopes: Option>, - pub email: Option, -} - -pub async fn list_tokens_internal( - db: &DB, - w_id: &str, - path: &str, - is_flow: bool, -) -> JsonResult> { - let tokens = if is_flow { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:flows:' || $2]::text[] - OR scopes @> ARRAY['run:flow/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - } else { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] - OR scopes @> ARRAY['run:script/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - }; - - Ok(Json(tokens)) -} diff --git a/backend/windmill-api/src/bedrock.rs b/backend/windmill-api/src/bedrock.rs index 4ed1f2cdfa..de0e9059d7 100644 --- a/backend/windmill-api/src/bedrock.rs +++ b/backend/windmill-api/src/bedrock.rs @@ -18,14 +18,16 @@ use axum::body::Bytes; use serde::Deserialize; +use windmill_common::ai_bedrock::build_tool_config; use windmill_common::ai_bedrock::{ bedrock_stream_event_is_block_stop, bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_start, format_bedrock_error, - BedrockClient + BedrockClient, +}; +use windmill_common::ai_types::{ + OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, ToolDefFunction, }; use windmill_common::error::{Error, Result}; -use windmill_common::ai_types::{OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, ToolDefFunction}; -use windmill_common::ai_bedrock::build_tool_config; // ============================================================================ // Shared Request Types for SDK-Based Handlers @@ -69,10 +71,7 @@ struct OpenAIToolFunction { /// Authentication configuration for Bedrock clients enum BedrockAuthConfig { BearerToken(String), - IamCredentials { - access_key_id: String, - secret_access_key: String, - }, + IamCredentials { access_key_id: String, secret_access_key: String }, Environment, } @@ -105,13 +104,8 @@ async fn create_bedrock_client( region: &str, ) -> Result { match determine_auth_config(api_key, aws_access_key_id, aws_secret_access_key) { - BedrockAuthConfig::BearerToken(key) => { - BedrockClient::from_bearer_token(key, region).await - } - BedrockAuthConfig::IamCredentials { - access_key_id, - secret_access_key, - } => { + BedrockAuthConfig::BearerToken(key) => BedrockClient::from_bearer_token(key, region).await, + BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key } => { BedrockClient::from_credentials(access_key_id, secret_access_key, None, region).await } BedrockAuthConfig::Environment => BedrockClient::from_env(region).await, @@ -185,10 +179,7 @@ async fn create_bedrock_control_client( .build(); Ok(aws_sdk_bedrock::Client::from_conf(config)) } - BedrockAuthConfig::IamCredentials { - access_key_id, - secret_access_key, - } => { + BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key } => { let credentials = aws_credential_types::Credentials::new( access_key_id, secret_access_key, diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index 25768c3c10..e00b4324a8 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -1,364 +1 @@ -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ - filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery, - ListQueueQuery, UnifiedJob, - }, - utils::check_scopes, -}; -use axum::extract::Path; -use axum::routing::{delete, get}; -use axum::{extract::Query, Extension, Json}; -use serde::Deserialize; - -use axum::Router; - -use serde::Serialize; -use sql_builder::bind::Bind; -use sql_builder::SqlBuilder; -use uuid::Uuid; -use windmill_common::db::UserDB; -use windmill_common::error::Error::PermissionDenied; -use windmill_common::error::{self, JsonResult}; -use windmill_common::utils::require_admin; - -pub fn global_service() -> Router { - Router::new() - .route("/list", get(list_concurrency_groups)) - .route("/prune/*concurrency_key", delete(prune_concurrency_group)) - .route("/:job_id/key", get(get_concurrency_key)) -} - -pub fn workspaced_service() -> Router { - Router::new().route("/list_jobs", get(get_concurrent_intervals)) -} - -#[derive(Serialize)] -pub struct ConcurrencyGroups { - concurrency_key: String, - total_running: i64, -} - -async fn list_concurrency_groups( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let concurrency_counts = sqlx::query_as::<_, (String, i64)>( - "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter", - ).fetch_all(&db) - .await?; - - let mut concurrency_groups: Vec = vec![]; - for (concurrency_key, count) in concurrency_counts { - concurrency_groups.push(ConcurrencyGroups { - concurrency_key: concurrency_key.clone(), - total_running: count, - }) - } - - return Ok(Json(concurrency_groups)); -} - -async fn prune_concurrency_group( - authed: ApiAuthed, - Extension(db): Extension, - Path(concurrency_key): Path, -) -> JsonResult<()> { - if !authed.is_admin { - return Err(PermissionDenied( - "Only administrators can delete concurrency groups".to_string(), - )); - } - let mut tx = db.begin().await?; - - let concurrency_group = sqlx::query_as::<_, (String, i64)>( - "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter WHERE concurrency_id = $1 FOR UPDATE", - ) - .bind(concurrency_key.clone()) - .fetch_optional(&mut *tx) - .await?; - - let n_job_uuids = concurrency_group.map(|cg| cg.1).unwrap_or_default(); - - if n_job_uuids > 0 { - tx.commit().await?; - return Err(error::Error::internal_err( - "Concurrency group is currently in use, unable to remove it. Retry later.".to_string(), - )); - } - - sqlx::query!( - "DELETE FROM concurrency_counter WHERE concurrency_id = $1", - concurrency_key.clone(), - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM concurrency_key WHERE key = $1", - concurrency_key.clone(), - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok(Json(())) -} - -#[derive(Serialize)] -struct ExtendedJobs { - jobs: Vec, - obscured_jobs: Vec, - omitted_obscured_jobs: bool, -} - -#[derive(Serialize)] -struct ObscuredJob { - typ: String, - started_at: Option>, - duration_ms: Option, -} -#[derive(Deserialize)] -struct ExtendedJobsParams { - row_limit: Option, -} - -pub fn join_concurrency_key<'c>( - concurrency_key: Option<&String>, - mut sqlb: SqlBuilder, -) -> SqlBuilder { - if let Some(key) = concurrency_key { - sqlb.join("concurrency_key") - .on_eq("id", "concurrency_key.job_id") - .and_where_eq("key", "?".bind(key)); - } - - sqlb -} - -async fn get_concurrent_intervals( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Query(iq): Query, - Query(lq): Query, -) -> JsonResult { - check_scopes(&authed, || format!("jobs:read"))?; - - if lq.success.is_some() && lq.running.is_some_and(|x| x) { - return Err(error::Error::BadRequest( - "cannot specify both success and running".to_string(), - )); - } - - let row_limit = iq.row_limit.unwrap_or(1000); - - let lq = ListCompletedQuery { order_desc: Some(true), ..lq }; - let lqc = lq.clone(); - let lqq: ListQueueQuery = lqc.into(); - let mut sqlb_q = SqlBuilder::select_from("v2_job_queue") - .fields(UnifiedJob::queued_job_fields()) - .order_by("created_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - let mut sqlb_c = SqlBuilder::select_from("v2_job_completed") - .fields(UnifiedJob::completed_job_fields()) - .order_by("completed_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue") - .fields(&["id"]) - .order_by("created_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed") - .fields(&["id"]) - .order_by("completed_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - - sqlb_q = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q); - sqlb_c = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c); - sqlb_q_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q_user); - sqlb_c_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c_user); - - let should_fetch_obscured_jobs = match lq { - ListCompletedQuery { - script_path_start: None, - script_path_exact: None, - script_hash: None, - created_by: None, - success: None, - running: None, - parent_job: None, - is_skipped: None | Some(false), - suspended: None, - schedule_path: None, - args: None, - result: None, - tag: None, - has_null_parent: None, - worker: None, - label: None, - trigger_path: None, - scheduled_for_before_now: _, - is_not_schedule: _, - started_before: _, - started_after: _, - created_before: _, - created_after: _, - created_before_queue: _, - created_after_queue: _, - completed_after: _, - completed_before: _, - created_or_started_before: _, - created_or_started_after: _, - created_or_started_after_completed_jobs: _, - order_desc: _, - job_kinds: _, - is_flow_step: _, - all_workspaces: _, - concurrency_key: Some(_), - allow_wildcards: None, - trigger_kind: _, - include_args: _, - } => true, - _ => false, - }; - - // When we have a concurrency key defined, fetch jobs from other workspaces - // as obscured unless we're in the admins workspace. This is to show the - // potential concurrency races without showing jobs that don't belong to - // the workspace. - // To avoid infering information through filtering, don't return obscured - // jobs if the filters are too specific - if should_fetch_obscured_jobs && w_id != "admins" { - // Get the obscured jobs from all workspaces (concurrency key could be global) - let (sqlb_q, sqlb_c) = ( - filter_list_queue_query( - sqlb_q, - &ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() }, - "admins", - true, - ), - filter_list_completed_query( - sqlb_c, - &ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() }, - "admins", - true, - ), - ); - - sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str(), true); - sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str(), true); - - let sql_q_user = sqlb_q_user.query()?; - let sql_c_user = sqlb_c_user.query()?; - let sql_q = sqlb_q.query()?; - let sql_c = sqlb_c.query()?; - - // This first transaction uses the user_db to know which uuids are - // accessible to the user. - let mut tx = user_db.begin(&authed).await?; - let running_jobs_user: Vec = if lq.success.is_none() { - sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await? - } else { - vec![] - }; - let completed_jobs_user: Vec = if lq.running.is_none() { - sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await? - } else { - vec![] - }; - tx.commit().await?; - - // This second transaction uses the db, so it will fetch information - // potentially forbidden to the user. It must be obscured before - // returning it - let running_jobs_db: Vec = if lq.success.is_none() { - sqlx::query_as(&sql_q).fetch_all(&db).await? - } else { - vec![] - }; - let completed_jobs_db: Vec = if lq.running.is_none() { - sqlx::query_as(&sql_c).fetch_all(&db).await? - } else { - vec![] - }; - - let obscured_jobs = running_jobs_db - .iter() - .filter(|j| !running_jobs_user.iter().any(|id| j.id == *id)) - .chain( - completed_jobs_db - .iter() - .filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)), - ) - .map(|j| ObscuredJob { - typ: j.typ.clone(), - started_at: j.started_at, - duration_ms: j.duration_ms, - }) - .collect(); - - let jobs = running_jobs_db - .into_iter() - .filter(|j| running_jobs_user.iter().any(|id| j.id == *id)) - .chain( - completed_jobs_db - .into_iter() - .filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)), - ) - .map(From::from) - .collect(); - - Ok(Json(ExtendedJobs { - jobs, - obscured_jobs, - omitted_obscured_jobs: !should_fetch_obscured_jobs, - })) - } else { - sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str(), true); - sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str(), true); - let sql_q = sqlb_q.query()?; - let sql_c = sqlb_c.query()?; - - let mut tx = user_db.begin(&authed).await?; - let running_jobs: Vec = if lq.success.is_none() { - sqlx::query_as(&sql_q).fetch_all(&mut *tx).await? - } else { - vec![] - }; - let completed_jobs: Vec = if lq.running.is_none() { - sqlx::query_as(&sql_c).fetch_all(&mut *tx).await? - } else { - vec![] - }; - tx.commit().await?; - - let jobs = running_jobs - .into_iter() - .chain(completed_jobs.into_iter()) - .map(From::from) - .collect(); - - Ok(Json(ExtendedJobs { - jobs, - obscured_jobs: vec![], - omitted_obscured_jobs: !should_fetch_obscured_jobs, - })) - } -} - -async fn get_concurrency_key( - Extension(db): Extension, - Path(job_id): Path, -) -> JsonResult> { - let key = sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id) - .fetch_optional(&db) - .await?; - Ok(Json(key)) -} +pub use windmill_api_jobs::concurrency_groups::*; diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index 719437b29c..1d149a08b5 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -279,14 +279,14 @@ async fn native_kubernetes_autoscaling_healthcheck() -> Result<(), error::Error> } async fn list_available_python_versions() -> error::JsonResult> { - #[cfg(not(feature = "python"))] + #[cfg(not(all(feature = "python", feature = "inline_preview")))] return Err(error::Error::BadRequest( "Python listing available only with 'python' feature enabled".to_string(), )); - #[cfg(feature = "python")] + #[cfg(all(feature = "python", feature = "inline_preview"))] use itertools::Itertools; - #[cfg(feature = "python")] + #[cfg(all(feature = "python", feature = "inline_preview"))] return Ok(Json( windmill_worker::PyV::list_available_python_versions() .await diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index e03c02381f..696d752bab 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -14,13 +14,11 @@ use sqlx::{ }; use tokio::task::JoinHandle; -use windmill_audit::audit_oss::AuditAuthorable; pub use windmill_common::db::DB; -use windmill_common::{ - db::{Authable, Authed, AuthedRef}, - error::Error, - utils::generate_lock_id, -}; +use windmill_common::{error::Error, utils::generate_lock_id}; + +#[allow(unused_imports)] +pub use windmill_api_auth::{ApiAuthed, OptJobAuthed}; async fn current_database(conn: &mut PgConnection) -> Result { // language=SQL @@ -64,6 +62,19 @@ lazy_static::lazy_static! { (20260126235947, include_str!( "../../custom_migrations/lowercase_emails_safe.sql" ).to_string()), + (20260206000000, "".to_string()), + (20260207000001, include_str!( + "../../migrations/20260207000001_concurrent_indexes_v2_job.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260207000002, include_str!( + "../../migrations/20260207000002_concurrent_indexes_v2_job_completed.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY").replace("DROP INDEX CONCURRENTLY IF EXISTS labeled_jobs_on_jobs;", "")), + (20260207000003, include_str!( + "../../migrations/20260207000003_concurrent_indexes_v2_job_queue.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), + (20260207000004, include_str!( + "../../migrations/20260207000004_concurrent_indexes_other.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")), ].into_iter().collect(); } @@ -177,11 +188,31 @@ impl Migrate for CustomMigrator { if let Some(migration_sql) = OVERRIDDEN_MIGRATIONS.get(&migration.version) { tracing::info!("Using custom migration for version {}", migration.version); - // tracing::info!("Migration SQL: {}", migration_sql); - self.inner - .execute(&**migration_sql) - .await?; + if migration_sql.contains("CONCURRENTLY") { + // CONCURRENTLY operations cannot run inside a transaction block + // or a multi-statement query (PostgreSQL requires top-level execution). + // Split into individual statements and execute each separately. + for stmt in migration_sql.split(';') { + let stmt = stmt.trim(); + if !stmt.is_empty() + && stmt.lines().any(|l| { + let t = l.trim(); + !t.is_empty() && !t.starts_with("--") + }) + { + let summary: String = stmt.lines() + .filter(|l| !l.trim().is_empty() && !l.trim().starts_with("--")) + .collect::>() + .join(" "); + tracing::info!("Executing: {summary}"); + self.inner.execute(stmt).await?; + tracing::info!("Done: {summary}"); + } + } + } else if !migration_sql.is_empty() { + self.inner.execute(&**migration_sql).await?; + } let _ = sqlx::query( r#" INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time ) @@ -224,7 +255,8 @@ pub async fn migrate( if let Err(err) = sqlx::query!( "DELETE FROM _sqlx_migrations WHERE version=20250131115248 OR version=20250902085503 OR version=20250201145630 OR - version=20250201145631 OR version=20250201145632 OR version=20251006143821" + version=20250201145631 OR version=20250201145632 OR version=20251006143821 OR + version=20260207000001 OR version=20260207000002 OR version=20260207000003 OR version=20260207000004" ) .execute(db) .await @@ -257,120 +289,3 @@ pub async fn migrate( crate::live_migrations::custom_migrations(&mut custom_migrator, db).await?; Ok(None) } - -#[derive(Default, Clone, Debug)] -pub struct OptJobAuthed { - pub job_id: Option, - pub authed: ApiAuthed, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] -pub struct ApiAuthed { - pub email: String, - pub username: String, - pub is_admin: bool, - pub is_operator: bool, - pub groups: Vec, - // (folder name, can write, is owner) - pub folders: Vec<(String, bool, bool)>, - pub scopes: Option>, - pub username_override: Option, - pub token_prefix: Option, -} - -impl ApiAuthed { - pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> { - AuthedRef { - email: &self.email, - username: &self.username, - is_admin: &self.is_admin, - is_operator: &self.is_operator, - groups: &self.groups, - folders: &self.folders, - scopes: &self.scopes, - token_prefix: &self.token_prefix, - } - } -} - -impl From for Authed { - fn from(value: ApiAuthed) -> Self { - Self { - email: value.email, - username: value.username, - is_admin: value.is_admin, - is_operator: value.is_operator, - groups: value.groups, - folders: value.folders, - scopes: value.scopes, - token_prefix: value.token_prefix, - } - } -} - -impl From for ApiAuthed { - fn from(value: Authed) -> Self { - Self { - email: value.email, - username: value.username, - is_admin: value.is_admin, - is_operator: value.is_operator, - groups: value.groups, - folders: value.folders, - scopes: value.scopes, - username_override: None, // Authed doesn't have this field, so default to None - token_prefix: value.token_prefix, - } - } -} - -impl ApiAuthed { - pub fn display_username(&self) -> &str { - self.username_override.as_ref().unwrap_or(&self.username) - } -} - -impl AuditAuthorable for ApiAuthed { - fn username(&self) -> &str { - self.username.as_str() - } - fn email(&self) -> &str { - self.email.as_str() - } - fn username_override(&self) -> Option<&str> { - self.username_override.as_deref() - } - fn token_prefix(&self) -> Option<&str> { - self.token_prefix.as_deref() - } -} - -impl Authable for ApiAuthed { - fn is_admin(&self) -> bool { - self.is_admin - } - - fn is_operator(&self) -> bool { - self.is_operator - } - - fn groups(&self) -> &[String] { - &self.groups - } - - fn folders(&self) -> &[(String, bool, bool)] { - &self.folders - } - - fn scopes(&self) -> Option<&[std::string::String]> { - self.scopes.as_ref().map(|x| x.as_slice()) - } - - fn email(&self) -> &str { - &self.email - } - - fn username(&self) -> &str { - &self.username - } -} diff --git a/backend/windmill-api/src/debug.rs b/backend/windmill-api/src/debug.rs index fc5e8084c5..bdf769833d 100644 --- a/backend/windmill-api/src/debug.rs +++ b/backend/windmill-api/src/debug.rs @@ -21,22 +21,23 @@ //! - A completed job entry in v2_job_completed //! - An audit log entry identical to script preview runs -use axum::{extract::Path, routing::{get, post}, Extension, Json, Router}; +use axum::{ + extract::Path, + routing::{get, post}, + Extension, Json, Router, +}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use chrono::Utc; -use ed25519_dalek::{SigningKey, Signer}; +use ed25519_dalek::{Signer, SigningKey}; use serde::{Deserialize, Serialize}; -use sha2::{Sha256, Digest}; +use sha2::{Digest, Sha256}; use sqlx::types::Json as SqlxJson; use std::sync::Arc; use tokio::sync::RwLock; use uuid::Uuid; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ - db::UserDB, - error::JsonResult, - jobs::JobKind, - scripts::ScriptLang, + db::UserDB, error::JsonResult, jobs::JobKind, scripts::ScriptLang, users::username_to_permissioned_as, }; @@ -279,7 +280,8 @@ async fn sign_debug_request( job_id, w_id, now, - SqlxJson(serde_json::json!({"debug_session": true, "language": request.language})) as SqlxJson, + SqlxJson(serde_json::json!({"debug_session": true, "language": request.language})) + as SqlxJson, ) .execute(&mut *tx) .await?; @@ -400,10 +402,13 @@ async fn sign_expression( ActionKind::Execute, &w_id, Some(&resource), - Some([ - ("job_id", request.job_id.as_str()), - ("expression", request.expression.as_str()), - ].into()), + Some( + [ + ("job_id", request.job_id.as_str()), + ("expression", request.expression.as_str()), + ] + .into(), + ), ) .await?; diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index 6e54b45dc5..a1f7a54d20 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -1,17 +1,11 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::ee::*; -#[cfg(all( - feature = "enterprise", - any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), - not(feature = "private") -))] -use {crate::db::ApiAuthed, windmill_common::DB}; +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub use windmill_api_auth::ee_oss::ExternalJwks; #[cfg(not(feature = "private"))] use anyhow::anyhow; -#[cfg(all(feature = "enterprise", not(feature = "private")))] -use {std::sync::Arc, tokio::sync::RwLock}; #[cfg(not(feature = "private"))] pub async fn validate_license_key( _license_key: String, @@ -21,42 +15,9 @@ pub async fn validate_license_key( Err(anyhow!("License can't be validated in Windmill CE")) } -#[cfg(all(feature = "enterprise", not(feature = "private")))] -pub async fn jwt_ext_auth( - _w_id: Option<&String>, - _token: &str, - _external_jwks: Option>>, - _db: &crate::db::DB, -) -> anyhow::Result<(crate::db::ApiAuthed, usize, Option)> { - // Implementation is not open source - - Err(anyhow!("External JWT auth is not open source")) -} - -#[cfg(all(feature = "enterprise", not(feature = "private")))] -pub struct ExternalJwks; - -#[cfg(all(feature = "enterprise", not(feature = "private")))] -impl ExternalJwks { - pub async fn load() -> Option>> { - // Implementation is not open source - None - } -} - +// interpolate moved to windmill-store/src/resources.rs #[cfg(all( feature = "enterprise", - any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), - not(feature = "private") + any(feature = "nats", feature = "kafka", feature = "sqs_trigger") ))] -pub async fn interpolate( - _authed: &ApiAuthed, - _db: &DB, - _w_id: &str, - _s: String, -) -> Result { - // Implementation is not open source - Err(anyhow!( - "Interpolation is not available in open source version" - )) -} +pub use windmill_store::resources::interpolate; diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 875bcf683a..6212d38e31 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -1,630 +1 @@ -#[cfg(feature = "embedding")] -use anyhow::{anyhow, Error, Result}; -#[cfg(feature = "embedding")] -use std::{collections::HashMap, path::PathBuf, sync::Arc}; -#[cfg(feature = "embedding")] -use windmill_common::DEFAULT_HUB_BASE_URL; -#[cfg(feature = "embedding")] -use windmill_common::HUB_BASE_URL; - -use axum::Router; - -#[cfg(feature = "embedding")] -use axum::{ - extract::{Path, Query}, - Json, -}; - -#[cfg(feature = "embedding")] -use axum::routing::get; -#[cfg(feature = "embedding")] -use candle_core::{Device, Tensor}; -#[cfg(feature = "embedding")] -use candle_nn::VarBuilder; -#[cfg(feature = "embedding")] -use candle_transformers::models::bert::{BertModel, Config, DTYPE}; -#[cfg(feature = "embedding")] -use hf_hub::api::tokio::Api; -#[cfg(feature = "embedding")] -use serde::Deserialize; -#[cfg(feature = "embedding")] -use serde::Serialize; -#[cfg(feature = "embedding")] -use sqlx::{Pool, Postgres}; -#[cfg(feature = "embedding")] -use tinyvector::{ - db::{Db, Embedding}, - similarity::Distance, -}; -#[cfg(feature = "embedding")] -use tokenizers::Tokenizer; -#[cfg(feature = "embedding")] -use tokio::sync::RwLock; -#[cfg(feature = "embedding")] -use windmill_common::utils::http_get_from_hub; - -#[cfg(feature = "embedding")] -use windmill_common::error::JsonResult; - -#[cfg(feature = "embedding")] -use crate::{resources::ResourceType, HTTP_CLIENT}; - -#[cfg(feature = "embedding")] -lazy_static::lazy_static! { - pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); - pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); - pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize)] -struct HubScriptsQuery { - text: String, - limit: Option, - kind: Option, - app: Option, -} - -#[cfg(feature = "embedding")] -#[derive(Serialize)] -pub struct HubScriptResult { - ask_id: i64, - id: i64, - version_id: i64, - summary: String, - app: String, - kind: String, - score: f32, -} - -#[cfg(feature = "embedding")] -async fn query_hub_scripts( - Query(query): Query, -) -> JsonResult> { - let embeddings_db = EMBEDDINGS_DB.read().await; - - if let Some(embeddings_db) = embeddings_db.as_ref() { - let results = embeddings_db - .query_hub_scripts(&query.text, query.limit, query.kind, query.app) - .await?; - - Ok(Json(results)) - } else { - Err(windmill_common::error::Error::internal_err( - "Embeddings db not initialized".to_string(), - )) - } -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize)] -struct ResourceTypesQuery { - text: String, - limit: Option, -} - -#[cfg(feature = "embedding")] -#[derive(Serialize)] -pub struct ResourceTypeResult { - name: String, - score: f32, - schema: Option, -} -#[cfg(feature = "embedding")] -async fn query_resource_types( - Query(query): Query, - Path(w_id): Path, -) -> JsonResult> { - let embeddings_db = EMBEDDINGS_DB.read().await; - - if let Some(embeddings_db) = embeddings_db.as_ref() { - let results = embeddings_db - .query_resource_types(w_id, &query.text, query.limit) - .await?; - - Ok(Json(results)) - } else { - Err(windmill_common::error::Error::internal_err( - "Embeddings db not initialized".to_string(), - )) - } -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize, Debug, Clone)] -struct HubScript { - ask_id: i64, - id: i64, - version_id: i64, - summary: String, - app: String, - kind: String, - embedding: Vec, -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize, Debug)] -struct HubResourceType { - name: String, - embedding: Vec, -} - -#[cfg(feature = "embedding")] -pub struct ModelInstance { - model: BertModel, - tokenizer: Tokenizer, -} - -#[cfg(feature = "embedding")] -impl ModelInstance { - pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> { - let api = Api::new()?; - let repo_api = api.model("thenlper/gte-small".to_string()); - - let (config_filename, tokenizer_filename, weights_filename) = - ( - repo_api - .get("config.json") - .await - .map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?, - repo_api.get("tokenizer.json").await.map_err(|e| { - anyhow!("Failed to get tokenizer.json from hugging face: {}", e) - })?, - repo_api.get("model.safetensors").await.map_err(|e| { - anyhow!("Failed to get model.safetensors from hugging face: {}", e) - })?, - ); - - Ok((config_filename, tokenizer_filename, weights_filename)) - } - - pub async fn new() -> Result { - tracing::info!("Loading embedding model..."); - let device = Device::Cpu; - let (config_filename, tokenizer_filename, weights_filename) = - Self::load_model_files().await?; - let config = std::fs::read_to_string(config_filename)?; - let config: Config = serde_json::from_str(&config)?; - let tokenizer = Tokenizer::from( - Tokenizer::from_file(tokenizer_filename) - .map_err(Error::msg)? - .with_padding(None) - .to_owned(), - ); - - let vb = - unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? }; - let model = BertModel::load(vb, &config)?; - tracing::info!("Loaded embedding model"); - Ok(Self { model, tokenizer }) - } - - pub async fn create_embedding(self: Arc, sentence: &str) -> Result> { - let sentence = sentence.to_owned(); - tokio::task::spawn_blocking(move || { - let tokens = self - .tokenizer - .encode(sentence, true) - .map_err(Error::msg)? - .get_ids() - .to_vec(); - - let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?; - let token_type_ids = token_ids.zeros_like()?; - - let embedding = self.model.forward(&token_ids, &token_type_ids, None)?; - let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?; - let embedding = normalize_l2(&embedding)?; - - let embedding = embedding.get(0)?.to_vec1()?; - - Ok(embedding) - }) - .await? - } -} - -#[cfg(feature = "embedding")] -pub struct EmbeddingsDb { - db: Db, - model_instance: Arc, -} - -#[cfg(feature = "embedding")] -impl EmbeddingsDb { - pub async fn new(pg_db: &Pool, model_instance: Arc) -> Result { - let db = Db::new(); - - let mut embeddings_db = Self { db, model_instance: model_instance.clone() }; - - embeddings_db.fill_db(pg_db).await?; - - Ok(embeddings_db) - } - - async fn fill_db(&mut self, pg_db: &Pool) -> Result<()> { - if self.db.get_collection("scripts").is_some() { - self.db.delete_collection("scripts")?; - } - - self.db - .create_collection("scripts".to_string(), 384, Distance::Cosine)?; - - if self.db.get_collection("resource_types").is_some() { - self.db.delete_collection("resource_types")?; - } - - self.db - .create_collection("resource_types".to_string(), 384, Distance::Cosine)?; - - let hub_base_url = HUB_BASE_URL.read().await.clone(); - - let response = match hub_base_url.as_str() { - DEFAULT_HUB_BASE_URL => { - let response = HTTP_CLIENT - .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json") - .send() - .await; - - if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { - tracing::warn!("Failed to get scripts embeddings from bucket, trying hub..."); - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/scripts/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } else { - response.unwrap() - } - } - _ => { - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/scripts/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } - }; - - if response.error_for_status_ref().is_err() { - return Err(anyhow!( - "Failed to get scripts embeddings from hub with error code: {}", - response.status() - )); - } - - let hub_scripts = response.json::>().await?; - - for script in &hub_scripts { - let mut hm = HashMap::new(); - hm.insert("ask_id".to_string(), script.ask_id.clone().to_string()); - hm.insert("summary".to_string(), script.summary.clone()); - hm.insert("app".to_string(), script.app.clone()); - hm.insert("kind".to_string(), script.kind.clone()); - hm.insert("id".to_string(), script.id.clone().to_string()); - hm.insert( - "version_id".to_string(), - script.version_id.clone().to_string(), - ); - let embedding = Embedding { - id: script.ask_id.clone().to_string(), - vector: script.embedding.clone(), - metadata: Some(hm), - }; - self.db.insert_into_collection("scripts", embedding)?; - } - - let response = match hub_base_url.as_str() { - DEFAULT_HUB_BASE_URL => { - let response = HTTP_CLIENT - .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json") - .send() - .await; - if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { - tracing::warn!( - "Failed to get resource types embeddings from bucket, trying hub..." - ); - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/resource_types/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } else { - response.unwrap() - } - } - _ => { - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/resource_types/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } - }; - - if response.error_for_status_ref().is_err() { - return Err(anyhow!( - "Failed to get resource types embeddings from hub with error code: {}", - response.status() - )); - } - let hub_resource_types = response.json::>().await?; - - let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",) - .fetch_all(pg_db) - .await?; - - for rt in resource_types { - let mut hm = HashMap::new(); - hm.insert("name".to_string(), rt.name.clone()); - if let Some(schema) = rt.schema.clone() { - hm.insert("schema".to_string(), serde_json::to_string(&schema)?); - } - hm.insert("workspace".to_string(), rt.workspace_id.clone()); - let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name); - - let vector = if let Some(hub_rt) = hub_rt { - hub_rt.embedding.clone() - } else { - self.model_instance - .clone() - .create_embedding(&format!( - "{};{}", - rt.name, - rt.description.unwrap_or_default() - )) - .await? - }; - - let embedding = Embedding { - id: format!("{}_{}", rt.workspace_id, rt.name), - vector, - metadata: Some(hm), - }; - - self.db - .insert_into_collection("resource_types", embedding)?; - } - - Ok(()) - } - - pub async fn query_hub_scripts( - &self, - query: &str, - limit: Option, - kind: Option, - app: Option, - ) -> Result> { - let model_instance = self.model_instance.clone(); - let query_embedding = model_instance.create_embedding(query).await?; - - let collection = self.db.get_collection("scripts"); - - let collection = collection.ok_or(Error::msg("no collection found"))?; - - let filter = |embedding: &Embedding| { - if let Some(metadata) = embedding.metadata.as_ref() { - match ( - metadata.get("kind"), - kind.clone(), - metadata.get("app"), - app.clone(), - ) { - (Some(script_kind), Some(kind), Some(script_app), Some(app)) => { - &kind == script_kind && &app == script_app - } - (Some(script_kind), Some(kind), _, _) => &kind == script_kind, - (_, _, Some(script_app), Some(app)) => &app == script_app, - (_, None, _, None) => true, - _ => false, - } - } else { - false - } - }; - - let results = collection.get_similarity( - &query_embedding, - limit.unwrap_or(10) as usize, - Some(&filter), - Some(0.8), - ); - - let results: Result> = results - .iter() - .map(|r| { - let metadata = r - .embedding - .metadata - .as_ref() - .ok_or(Error::msg("no metadata"))?; - - Ok(HubScriptResult { - ask_id: metadata - .get("ask_id") - .ok_or(Error::msg("no ask_id"))? - .parse::()?, - summary: metadata - .get("summary") - .ok_or(Error::msg("no summary"))? - .to_owned(), - app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(), - kind: metadata - .get("kind") - .ok_or(Error::msg("no kind"))? - .to_owned(), - id: metadata - .get("id") - .ok_or(Error::msg("no id"))? - .parse::()?, - version_id: metadata - .get("version_id") - .ok_or(Error::msg("no version_id"))? - .parse::()?, - score: r.score, - }) - }) - .collect(); - - let mut results = results?; - - if results.len() > 1 { - let top_score = results[0].score; - results = results - .into_iter() - .take_while(|r| (top_score - r.score) / top_score <= 0.05) - .collect(); - } - - Ok(results) - } - - pub async fn query_resource_types( - &self, - workspace: String, - query: &str, - limit: Option, - ) -> Result> { - let model_instance = self.model_instance.clone(); - let query_embedding = model_instance.create_embedding(query).await?; - - let collection = self.db.get_collection("resource_types"); - - if collection.is_none() { - return Ok(vec![]); - } - - let collection = collection.ok_or(Error::msg("no collection found"))?; - - let filter = |embedding: &Embedding| { - if let Some(metadata) = embedding.metadata.as_ref() { - match metadata.get("workspace").map(|x| x.as_str()) { - Some("admins") => true, - Some(rt_workspace) => &workspace == rt_workspace, - _ => false, - } - } else { - false - } - }; - - let results = collection.get_similarity( - &query_embedding, - limit.unwrap_or(10) as usize, - Some(&filter), - Some(0.75), - ); - - let results: Result<_> = results - .iter() - .map(|r| { - let metadata = r - .embedding - .metadata - .as_ref() - .ok_or(Error::msg("no metadata"))?; - Ok(ResourceTypeResult { - name: metadata - .get("name") - .ok_or(Error::msg("no name"))? - .to_owned(), - schema: match metadata.get("schema") { - Some(schema) => serde_json::from_str(schema)?, - None => None, - }, - score: r.score, - }) - }) - .collect(); - - results - } -} - -#[cfg(feature = "embedding")] -fn normalize_l2(v: &Tensor) -> Result { - Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?) -} - -#[cfg(feature = "embedding")] -pub fn load_embeddings_db(db: &Pool) -> () { - let disable_embedding = std::env::var("DISABLE_EMBEDDING") - .ok() - .map(|x| x.parse::().unwrap_or(false)) - .unwrap_or(false); - - if !disable_embedding { - let db_clone = db.clone(); - tokio::spawn(async move { - let model_instance = ModelInstance::new().await; - if let Ok(model_instance) = model_instance { - let mut model_instance_lock = MODEL_INSTANCE.write().await; - *model_instance_lock = Some(Arc::new(model_instance)); - drop(model_instance_lock); - loop { - update_embeddings_db(&db_clone).await; - tokio::time::sleep(std::time::Duration::from_secs( - *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, - )) - .await; - } - } else { - tracing::error!( - "Failed to initialize model instance: {}", - model_instance.err().unwrap() - ); - } - }); - } -} - -#[cfg(feature = "embedding")] -pub async fn update_embeddings_db(db: &Pool) -> () { - if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { - tracing::info!("Creating embeddings DB..."); - let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; - if let Err(e) = new_embeddings_db.as_ref() { - tracing::error!("Failed to create embeddings db: {}", e); - } else { - let mut embeddings_db = EMBEDDINGS_DB.write().await; - *embeddings_db = new_embeddings_db.ok(); - tracing::info!("Created embeddings DB"); - } - } else { - tracing::error!("Could not update embeddings DB, model instance not initialized"); - } -} - -#[cfg(feature = "embedding")] -pub fn workspaced_service() -> Router { - Router::new().route("/query_resource_types", get(query_resource_types)) -} - -#[cfg(feature = "embedding")] -pub fn global_service() -> Router { - Router::new().route("/query_hub_scripts", get(query_hub_scripts)) -} - -#[cfg(not(feature = "embedding"))] -pub fn workspaced_service() -> Router { - Router::new() -} - -#[cfg(not(feature = "embedding"))] -pub fn global_service() -> Router { - Router::new() -} +pub use windmill_api_embeddings::*; diff --git a/backend/windmill-api/src/flow_conversations.rs b/backend/windmill-api/src/flow_conversations.rs index 4854615c18..6f3b879ff7 100644 --- a/backend/windmill-api/src/flow_conversations.rs +++ b/backend/windmill-api/src/flow_conversations.rs @@ -10,6 +10,7 @@ use sqlx::{FromRow, Postgres}; use uuid::Uuid; use crate::db::ApiAuthed; +pub use windmill_common::flow_conversations::FlowConversation; use windmill_common::{ db::{UserDB, DB}, error::{JsonResult, Result}, @@ -24,17 +25,6 @@ pub fn workspaced_service() -> Router { .route("/:conversation_id/messages", get(list_messages)) } -#[derive(Serialize, FromRow, Debug)] -pub struct FlowConversation { - pub id: Uuid, - pub workspace_id: String, - pub flow_path: String, - pub title: Option, - pub created_at: DateTime, - pub updated_at: DateTime, - pub created_by: String, -} - #[derive(Serialize, FromRow, Debug)] pub struct FlowConversationMessage { pub id: Uuid, @@ -104,55 +94,6 @@ async fn list_conversations( Ok(Json(conversations)) } -pub async fn get_or_create_conversation_with_id( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - w_id: &str, - flow_path: &str, - username: &str, - title: &str, - conversation_id: Uuid, -) -> Result { - // Check if conversation already exists - let existing_conversation = sqlx::query_as!( - FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by - FROM flow_conversation - WHERE id = $1 AND workspace_id = $2", - conversation_id, - w_id - ) - .fetch_optional(&mut **tx) - .await?; - - if let Some(existing) = existing_conversation { - return Ok(existing); - } - - // Truncate title to 25 char characters max - let title = if title.len() > 25 { - format!("{}...", &title[..25]) - } else { - title.to_string() - }; - - // Create new conversation with provided ID - let conversation = sqlx::query_as!( - FlowConversation, - "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) - VALUES ($1, $2, $3, $4, $5) - RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", - conversation_id, - w_id, - flow_path, - username, - title - ) - .fetch_one(&mut **tx) - .await?; - - Ok(conversation) -} - async fn delete_conversation( authed: ApiAuthed, Extension(user_db): Extension, @@ -187,24 +128,26 @@ async fn delete_conversation( tx.commit().await?; // Delete associated memory in background (non-blocking cleanup) - let w_id_clone = w_id.clone(); - let db_clone = db.clone(); - tokio::spawn(async move { - if let Err(e) = windmill_worker::memory_oss::delete_conversation_memory( - &db_clone, - &w_id_clone, - conversation_id, - ) - .await - { - tracing::error!( - "Failed to delete memory for conversation {} in workspace {}: {:?}", + { + let w_id_clone = w_id.clone(); + let db_clone = db.clone(); + tokio::spawn(async move { + if let Err(e) = windmill_common::flow_conversations::delete_conversation_memory( + &db_clone, + &w_id_clone, conversation_id, - w_id_clone, - e - ); - } - }); + ) + .await + { + tracing::error!( + "Failed to delete memory for conversation {} in workspace {}: {:?}", + conversation_id, + w_id_clone, + e + ); + } + }); + } Ok(format!("Conversation {} deleted", conversation_id)) } diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index fbfd64e2ef..cc1c3e97c0 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -6,1185 +6,22 @@ * LICENSE-AGPL for a copy of the license. */ -use std::collections::HashMap; +// Re-export everything from windmill-api-scripts +pub use windmill_api_scripts::flows::*; -use crate::utils::WithStarredInfoQuery; -use crate::{ - auth::{list_tokens_internal, TruncatedTokenWithEmail}, - db::{ApiAuthed, DB}, - schedule::clear_schedule, - triggers::{get_triggers_count_internal, TriggersCount}, - users::{maybe_refresh_folders, require_owner_of_path}, - utils::check_scopes, - webhook_util::{WebhookMessage, WebhookShared}, - HTTP_CLIENT, -}; -use axum::response::IntoResponse; +use crate::triggers::{get_triggers_count_internal, TriggersCount}; use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, + extract::{Extension, Path}, + routing::get, + Router, }; +use windmill_common::{error::JsonResult, utils::StripPath, DB}; -use hyper::StatusCode; -use serde::{Deserialize, Serialize}; -use sql_builder::prelude::*; -use sqlx::{FromRow, Postgres, Transaction}; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::assets::{clear_static_asset_usage, AssetUsageKind}; -use windmill_common::min_version::{ - MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2, -}; -use windmill_common::runnable_settings::RunnableSettingsTrait; -use windmill_common::utils::query_elems_from_hub; -use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; -use windmill_common::HUB_BASE_URL; -use windmill_common::{ - db::UserDB, - error::{self, to_anyhow, Error, JsonResult, Result}, - flows::{Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, - jobs::JobPayload, - schedule::Schedule, - scripts::Schema, - utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, -}; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; -use windmill_queue::WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT; -use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel}; -use windmill_worker::scoped_dependency_map::ScopedDependencyMap; - +/// Wraps the subcrate's workspaced_service with the trigger count route +/// that depends on windmill-api internals. pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_flows)) - .route("/list_search", get(list_search_flows)) - .route("/create", post(create_flow)) - .route("/update/*path", post(update_flow)) - .route("/archive/*path", post(archive_flow_by_path)) - .route("/delete/*path", delete(delete_flow_by_path)) + windmill_api_scripts::flows::workspaced_service() .route("/get_triggers_count/*path", get(get_triggers_count)) - .route("/list_tokens/*path", get(list_tokens)) - .route("/get/*path", get(get_flow_by_path)) - .route("/deployment_status/p/*path", get(get_deployment_status)) - .route("/get/draft/*path", get(get_flow_by_path_w_draft)) - .route("/exists/*path", get(exists_flow_by_path)) - .route("/list_paths", get(list_paths)) - .route("/history/p/*path", get(get_flow_history)) - .route("/get_latest_version/*path", get(get_latest_version)) - .route( - "/list_paths_from_workspace_runnable/:runnable_kind/*path", - get(list_paths_from_workspace_runnable), - ) - .route("/history_update/v/:version", post(update_flow_history)) - .route("/get/v/:version", get(get_flow_version_by_id)) - .route("/get/v/:version/p/*path", get(get_flow_version)) - .route( - "/toggle_workspace_error_handler/*path", - post(toggle_workspace_error_handler), - ) -} - -pub fn global_service() -> Router { - Router::new() - .route("/hub/list", get(list_hub_flows)) - .route("/hub/get/:id", get(get_hub_flow_by_id)) -} - -#[derive(Serialize, FromRow)] -pub struct SearchFlow { - path: String, - value: sqlx::types::Json>, -} -async fn list_search_flows( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult> { - #[cfg(feature = "enterprise")] - let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; - let mut tx = user_db.begin(&authed).await?; - - let rows = sqlx::query_as::<_, SearchFlow>( - "SELECT flow.path, flow_version.value - FROM flow - LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.workspace_id = $1 LIMIT $2", - ) - .bind(&w_id) - .bind(n) - .fetch_all(&mut *tx) - .await? - .into_iter() - .collect::>(); - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_flows( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, - Query(lq): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - - let mut sqlb = SqlBuilder::select_from("flow as o") - .fields(&[ - "o.workspace_id", - "o.path", - "summary", - if !lq.without_description.unwrap_or(false) { - "description" - } else { - "NULL as description" - }, - "fv.created_by as edited_by", - "fv.created_at as edited_at", - "archived", - "extra_perms", - "favorite.path IS NOT NULL as starred", - "draft.path IS NOT NULL as has_draft", - "draft_only", - "ws_error_handler_muted" - ]) - .left() - .join("favorite") - .on( - "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?" - .bind(&authed.username), - ) - .left() - .join("draft") - .on( - "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'" - ) - .left() - .join("flow_version fv") - .on( - "fv.id = o.versions[array_upper(o.versions, 1)]" - ) - .order_desc("favorite.path IS NOT NULL") - .order_by("fv.created_at", lq.order_desc.unwrap_or(true)) - .and_where("o.workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page) - .clone(); - - sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false)); - - if let Some(ps) = &lq.path_start { - sqlb.and_where_like_left("o.path", ps); - } - if let Some(p) = &lq.path_exact { - sqlb.and_where_eq("o.path", "?".bind(p)); - } - if let Some(cb) = &lq.edited_by { - sqlb.and_where_eq("fv.created_by", "?".bind(cb)); - } - if lq.starred_only.unwrap_or(false) { - sqlb.and_where_is_not_null("favorite.path"); - } - - if !lq.include_draft_only.unwrap_or(false) || authed.is_operator { - sqlb.and_where("o.draft_only IS NOT TRUE"); - } - if let Some(dw) = &lq.dedicated_worker { - sqlb.and_where_eq("dedicated_worker", dw); - } - - if lq.with_deployment_msg.unwrap_or(false) { - sqlb.join("deployment_metadata dm") - .left() - .on("dm.flow_version = o.versions[array_upper(o.versions, 1)]") - .fields(&["dm.deployment_msg"]); - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as::<_, ListableFlow>(&sql) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_hub_flows(Extension(db): Extension) -> impl IntoResponse { - let (status_code, headers, response) = query_elems_from_hub( - &HTTP_CLIENT, - &format!( - "{}/searchFlowData?approved=true", - *HUB_BASE_URL.read().await - ), - None, - &db, - ) - .await?; - Ok::<_, Error>((status_code, headers, response)) -} - -async fn list_paths( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - - let flows = sqlx::query_scalar!( - "SELECT distinct(path) FROM flow WHERE workspace_id = $1", - w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(flows)) -} - -pub async fn get_hub_flow_by_id( - Path(id): Path, - Extension(db): Extension, -) -> JsonResult> { - let value = http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/flows/{}/json", *HUB_BASE_URL.read().await, id), - false, - None, - Some(&db), - ) - .await? - .json() - .await - .map_err(to_anyhow)?; - Ok(Json(value)) -} - -#[derive(Deserialize)] -pub struct ToggleWorkspaceErrorHandler { - #[cfg(feature = "enterprise")] - pub muted: Option, -} - -#[cfg(not(feature = "enterprise"))] -async fn toggle_workspace_error_handler( - _authed: ApiAuthed, - Extension(_user_db): Extension, - Path((_w_id, _path)): Path<(String, StripPath)>, - Json(_req): Json, -) -> Result { - return Err(Error::BadRequest( - "Muting the error handler for certain flow is only available in enterprise version" - .to_string(), - )); -} - -#[cfg(feature = "enterprise")] -async fn toggle_workspace_error_handler( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(req): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - let error_handler_maybe: Option = sqlx::query_scalar!( - r#" - SELECT - error_handler->>'path' - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(None); - - let response = match error_handler_maybe { - Some(_) => { - sqlx::query_scalar!( - r#" - UPDATE - flow - SET - ws_error_handler_muted = $3 - WHERE - path = $1 AND - workspace_id = $2 - "#, - path.to_path(), - w_id, - req.muted, - ) - .execute(&mut *tx) - .await?; - Ok("".to_string()) - } - None => Err(Error::BadRequest( - "Workspace error handler needs to be defined".to_string(), - )), - }; - - tx.commit().await?; - - return response; -} - -async fn check_path_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - path: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!("Flow {} already exists", path))); - } - return Ok(()); -} - -#[derive(Deserialize)] -struct ListPathsFromWorkspaceRunnableQuery { - match_path_start: Option, -} - -async fn list_paths_from_workspace_runnable( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, - Query(query): Query, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || { - format!("flows:read:{}", format!("{}/{}", runnable_kind, path)) - })?; - let mut tx = user_db.begin(&authed).await?; - - let runnables = if query.match_path_start.unwrap_or(false) { - sqlx::query_scalar!( - r#"SELECT DISTINCT f.path - FROM workspace_runnable_dependencies wru - JOIN flow f - ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path LIKE $1 || '%' AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, - path, - matches!(runnable_kind, RunnableKind::Flow), - w_id - ) - .fetch_all(&mut *tx) - .await? - } else { - sqlx::query_scalar!( - r#"SELECT f.path - FROM workspace_runnable_dependencies wru - JOIN flow f - ON wru.flow_path = f.path AND wru.workspace_id = f.workspace_id - WHERE wru.runnable_path = $1 AND wru.runnable_is_flow = $2 AND wru.workspace_id = $3"#, - path, - matches!(runnable_kind, RunnableKind::Flow), - w_id - ) - .fetch_all(&mut *tx) - .await? - }; - - tx.commit().await?; - Ok(Json(runnables)) -} - -async fn validate_flow(new_flow: &NewFlow) -> error::Result<()> { - #[cfg(not(feature = "enterprise"))] - if new_flow.ws_error_handler_muted.is_some_and(|val| val) { - return Err(Error::BadRequest( - "Muting the error handler for certain flow is only available in enterprise version" - .to_string(), - )); - } - - guard_flow_from_debounce_data(new_flow).await?; - - return Ok(()); -} - -async fn create_flow( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(nf): Json, -) -> Result<(StatusCode, String)> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot create flows for security reasons".to_string(), - )); - } - check_scopes(&authed, || format!("flows:write:{}", nf.path))?; - validate_flow(&nf).await?; - if *CLOUD_HOSTED { - let nb_flows = - sqlx::query_scalar!("SELECT COUNT(*) FROM flow WHERE workspace_id = $1", &w_id) - .fetch_one(&db) - .await?; - if nb_flows.unwrap_or(0) >= 1000 { - return Err(Error::BadRequest( - "You have reached the maximum number of flows (1000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - if nf.summary.len() > 300 { - return Err(Error::BadRequest( - "Summary must be less than 300 characters on cloud".to_string(), - )); - } - if nf - .description - .as_ref() - .is_some_and(|desc| desc.len() > 3000) - { - return Err(Error::BadRequest( - "Description must be less than 3000 characters on cloud".to_string(), - )); - } - } - - // cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?; - let authed = maybe_refresh_folders(&nf.path, &w_id, authed, &db).await; - - let mut tx = user_db.clone().begin(&authed).await?; - - check_path_conflict(&mut tx, &w_id, &nf.path).await?; - check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; - - let schema_str = nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()); - sqlx::query!( - r#"INSERT INTO flow ( - workspace_id, path, summary, description, - dependency_job, lock_error_logs, draft_only, tag, - dedicated_worker, visible_to_runner_only, on_behalf_of_email, - ws_error_handler_muted, - value, schema, edited_by, edited_at - ) VALUES ( - $1, $2, $3, $4, - NULL, '', $5, $6, - $7, $8, $9, - $10, - $11, $12::text::json, $13, now() - )"#, - w_id, - nf.path, - nf.summary, - nf.description.as_deref().unwrap_or(""), - nf.draft_only, - nf.tag, - nf.dedicated_worker, - nf.visible_to_runner_only.unwrap_or(false), - nf.on_behalf_of_email.and(Some(&authed.email)), - nf.ws_error_handler_muted.unwrap_or(false), - sqlx::types::Json(&nf.value) as _, - schema_str, - &authed.username, - ) - .execute(&mut *tx) - .await?; - - let version = sqlx::query_scalar!( - "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) - VALUES ($1, $2, $3, $4::text::json, $5) - RETURNING id", - w_id, - nf.path, - sqlx::types::Json(nf.value) as _, - schema_str, - &authed.username, - ) - .fetch_one(&mut *tx) - .await?; - - sqlx::query!( - "UPDATE flow SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3", - version, - nf.path, - w_id - ).execute(&mut *tx).await?; - - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'", - nf.path, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "flows.create", - ActionKind::Create, - &w_id, - Some(&nf.path.to_string()), - Some( - [Some(("flow", nf.path.as_str()))] - .into_iter() - .flatten() - .collect(), - ), - ) - .await?; - - let mut args: HashMap> = HashMap::new(); - if let Some(dm) = nf.deployment_message { - args.insert("deployment_message".to_string(), to_raw_value(&dm)); - } - - let tx = PushIsolationLevel::Transaction(tx); - let (dependency_job_uuid, mut new_tx) = push( - &db, - tx, - &w_id, - JobPayload::FlowDependencies { - path: nf.path.clone(), - dedicated_worker: nf.dedicated_worker, - version: version, - debouncing_settings: Default::default(), - }, - windmill_queue::PushArgs { args: &args, extra: None }, - &authed.username, - &authed.email, - windmill_common::users::username_to_permissioned_as(&authed.username), - authed.token_prefix.as_deref(), - None, - None, - None, - None, - None, - None, - false, - false, - None, - true, - None, - None, - None, - None, - Some(&authed.clone().into()), - false, - None, - None, - None, - ) - .await?; - - sqlx::query!( - "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3", - dependency_job_uuid, - nf.path, - w_id - ) - .execute(&mut *new_tx) - .await?; - - // Store the job_id in deployment_metadata for this flow deployment - sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) - VALUES ($1, $2, $3, $4) - ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL - DO UPDATE SET job_id = EXCLUDED.job_id", - w_id, - nf.path, - version, - dependency_job_uuid - ) - .execute(&mut *new_tx) - .await?; - - new_tx.commit().await?; - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() }, - ); - - Ok((StatusCode::CREATED, nf.path.to_string())) -} - -async fn check_schedule_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - path: &str, -) -> error::Result<()> { - let exists_flow = sqlx::query_scalar!( - "SELECT EXISTS (SELECT 1 FROM schedule WHERE path = $1 AND workspace_id = $2 AND path != \ - script_path)", - path, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists_flow { - return Err(error::Error::BadConfig(format!( - "A flow cannot have the same path as a schedule if the schedule does not trigger that \ - same flow: {path}", - ))); - }; - Ok(()) -} - -pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { - return crate::users::require_is_writer( - authed, - path, - w_id, - db, - "SELECT extra_perms FROM flow WHERE path = $1 AND workspace_id = $2", - "flow", - ) - .await; -} - -#[derive(Serialize)] -pub struct FlowVersion { - pub id: i64, - pub created_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, -} - -async fn get_flow_history( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let flows = sqlx::query_as!( - FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version - LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version - WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 - ORDER BY flow_version.created_at DESC", - path, - w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(flows)) -} - -async fn get_latest_version( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let version = sqlx::query_as!( - FlowVersion, - "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version - LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version - WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 - ORDER BY flow_version.created_at DESC", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(version)) -} - -async fn get_flow_version( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, version, path)): Path<(String, i64, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let flow = sqlx::query_as::<_, Flow>( - "SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by - FROM flow - LEFT JOIN flow_version ON flow_version.path = flow.path AND flow_version.workspace_id = flow.workspace_id - WHERE flow.path = $1 AND flow.workspace_id = $2 AND flow_version.id = $3", - ) - .bind(path) - .bind(w_id) - .bind(version) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - - let flow = not_found_if_none(flow, "Flow version", version.to_string())?; - - Ok(Json(flow)) -} - -async fn get_flow_version_by_id( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, version)): Path<(String, i64)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - - // First, fetch the path to perform authorization check early - let path: Option = - sqlx::query_scalar("SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2") - .bind(version) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - let path = not_found_if_none( - path, - "Flow version", - format!("{} in workspace {}", version, w_id), - )?; - - // Perform authorization check before fetching full data - check_scopes(&authed, || format!("flows:read:{}", path))?; - - // Now fetch the full flow data with INNER JOIN to ensure flow exists - let flow = sqlx::query_as::<_, Flow>( - "SELECT - flow.workspace_id, - flow.path, - flow.summary, - flow.description, - flow.archived, - flow.extra_perms, - flow.draft_only, - flow.dedicated_worker, - flow.tag, - flow.ws_error_handler_muted, - flow.timeout, - flow.visible_to_runner_only, - flow.on_behalf_of_email, - flow_version.schema, - flow_version.value, - flow_version.created_at as edited_at, - flow_version.created_by as edited_by - FROM flow - INNER JOIN flow_version - ON flow_version.path = flow.path - AND flow_version.workspace_id = flow.workspace_id - WHERE flow_version.id = $1 AND flow.workspace_id = $2", - ) - .bind(version) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - - let flow = not_found_if_none( - flow, - "Flow", - format!("for version {} (flow may have been deleted)", version), - )?; - - Ok(Json(flow)) -} - -#[derive(Deserialize)] -pub struct FlowHistoryUpdate { - pub deployment_msg: String, -} - -async fn update_flow_history( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, version)): Path<(String, i64)>, - Json(history_update): Json, -) -> Result<()> { - let mut tx = user_db.begin(&authed).await?; - - // Fetch path and perform authorization check early - let path: Option = - sqlx::query_scalar("SELECT path FROM flow_version WHERE workspace_id = $1 AND id = $2") - .bind(&w_id) - .bind(version) - .fetch_optional(&mut *tx) - .await?; - - let path = not_found_if_none( - path, - "Flow version", - format!("{} in workspace {}", version, w_id), - )?; - - // Perform authorization check before any modifications - check_scopes(&authed, || format!("flows:write:{}", path))?; - - // Insert or update deployment metadata - sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, flow_version, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", - &w_id, - path, - version, - history_update.deployment_msg, - ) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - Ok(()) -} - -async fn update_flow( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Extension(webhook): Extension, - Path((w_id, flow_path)): Path<(String, StripPath)>, - Json(nf): Json, -) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update flows for security reasons".to_string(), - )); - } - let flow_path = flow_path.to_path(); - check_scopes(&authed, || format!("flows:write:{}", flow_path))?; - validate_flow(&nf).await?; - - let authed = maybe_refresh_folders(&flow_path, &w_id, authed, &db).await; - let mut tx = user_db.clone().begin(&authed).await?; - - check_schedule_conflict(&mut tx, &w_id, flow_path).await?; - - let schema = nf.schema.map(|x| x.0); - let old_dep_job = sqlx::query_scalar!( - "SELECT dependency_job FROM flow WHERE path = $1 AND workspace_id = $2", - flow_path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?; - let is_new_path = nf.path != flow_path; - let schema_str = schema.and_then(|x| serde_json::to_string(&x).ok()); - - sqlx::query!( - " - UPDATE - flow - SET - path = $1, - summary = $2, - description = $3, - dependency_job = NULL, - lock_error_logs = '', - draft_only = NULL, - tag = $4, - dedicated_worker = $5, - visible_to_runner_only = $6, - on_behalf_of_email = $7, - ws_error_handler_muted = $8, - value = $9, - schema = $10::text::json, - edited_by = $11, - edited_at = now() - WHERE - path = $12 AND workspace_id = $13", - if is_new_path { flow_path } else { &nf.path }, - nf.summary, - nf.description.as_deref().unwrap_or(""), - nf.tag, - nf.dedicated_worker, - nf.visible_to_runner_only.unwrap_or(false), - nf.on_behalf_of_email.and(Some(&authed.email)), - nf.ws_error_handler_muted.unwrap_or(false), - sqlx::types::Json(&nf.value) as _, - schema_str, - authed.username, - flow_path, - w_id, - ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!("Error updating flow due to flow update: {e:#}")) - })?; - - if is_new_path { - // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint - sqlx::query!( - "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) - SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at - FROM flow - WHERE path = $2 AND workspace_id = $3", - nf.path, - flow_path, - w_id - ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!("Error updating flow due to create new flow: {e:#}")) - })?; - - sqlx::query!( - "UPDATE flow_version SET path = $1 WHERE path = $2 AND workspace_id = $3", - nf.path, - flow_path, - w_id - ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating flow due to updating flow history path: {e:#}" - )) - })?; - - sqlx::query!( - "DELETE FROM flow WHERE path = $1 AND workspace_id = $2", - flow_path, - w_id - ) - .execute(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating flow due to deleting old flow: {e:#}" - )) - })?; - - sqlx::query!( - "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS TRUE", - nf.path, - flow_path, - w_id - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS TRUE", - nf.path, - flow_path, - w_id - ) - .execute(&mut *tx) - .await?; - } - - // tracing::error!("Updating flow: {:?}", nf.value.get()); - - // This will lock anyone who is trying to iterate on flow_versions with given path and parameters. - let version = sqlx::query_scalar!( - "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) RETURNING id", - w_id, - nf.path, - sqlx::types::Json(nf.value) as _, - schema_str, - &authed.username, - ) - .fetch_one(&mut *tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating flow due to flow history insert: {e:#}" - )) - })?; - - // TODO: This should happen only after we are done with dependency job. - sqlx::query!( - "UPDATE flow SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3", - version, nf.path, w_id - ).execute(&mut *tx).await?; - - if is_new_path { - check_schedule_conflict(&mut tx, &w_id, &nf.path).await?; - - if !authed.is_admin { - require_owner_of_path(&authed, flow_path)?; - } - } - - let mut schedulables: Vec = sqlx::query_as::<_, Schedule>( - "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *") - .bind(&nf.path) - .bind(&flow_path) - .bind(&w_id) - .fetch_all(&mut *tx) - .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to related schedules update: {e:#}")))?; - - let schedule = sqlx::query_as::<_, Schedule>( - "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *") - .bind(&nf.path) - .bind(&flow_path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await.map_err(|e| error::Error::internal_err(format!("Error updating flow due to related schedule update: {e:#}")))?; - - if let Some(schedule) = schedule { - clear_schedule(&mut tx, &flow_path, &w_id).await?; - schedulables.push(schedule); - } - - for schedule in schedulables.into_iter() { - clear_schedule(&mut tx, &schedule.path, &w_id).await?; - - if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; - } - } - - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'", - flow_path, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "flows.update", - ActionKind::Create, - &w_id, - Some(&nf.path.to_string()), - Some( - [Some(("flow", nf.path.as_str()))] - .into_iter() - .flatten() - .collect(), - ), - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateFlow { - workspace: w_id.clone(), - old_path: flow_path.to_owned(), - new_path: nf.path.clone(), - }, - ); - - let tx = PushIsolationLevel::Transaction(tx); - - let mut args: HashMap> = HashMap::new(); - if let Some(dm) = nf.deployment_message { - args.insert("deployment_message".to_string(), to_raw_value(&dm)); - } - args.insert("parent_path".to_string(), to_raw_value(&flow_path)); - - let (dependency_job_uuid, mut new_tx) = push( - &db, - tx, - &w_id, - JobPayload::FlowDependencies { - path: nf.path.clone(), - dedicated_worker: nf.dedicated_worker, - version, - debouncing_settings: Default::default(), - }, - windmill_queue::PushArgs { args: &args, extra: None }, - &authed.username, - &authed.email, - windmill_common::users::username_to_permissioned_as(&authed.username), - authed.token_prefix.as_deref(), - None, - None, - None, - None, - None, - None, - false, - false, - None, - true, - None, - None, - None, - None, - Some(&authed.clone().into()), - false, - None, - None, - None, - ) - .await?; - - sqlx::query!( - "UPDATE flow SET dependency_job = $1 WHERE path = $2 AND workspace_id = $3", - dependency_job_uuid, - nf.path, - w_id - ) - .execute(&mut *new_tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating flow due to updating dependency job field: {e:#}" - )) - })?; - - // Store the job_id in deployment_metadata for this flow deployment - sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) - VALUES ($1, $2, $3, $4) - ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL - DO UPDATE SET job_id = EXCLUDED.job_id", - w_id, - nf.path, - version, - dependency_job_uuid - ) - .execute(&mut *new_tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating deployment_metadata with job_id: {e:#}" - )) - })?; - - if let Some(old_dep_job) = old_dep_job { - sqlx::query!( - "UPDATE v2_job_queue SET - canceled_by = $2, - canceled_reason = 're-deployment' - WHERE id = $1", - old_dep_job, - &authed.username - ) - .execute(&mut *new_tx) - .await - .map_err(|e| { - error::Error::internal_err(format!( - "Error updating flow due to cancelling dependency job: {e:#}" - )) - })?; - } - - new_tx.commit().await?; - - Ok(nf.path.to_string()) } async fn get_triggers_count( @@ -1194,762 +31,3 @@ async fn get_triggers_count( let path = path.to_path(); get_triggers_count_internal(&db, &w_id, &path, true).await } - -async fn list_tokens( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - list_tokens_internal(&db, &w_id, &path, true).await -} - -#[derive(Serialize)] -struct DeploymentStatus { - lock_error_logs: Option, - job_id: Option, -} -async fn get_deployment_status( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let mut tx = db.begin().await?; - let status_o = sqlx::query!( - "SELECT f.lock_error_logs, dm.job_id - FROM flow f - LEFT JOIN deployment_metadata dm ON f.versions[array_upper(f.versions, 1)] = dm.flow_version - AND f.workspace_id = dm.workspace_id AND f.path = dm.path - WHERE f.path = $1 AND f.workspace_id = $2", - path, - w_id, - ) - .fetch_optional(&mut *tx) - .await?; - - let status = not_found_if_none(status_o, "DeploymentStatus", path)?; - - let deployment_status = - DeploymentStatus { lock_error_logs: status.lock_error_logs, job_id: status.job_id }; - - tx.commit().await?; - Ok(Json(deployment_status)) -} - -async fn get_flow_by_path( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let flow_o = if query.with_starred_info.unwrap_or(false) { - sqlx::query_as::<_, FlowWithStarred>( - r#" - SELECT - flow.workspace_id, - flow.path, - flow.lock_error_logs, - flow.summary, - flow.description, - flow.archived, - flow.extra_perms, - flow.draft_only, - flow.dedicated_worker, - flow.tag, - flow.ws_error_handler_muted, - flow.timeout, - flow.visible_to_runner_only, - flow.on_behalf_of_email, - flow_version.id AS version_id, - flow_version.schema, - flow_version.value, - flow_version.created_at AS edited_at, - flow_version.created_by AS edited_by, - favorite.path IS NOT NULL AS starred - FROM flow - LEFT JOIN favorite - ON favorite.favorite_kind = 'flow' - AND favorite.workspace_id = flow.workspace_id - AND favorite.path = flow.path - AND favorite.usr = $3 - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 AND flow.workspace_id = $2 - "#, - ) - .bind(path) - .bind(w_id) - .bind(&authed.username) - .fetch_optional(&mut *tx) - .await? - } else { - sqlx::query_as::<_, FlowWithStarred>( - r#" - SELECT - flow.workspace_id, - flow.path, - flow.lock_error_logs, - flow.summary, - flow.description, - flow.archived, - flow.extra_perms, - flow.draft_only, - flow.dedicated_worker, - flow.tag, - flow.ws_error_handler_muted, - flow.timeout, - flow.visible_to_runner_only, - flow.on_behalf_of_email, - flow_version.id AS version_id, - flow_version.schema, - flow_version.value, - flow_version.created_at AS edited_at, - flow_version.created_by AS edited_by, - NULL AS starred - FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 AND flow.workspace_id = $2 - "#, - ) - .bind(path) - .bind(w_id) - .fetch_optional(&mut *tx) - .await? - }; - - tx.commit().await?; - - let flow = not_found_if_none(flow_o, "Flow", path)?; - Ok(Json(flow)) -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct FlowWDraft { - pub path: String, - pub summary: String, - pub description: String, - pub schema: Option, - pub value: sqlx::types::Json>, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dedicated_worker: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub visible_to_runner_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_behalf_of_email: Option, -} - -async fn get_flow_by_path_w_draft( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let flow_o = sqlx::query_as::<_, FlowWDraft>( - "SELECT - flow.path, - flow.summary, - flow.description, - flow_version.schema, - flow_version.value, - flow.extra_perms, - flow.draft_only, - flow.ws_error_handler_muted, - flow.dedicated_worker, - draft.value AS draft, - flow.tag, - flow.visible_to_runner_only, - flow.on_behalf_of_email - FROM flow - LEFT JOIN draft - ON flow.path = draft.path - AND draft.workspace_id = $2 - AND draft.typ = 'flow' - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 - AND flow.workspace_id = $2", - ) - .bind(path) - .bind(w_id) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - - let flow = not_found_if_none(flow_o, "Flow", path)?; - Ok(Json(flow)) -} - -async fn exists_flow_by_path( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM flow WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -#[derive(Deserialize)] -struct Archived { - archived: Option, -} - -async fn archive_flow_by_path( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(archived): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3", - archived.archived.unwrap_or(true), - path, - &w_id - ) - .execute(&mut *tx) - .await?; - - clear_static_asset_usage(&mut *tx, &w_id, path, AssetUsageKind::Flow).await?; - - audit_log( - &mut *tx, - &authed, - "flows.archive", - ActionKind::Delete, - &w_id, - Some(path), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - - ScopedDependencyMap::clear_map_for_item(path, &w_id, "flow", tx, &None) - .await - .commit() - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Flow { - path: path.to_string(), - parent_path: Some(path.to_string()), - version: 0, // dummy version as it will not get inserted in db - }, - Some(format!( - "Flow '{}' {}", - path, - if archived.archived.unwrap_or(true) { - "archived" - } else { - "unarchived" - } - )), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::ArchiveFlow { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("Flow {path} archived")) -} - -/// Validates that flow debouncing configuration is supported by all workers -/// Returns an error if debouncing is configured but workers are behind required version -async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> { - if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await - && !nf.parse_flow_value()?.debouncing_settings.is_default() - { - tracing::warn!( - "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" - ); - Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() }) - } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await - && !nf - .parse_flow_value()? - .debouncing_settings - .is_legacy_compatible() - && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT - { - tracing::warn!( - "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" - ); - Err(Error::WorkersAreBehind { - feature: "V2 Debouncing".into(), - min_version: "1.597.0".into(), - }) - } else { - Ok(()) - } -} - -#[derive(Deserialize)] -struct DeleteFlowQuery { - keep_captures: Option, -} - -async fn delete_flow_by_path( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("flows:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'", - path, - &w_id - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM flow WHERE path = $1 AND workspace_id = $2", - path, - &w_id - ) - .execute(&mut *tx) - .await?; - - if !query.keep_captures.unwrap_or(false) { - sqlx::query!( - "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", - path, - &w_id - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", - path, - &w_id - ) - .execute(&mut *tx) - .await?; - } - - audit_log( - &mut *tx, - &authed, - "flows.delete", - ActionKind::Delete, - &w_id, - Some(path), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Flow { - path: path.to_string(), - parent_path: Some(path.to_string()), - version: 0, // dummy version as it will not get inserted in db - }, - Some(format!("Flow '{}' deleted", path)), - true, - None, - ) - .await?; - - sqlx::query!( - "DELETE FROM deployment_metadata WHERE path = $1 AND workspace_id = $2 AND script_hash IS NULL and app_version IS NULL", - path, - w_id - ) - .execute(&db) - .await - .map_err(|e| { - Error::internal_err(format!( - "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" - )) - })?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteFlow { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("Flow {path} deleted")) -} - -#[cfg(test)] -mod tests { - - use std::{collections::HashMap, time::Duration}; - - use windmill_common::{ - flows::{ - ConstantDelay, ExponentialDelay, FlowModule, FlowModuleValue, FlowValue, - InputTransform, Retry, StopAfterIf, - }, - runnable_settings::{ - ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, - }, - scripts, - }; - - const SECOND: Duration = Duration::from_secs(1); - - #[test] - fn flowmodule_serde() { - let fv = FlowValue { - modules: vec![ - FlowModule { - id: "a".to_string(), - value: windmill_common::worker::to_raw_value(&FlowModuleValue::Script { - path: "test".to_string(), - input_transforms: [( - "test".to_string(), - InputTransform::Static { - value: windmill_common::worker::to_raw_value(&"test2".to_string()), - }, - )] - .into(), - hash: None, - tag_override: None, - is_trigger: None, - pass_flow_input_directly: None, - }), - stop_after_if: None, - stop_after_all_iters_if: None, - summary: None, - suspend: Default::default(), - retry: None, - sleep: None, - cache_ttl: None, - cache_ignore_s3_path: None, - mock: None, - timeout: None, - priority: None, - delete_after_use: None, - continue_on_error: None, - skip_if: None, - apply_preprocessor: None, - pass_flow_input_directly: None, - }, - FlowModule { - id: "b".to_string(), - value: windmill_common::worker::to_raw_value(&FlowModuleValue::RawScript { - input_transforms: HashMap::new(), - content: "test".to_string(), - language: scripts::ScriptLang::Deno, - path: None, - lock: None, - tag: None, - is_trigger: None, - assets: None, - concurrency_settings: ConcurrencySettingsWithCustom::default(), - }), - stop_after_if: Some(StopAfterIf { - expr: "foo = 'bar'".to_string(), - ..Default::default() - }), - stop_after_all_iters_if: None, - summary: None, - suspend: Default::default(), - retry: None, - sleep: None, - cache_ttl: None, - cache_ignore_s3_path: None, - mock: None, - timeout: None, - priority: None, - delete_after_use: None, - continue_on_error: None, - skip_if: None, - apply_preprocessor: None, - pass_flow_input_directly: None, - }, - FlowModule { - id: "c".to_string(), - value: windmill_common::worker::to_raw_value(&FlowModuleValue::ForloopFlow { - iterator: InputTransform::Static { - value: windmill_common::worker::to_raw_value(&[1, 2, 3]), - }, - modules: vec![], - modules_node: None, - skip_failures: true, - parallel: false, - parallelism: None, - squash: None, - }), - stop_after_if: Some(StopAfterIf { - expr: "previous.isEmpty()".to_string(), - ..Default::default() - }), - stop_after_all_iters_if: None, - summary: None, - suspend: Default::default(), - retry: None, - sleep: None, - cache_ttl: None, - cache_ignore_s3_path: None, - mock: None, - timeout: None, - priority: None, - delete_after_use: None, - continue_on_error: None, - skip_if: None, - apply_preprocessor: None, - pass_flow_input_directly: None, - }, - ], - failure_module: Some(Box::new(FlowModule { - id: "d".to_string(), - value: FlowModuleValue::Script { - path: "test".to_string(), - input_transforms: HashMap::new(), - hash: None, - tag_override: None, - is_trigger: None, - pass_flow_input_directly: None, - } - .into(), - stop_after_if: Some(StopAfterIf { - expr: "previous.isEmpty()".to_string(), - ..Default::default() - }), - stop_after_all_iters_if: None, - summary: None, - suspend: Default::default(), - retry: None, - sleep: None, - cache_ttl: None, - cache_ignore_s3_path: None, - mock: None, - timeout: None, - priority: None, - delete_after_use: None, - continue_on_error: None, - skip_if: None, - apply_preprocessor: None, - pass_flow_input_directly: None, - })), - preprocessor_module: None, - same_worker: false, - skip_expr: None, - cache_ttl: None, - cache_ignore_s3_path: None, - priority: None, - early_return: None, - chat_input_enabled: None, - flow_env: None, - concurrency_settings: ConcurrencySettings::default(), - debouncing_settings: DebouncingSettings::default(), - }; - let expect = serde_json::json!({ - "modules": [ - { - "id": "a", - "value": { - "input_transforms": { - "test": { - "type": "static", - "value": "test2" - } - }, - "type": "script", - "path": "test", - }, - }, - { - "id": "b", - "value": { - "input_transforms": {}, - "type": "rawscript", - "content": "test", - "language": "deno" - }, - "stop_after_if": { - "expr": "foo = 'bar'", - "skip_if_stopped": false, - "error_message": null - } - }, - { - "id": "c", - "value": { - "type": "forloopflow", - "iterator": { - "type": "static", - "value": [ - 1, - 2, - 3 - ] - }, - "parallel": false, - "skip_failures": true, - "modules": [] - }, - "stop_after_if": { - "expr": "previous.isEmpty()", - "skip_if_stopped": false, - "error_message": null - } - } - ], - "failure_module": { - "id": "d", - "value": { - "input_transforms": {}, - "type": "script", - "path": "test", - }, - "stop_after_if": { - "expr": "previous.isEmpty()", - "skip_if_stopped": false, - "error_message": null - } - }, - }); - assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect)); - } - - #[test] - fn retry_serde() { - assert_eq!(Retry::default(), serde_json::from_str(r#"{}"#).unwrap()); - - assert_eq!( - Retry::default(), - serde_json::from_str( - r#" - { - "constant": { - "seconds": 0 - }, - "exponential": { - "multiplier": 1, - "seconds": 0 - }, - "retry_if": null - } - "# - ) - .unwrap() - ); - - assert_eq!( - Retry { - constant: Default::default(), - exponential: ExponentialDelay { - attempts: 0, - multiplier: 1, - seconds: 123, - random_factor: None - }, - retry_if: None - }, - serde_json::from_str( - r#" - { - "constant": {}, - "exponential": { "seconds": 123 }, - "retry_if" : null - } - "# - ) - .unwrap() - ); - } - - #[test] - fn retry_exponential() { - let retry = Retry { - constant: ConstantDelay::default(), - exponential: ExponentialDelay { - attempts: 3, - multiplier: 4, - seconds: 3, - random_factor: None, - }, - retry_if: None, - }; - assert_eq!( - vec![ - Some(12 * SECOND), - Some(36 * SECOND), - Some(108 * SECOND), - None - ], - (0..4) - .map(|previous_attempts| retry.interval(previous_attempts, false)) - .collect::>() - ); - - assert_eq!(Some(108 * SECOND), retry.max_interval()); - } - - #[test] - fn retry_both() { - let retry = Retry { - constant: ConstantDelay { attempts: 2, seconds: 4 }, - exponential: ExponentialDelay { - attempts: 2, - multiplier: 1, - seconds: 3, - random_factor: None, - }, - retry_if: None, - }; - assert_eq!( - vec![ - Some(4 * SECOND), - Some(4 * SECOND), - Some(27 * SECOND), - Some(81 * SECOND), - None, - ], - (0..5) - .map(|previous_attempts| retry.interval(previous_attempts, false)) - .collect::>() - ); - - assert_eq!(Some(81 * SECOND), retry.max_interval()); - } -} diff --git a/backend/windmill-api/src/folder_history.rs b/backend/windmill-api/src/folder_history.rs index 581b568eb1..d5cf1ab3c8 100644 --- a/backend/windmill-api/src/folder_history.rs +++ b/backend/windmill-api/src/folder_history.rs @@ -1,68 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::db::ApiAuthed; -use axum::{ - extract::{Extension, Path, Query}, - routing::get, - Router, -}; -use windmill_common::{ - db::UserDB, - error::JsonResult, - utils::{paginate, Pagination}, -}; - -use serde::Serialize; -use sqlx::FromRow; - -pub fn workspaced_service() -> Router { - Router::new().route("/get/:name", get(get_folder_permission_history)) -} - -#[derive(Serialize, FromRow)] -pub struct FolderPermissionChange { - pub id: i64, - pub changed_by: String, - pub changed_at: chrono::DateTime, - pub change_type: String, - pub affected: Option, -} - -async fn get_folder_permission_history( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, - Query(pagination): Query, -) -> JsonResult> { - // Check if user is owner of the folder (before starting transaction for performance) - crate::folders::require_is_owner(&authed, &name)?; - - let mut tx = user_db.begin(&authed).await?; - - let (per_page, offset) = paginate(pagination); - - let history = sqlx::query_as!( - FolderPermissionChange, - "SELECT id, changed_by, changed_at, change_type, affected - FROM folder_permission_history - WHERE workspace_id = $1 AND folder_name = $2 - ORDER BY id DESC - LIMIT $3 OFFSET $4", - w_id, - name, - per_page as i64, - offset as i64 - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(axum::Json(history)) -} +pub use windmill_api_groups::folder_history::*; diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index a73eb99b7e..ca8504a824 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -1,849 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use std::sync::Arc; - -use crate::{ - auth::AuthCache, - db::{ApiAuthed, DB}, - users::Tokened, - utils::check_scopes, - webhook_util::{WebhookMessage, WebhookShared}, -}; -use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, -}; -use lazy_static::lazy_static; -use regex::Regex; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::{ - db::UserDB, - error::{self, to_anyhow, JsonResult, Result}, - users::username_to_permissioned_as, - utils::{not_found_if_none, paginate, Pagination}, -}; - -use serde::{Deserialize, Serialize}; -use sqlx::{FromRow, Postgres, Transaction}; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_folders)) - .route("/listnames", get(list_foldernames)) - .route("/create", post(create_folder)) - .route("/get/:name", get(get_folder)) - .route("/exists/:name", get(exists_folder)) - .route("/update/:name", post(update_folder)) - .route("/getusage/:name", get(get_folder_usage)) - .route("/delete/:name", delete(delete_folder)) - .route("/addowner/:name", post(add_owner)) - .route("/removeowner/:name", post(remove_owner)) - .route("/is_owner/*path", get(is_owner_api)) -} - -#[derive(FromRow, Serialize, Deserialize, Clone)] -pub struct Folder { - pub workspace_id: String, - pub name: String, - pub display_name: String, - pub owners: Vec, - pub extra_perms: serde_json::Value, - pub summary: Option, - pub created_by: Option, - pub edited_at: Option>, -} - -#[derive(Deserialize)] -pub struct NewFolder { - pub name: String, - pub summary: Option, - pub display_name: Option, - pub owners: Option>, - pub extra_perms: Option, -} - -#[derive(Deserialize)] -pub struct UpdateFolder { - pub summary: Option, - pub display_name: Option, - pub owners: Option>, - pub extra_perms: Option, -} - -#[derive(Deserialize)] -pub struct Owner { - pub owner: String, - pub write: Option, -} - -async fn list_folders( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - let mut tx = user_db.begin(&authed).await?; - - let rows = sqlx::query_as!( - Folder, - "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", - w_id, - per_page as i64, - offset as i64 - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(rows)) -} -async fn list_foldernames( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - let mut tx = user_db.begin(&authed).await?; - - let rows = sqlx::query_scalar!( - "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", - w_id, - per_page as i64, - offset as i64 - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(rows)) -} - -fn validate_owner(owner: &str) -> Result<()> { - if !owner - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-') - { - return Err(error::Error::BadRequest( - "Invalid owner: must contain only alphanumeric characters, underscores, hyphens, or slashes".to_string(), - )); - } - Ok(()) -} - -async fn check_name_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - name: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", - name, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(windmill_common::error::Error::BadRequest(format!( - "Folder {} already exists", - name - ))); - } - return Ok(()); -} - -lazy_static! { - static ref VALID_FOLDER_NAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap(); -} - -async fn create_folder( - authed: ApiAuthed, - Tokened { token }: Tokened, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Extension(cache): Extension>, - Path(w_id): Path, - Json(ng): Json, -) -> Result { - let mut tx = user_db.clone().begin(&authed).await?; - - if !VALID_FOLDER_NAME.is_match(&ng.name) { - return Err(windmill_common::error::Error::BadRequest(format!( - "Folder name can only contain alphanumeric characters, underscores" - ))); - } - check_name_conflict(&mut tx, &w_id, &ng.name).await?; - cache.invalidate(&w_id, token).await; - let owner = username_to_permissioned_as(&authed.username); - let owners = ng.owners.unwrap_or_else(|| vec![owner.clone()]); - let owners = if owners.contains(&owner) { - owners.clone() - } else { - owners - .iter() - .cloned() - .chain(std::iter::once(owner)) - .collect() - }; - - let mut extra_perms = ng - .extra_perms - .unwrap_or_else(|| serde_json::Value::Object(serde_json::Map::new())); - - if extra_perms.is_object() { - let extra_mut = extra_perms.as_object_mut().unwrap(); - for o in &owners { - extra_mut.insert(o.clone(), serde_json::json!(true)); - } - } else { - return Err(error::Error::BadRequest( - "extra_perms must be an object".to_string(), - )); - } - - if let Err(e) = - sqlx::query_as!( - Folder, - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, now())", - w_id, - ng.name, - ng.display_name.unwrap_or(ng.name.clone()), - &owners, - extra_perms, - ng.summary, - authed.username - ) - .execute(&mut *tx) - .await { - drop(tx); - let mut tx = user_db.begin(&authed).await?; - - let exists_for_user = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2 AND $3 = ANY(owners))", - ng.name, - w_id, - authed.username - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", - ng.name, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - if !exists_for_user && exists { - return Err(windmill_common::error::Error::BadRequest(format!( - "Folder '{}' already exists in workspace '{}' but you do not have permission to read to it", ng.name, w_id - ))); - } else if exists { - return Err(windmill_common::error::Error::BadRequest(format!( - "Folder '{}' already exists in workspace '{}'", ng.name, w_id - ))); - } else { - return Err(windmill_common::error::Error::InternalErr(format!( - "Failed to create folder: {}", e - ))); - } - } - - audit_log( - &mut *tx, - &authed, - "folder.create", - ActionKind::Create, - &w_id, - Some(&ng.name.to_string()), - None, - ) - .await?; - - log_folder_permission_change(&mut *tx, &w_id, &ng.name, &authed.username, "create", None) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Folder { path: format!("f/{}", ng.name) }, - Some(format!("Folder '{}' created", ng.name)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateFolder { workspace: w_id, name: ng.name.clone() }, - ); - - Ok(format!("Created folder {}", ng.name)) -} - -pub async fn is_owner_api( - authed: ApiAuthed, - Path((_w_id, name)): Path<(String, String)>, -) -> JsonResult { - Ok(Json(is_owner(&authed, &name))) -} - -pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool { - if *is_admin { - true - } else { - folders.into_iter().any(|x| x.0 == name && x.2) - } -} - -pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> { - if is_owner(authed, name) { - Ok(()) - } else { - Err(windmill_common::error::Error::NotAuthorized(format!( - "You are not owner of the folder {}", - name - ))) - } -} - -async fn update_folder( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(mut ng): Json, -) -> Result { - use sql_builder::prelude::*; - - let mut sqlb = SqlBuilder::update_table("folder"); - sqlb.and_where_eq("name", "?".bind(&name)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - - if let Some(display_name) = ng.display_name { - sqlb.set("display_name", "?".bind(&display_name)); - } - - if let Some(summary) = ng.summary { - sqlb.set("summary", "?".bind(&summary)); - } - - sqlb.set("edited_at", "now()"); - - // Track whether permission-related fields are being updated - let owners_changed = ng.owners.is_some(); - let extra_perms_changed = ng.extra_perms.is_some(); - - if !authed.is_admin { - let prefixed_username = format!("u/{}", authed.username); - if ng.owners.as_ref().is_some_and(|x| { - !x.contains(&prefixed_username) - && !authed.groups.iter().any(|g| x.contains(&format!("g/{g}"))) - }) { - ng.owners.as_mut().unwrap().push(prefixed_username.clone()); - if ng.extra_perms.is_none() { - ng.extra_perms = Some(serde_json::Value::Object(serde_json::Map::new())); - } - ng.extra_perms - .as_mut() - .unwrap() - .as_object_mut() - .unwrap() - .insert(prefixed_username, serde_json::json!(true)); - } - } - if let Some(owners) = ng.owners { - sqlb.set( - "owners", - "?".bind(&format!( - "{{{}}}", - owners - .iter() - .map(|x| format!("\"{x}\"")) - .collect::>() - .join(","), - )), - ); - } - if let Some(extra_perms) = ng.extra_perms { - if !extra_perms.is_object() { - return Err(windmill_common::error::Error::BadRequest(format!( - "extra_perms must be an object, received {}", - extra_perms.to_string() - ))); - } - sqlb.set( - "extra_perms", - "?".bind(&serde_json::to_string(&extra_perms).map_err(to_anyhow)?), - ); - } - - sqlb.returning("*"); - - let mut tx = user_db.begin(&authed).await?; - - let sql = sqlb - .sql() - .map_err(|e| error::Error::internal_err(e.to_string()))?; - let nfolder = sqlx::query_as::<_, Folder>(&sql) - .fetch_optional(&mut *tx) - .await?; - - let nfolder = nfolder.ok_or_else(|| { - windmill_common::error::Error::NotAuthorized(format!( - "You are not an owner of {} and hence cannot modify it", - name - )) - })?; - - if let Some(extra_perms) = nfolder.extra_perms.as_object() { - for o in nfolder.owners { - if !extra_perms - .get(&o) - .and_then(|x| x.as_bool()) - .unwrap_or(false) - { - return Err(windmill_common::error::Error::BadRequest(format!( - "Owner {} would not have permission to write to folder and that is an invalid state", - o - ))); - } - } - } - - audit_log( - &mut *tx, - &authed, - "folder.update", - ActionKind::Update, - &w_id, - Some(&name.to_string()), - None, - ) - .await?; - - // Log permission changes if owners or extra_perms were updated - if owners_changed { - log_folder_permission_change( - &mut *tx, - &w_id, - &name, - &authed.username, - "update_owners", - None, - ) - .await?; - } - if extra_perms_changed { - log_folder_permission_change(&mut *tx, &w_id, &name, &authed.username, "update_acl", None) - .await?; - } - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Folder { path: format!("f/{}", name) }, - Some(format!("Folder '{}' updated", name)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone().clone(), - WebhookMessage::UpdateFolder { workspace: w_id, name: name.to_owned() }, - ); - - Ok(format!("Updated folder {}", name)) -} - -pub async fn get_folderopt<'c>( - db: &mut Transaction<'c, Postgres>, - w_id: &str, - name: &str, -) -> Result> { - let folderopt = sqlx::query_as!( - Folder, - "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at FROM folder WHERE name = $1 AND workspace_id = $2", - name, - w_id - ) - .fetch_optional(&mut **db) - .await?; - Ok(folderopt) -} - -async fn get_folder( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - check_scopes(&authed, || format!("folders:read:f/{}", name))?; - let mut tx = user_db.begin(&authed).await?; - - let folder = not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; - - tx.commit().await?; - Ok(Json(folder)) -} - -async fn exists_folder( - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2)", - name, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -#[derive(Serialize)] -struct FolderUsage { - pub scripts: i64, - pub schedules: i64, - pub flows: i64, - pub apps: i64, - pub resources: i64, - pub variables: i64, -} -async fn get_folder_usage( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - check_scopes(&authed, || format!("folders:read:f/{}", name))?; - let mut tx = user_db.begin(&authed).await?; - - let scripts = sqlx::query_scalar!( - "SELECT count(path) FROM script WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - let schedules = sqlx::query_scalar!( - "SELECT count(path) FROM schedule WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - let flows = sqlx::query_scalar!( - "SELECT count(path) FROM flow WHERE path LIKE 'f/' || $1 || '%' AND archived IS false AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - let apps = sqlx::query_scalar!( - "SELECT count(path) FROM app WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - let raw_apps = sqlx::query_scalar!( - "SELECT count(path) FROM raw_app WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - let resources = sqlx::query_scalar!( - "SELECT count(path) FROM resource WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - let variables = sqlx::query_scalar!( - "SELECT count(path) FROM variable WHERE path LIKE 'f/' || $1 || '%' AND workspace_id = $2", - name, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(0); - - tx.commit().await?; - Ok(Json(FolderUsage { - scripts, - flows, - schedules, - apps: apps + raw_apps, - resources, - variables, - })) -} - -async fn delete_folder( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; - - let del = sqlx::query_scalar!( - "DELETE FROM folder WHERE name = $1 AND workspace_id = $2 RETURNING 1", - name, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - if del.is_none() { - return Err(windmill_common::error::Error::NotAuthorized(format!( - "Not authorized to delete folder {}", - name - ))); - } - - audit_log( - &mut *tx, - &authed, - "folder.delete", - ActionKind::Delete, - &w_id, - Some(&name.to_string()), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Folder { path: format!("f/{}", name) }, - Some(format!("Folder '{}' deleted", name)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteFolder { workspace: w_id, name: name.clone() }, - ); - - Ok(format!("delete folder at name {}", name)) -} - -async fn add_owner( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(Owner { owner, .. }): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; - require_is_owner(&authed, &name)?; - - sqlx::query!( - "UPDATE folder SET owners = array_append(owners::text[], $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name", - owner, - &name, - &w_id, - ) - .fetch_optional(&mut *tx) - .await?; - - validate_owner(&owner)?; - sqlx::query(&format!( - "UPDATE folder SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \ - true) WHERE name = $2 AND workspace_id = $3 RETURNING extra_perms" - )) - .bind(true) - .bind(&name) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "folder.add_owner", - ActionKind::Update, - &w_id, - Some(&name.to_string()), - Some([("owner", owner.as_str())].into()), - ) - .await?; - - log_folder_permission_change( - &mut *tx, - &w_id, - &name, - &authed.username, - "grant_admin", - Some(&owner), - ) - .await?; - - tx.commit().await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateFolder { workspace: w_id, name: name.clone() }, - ); - - Ok(format!("Added {} to folder {}", owner, name)) -} - -async fn remove_owner( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(Owner { owner, write }): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; - require_is_owner(&authed, &name)?; - validate_owner(&owner)?; - - let folder = sqlx::query!( - "UPDATE folder SET owners = array_remove(owners, $1::varchar) WHERE name = $2 AND workspace_id = $3 AND $1 = ANY(owners) RETURNING name", - owner, - &name, - &w_id, - ) - .fetch_optional(&mut *tx) - .await?; - - if folder.is_none() && write.is_none() { - return Ok(format!( - "Owner {} is already not a member of folder {}", - owner, name - )); - } - - if let Some(write) = write { - let old_write = sqlx::query_scalar::<_, Option>(&format!( - "UPDATE folder SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \ - true) FROM (SELECT (extra_perms->>'{owner}')::boolean as old_val FROM folder WHERE name = $2 AND workspace_id = $3) old \ - WHERE name = $2 AND workspace_id = $3 RETURNING old.old_val" - )) - .bind(write) - .bind(&name) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - if folder.is_none() && old_write.is_none_or(|ow| ow == write) { - return Ok(format!( - "Owner {} is already not a member of folder {} and write permission was already {}", - owner, name, write - )); - } - } - - audit_log( - &mut *tx, - &authed, - "folder.remove_owner", - ActionKind::Update, - &w_id, - Some(&name.to_string()), - Some([("owner", owner.as_str())].into()), - ) - .await?; - - let change_type = match write { - Some(true) => "grant_writer_only", - Some(false) => "grant_viewer_only", - None => "revoke_all", - }; - log_folder_permission_change( - &mut *tx, - &w_id, - &name, - &authed.username, - change_type, - Some(&owner), - ) - .await?; - - tx.commit().await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateFolder { workspace: w_id, name: name.clone() }, - ); - - Ok(format!("Removed {} to folder {}", owner, name)) -} - -pub async fn log_folder_permission_change<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - folder_name: &str, - changed_by: &str, - change_type: &str, - affected: Option<&str>, -) -> Result<()> { - sqlx::query!( - "INSERT INTO folder_permission_history - (workspace_id, folder_name, changed_by, change_type, affected) - VALUES ($1, $2, $3, $4, $5)", - workspace_id, - folder_name, - changed_by, - change_type, - affected - ) - .execute(db) - .await?; - Ok(()) -} +pub use windmill_api_groups::folders::*; diff --git a/backend/windmill-api/src/granular_acls.rs b/backend/windmill-api/src/granular_acls.rs index 8a0354906a..98f373b8fa 100644 --- a/backend/windmill-api/src/granular_acls.rs +++ b/backend/windmill-api/src/granular_acls.rs @@ -1,395 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::{db::DB, users::require_owner_of_path}; -use axum::{ - extract::{Extension, Path}, - routing::{get, post}, - Json, Router, -}; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; - -use crate::db::ApiAuthed; - -use serde::{Deserialize, Serialize}; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::{not_found_if_none, StripPath}, -}; - -const KINDS: [&str; 18] = [ - "script", - "group_", - "resource", - "schedule", - "variable", - "flow", - "folder", - "app", - "raw_app", - "http_trigger", - "websocket_trigger", - "kafka_trigger", - "nats_trigger", - "postgres_trigger", - "mqtt_trigger", - "gcp_trigger", - "sqs_trigger", - "email_trigger", -]; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/get/*path", get(get_granular_acls)) - .route("/add/*path", post(add_granular_acl)) - .route("/remove/*path", post(remove_granular_acl)) -} - -#[derive(Serialize, Deserialize)] -pub struct GranularAcl { - pub owner: String, - pub write: Option, -} - -async fn add_granular_acl( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(GranularAcl { owner, write }): Json, -) -> Result { - let path = path.to_path(); - - let (kind, path) = path - .split_once('/') - .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?; - - if !KINDS.contains(&kind) { - return Err(Error::BadRequest("Invalid kind".to_string())); - } - - let mut tx = user_db.begin(&authed).await?; - - let identifier = if kind == "group_" || kind == "folder" { - "name" - } else { - "path" - }; - - if !authed.is_admin { - if kind == "folder" { - crate::folders::require_is_owner(&authed, path)?; - } else if kind == "group_" { - crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) - .await?; - } else { - require_owner_of_path(&authed, path)?; - } - } - - if kind == "folder" { - if let Some(obj) = sqlx::query_scalar!( - "SELECT owners FROM folder WHERE name = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - if obj.contains(&owner) { - if write != Some(true) { - return Err(Error::BadRequest( - "Cannot remove write permission for folder owner".to_string(), - )); - } - } - } - } - - let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( - "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \ - true) WHERE {identifier} = $3 AND workspace_id = $4 RETURNING extra_perms" - )) - .bind(vec![owner.clone()]) - .bind(write.unwrap_or(false)) - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - let _ = not_found_if_none(obj_o, &kind, &path)?; - - if kind == "folder" { - let change_type = if write.unwrap_or(false) { - "grant_read" - } else { - "grant_write" - }; - crate::folders::log_folder_permission_change( - &mut *tx, - &w_id, - path, - &authed.username, - change_type, - Some(&owner), - ) - .await?; - } else if kind == "group_" { - let change_type = if write.unwrap_or(false) { - "grant_admin" - } else { - "grant_member_only" - }; - crate::groups::log_group_permission_change( - &mut *tx, - &w_id, - path, - &authed.username, - change_type, - Some(&owner), - ) - .await?; - } - - tx.commit().await?; - - match kind { - "folder" => { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Folder { path: format!("f/{}", path) }, - Some(format!("Folder '{}' changed permissions", path)), - true, - None, - ) - .await? - } - // "app" => { - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 }, - // Some(format!("App '{}' changed permissions", path)), - // // true, - // ) - // .await? - // } - // "script" => { - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::Script { - // path: path.to_string(), - // parent_path: None, - // hash: ScriptHash(0), - // }, - // Some(format!("Script '{}' changed permissions", path)), - // // true, - // ) - // .await? - // } - // "flow" => { - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::Flow { path: path.to_string(), parent_path: None }, - // Some(format!("Flow '{}' changed permissions", path)), - // // true, - // ) - // .await? - // } - _ => (), - } - - Ok("Successfully modified granular acl".to_string()) -} - -async fn remove_granular_acl( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(GranularAcl { owner, .. }): Json, -) -> Result { - let path = path.to_path(); - - let (kind, path) = path - .split_once('/') - .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?; - - if !KINDS.contains(&kind) { - return Err(Error::BadRequest("Invalid kind".to_string())); - } - - if !authed.is_admin { - if kind == "folder" { - crate::folders::require_is_owner(&authed, path)?; - } else if kind == "group_" { - crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) - .await?; - } else { - require_owner_of_path(&authed, path)?; - } - } - - let mut tx = user_db.begin(&authed).await?; - - let identifier = if kind == "group_" || kind == "folder" { - "name" - } else { - "path" - }; - - if identifier == "path" { - require_owner_of_path(&authed, path)?; - } - - let obj_o = sqlx::query_scalar::<_, bool>(&format!( - "WITH old AS ( - SELECT extra_perms->$1 as old_write FROM {kind} - WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1 - ) - UPDATE {kind} SET extra_perms = extra_perms - $1 - WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1 - RETURNING (SELECT old_write FROM old)::bool" - )) - .bind(&owner) - .bind(path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - // Only log if something was actually removed (obj_o is Some) - if let Some(write) = obj_o { - // Log permission changes for folders and groups - if kind == "folder" { - let change_type = if write { "revoke_write" } else { "revoke_read" }; - crate::folders::log_folder_permission_change( - &mut *tx, - &w_id, - path, - &authed.username, - change_type, - Some(&owner), - ) - .await?; - } else if kind == "group_" { - crate::groups::log_group_permission_change( - &mut *tx, - &w_id, - path, - &authed.username, - "revoke_admin", - Some(&owner), - ) - .await?; - } - - tx.commit().await?; - - match kind { - "folder" => { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Folder { path: format!("f/{}", path) }, - Some(format!("Folder '{}' changed permissions", path)), - true, - None, - ) - .await? - } - // "app" => { - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 }, - // Some(format!("App '{}' changed permissions", path)), - // // true, - // ) - // .await? - // } - // "script" => { - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::Script { - // path: path.to_string(), - // parent_path: None, - // hash: ScriptHash(0), - // }, - // Some(format!("Script '{}' changed permissions", path)), - // // true, - // ) - // .await? - // } - // "flow" => { - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::Flow { path: path.to_string(), parent_path: None }, - // Some(format!("Flow '{}' changed permissions", path)), - // // true, - // ) - // .await? - // } - _ => (), - } - } - - Ok("Successfully removed granular acl".to_string()) -} - -async fn get_granular_acls( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let (kind, path) = path - .split_once('/') - .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?; - - if !KINDS.contains(&kind) { - return Err(Error::BadRequest("Invalid kind".to_string())); - } - - let mut tx = user_db.begin(&authed).await?; - - let identifier = if kind == "group_" { "name" } else { "path" }; - let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( - "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" - )) - .bind(path) - .bind(w_id) - .fetch_optional(&mut *tx) - .await?; - - let obj = not_found_if_none(obj_o, &kind, &path)?; - tx.commit().await?; - - Ok(Json(obj)) -} +pub use windmill_api_groups::granular_acls::*; diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index d67b97802b..84f735dc77 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -1,1064 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::db::ApiAuthed; -use crate::{db::DB, utils::require_super_admin}; - -use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, -}; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::worker::CLOUD_HOSTED; -use windmill_common::{ - auth::get_groups_for_user, - error::{Error, JsonResult, Result}, - utils::{not_found_if_none, paginate, Pagination}, -}; -use windmill_common::{db::UserDB, users::username_to_permissioned_as}; - -use serde::{Deserialize, Serialize}; -use sqlx::{query_scalar, FromRow, Postgres, Transaction}; -use windmill_git_sync::handle_deployment_metadata; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_groups)) - .route("/listnames", get(list_group_names)) - .route("/create", post(create_group)) - .route("/get/:name", get(get_group)) - .route("/update/:name", post(update_group)) - .route("/delete/:name", delete(delete_group)) - .route("/adduser/:name", post(add_user)) - .route("/removeuser/:name", post(remove_user)) - .route("/is_owner", get(is_owner)) -} - -pub fn global_service() -> Router { - Router::new() - .route("/list", get(list_igroups)) - .route("/list_with_workspaces", get(list_igroups_with_workspaces)) - .route("/get/:name", get(get_igroup)) - .route("/create", post(create_igroup)) - .route("/update/:name", post(update_igroup)) - .route("/delete/:name", delete(delete_igroup)) - .route("/adduser/:name", post(add_user_igroup)) - .route("/removeuser/:name", post(remove_user_igroup)) - .route("/export", get(export_igroups)) - .route("/overwrite", post(overwrite_igroups)) -} - -/// Normalize group names: replace spaces with underscores and convert to lowercase -/// Used when manually creating groups and SCIM-managed groups -pub fn convert_name(name: &str) -> String { - name.replace(" ", "_").to_lowercase() -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct Group { - pub workspace_id: String, - pub name: String, - pub summary: Option, - pub extra_perms: serde_json::Value, -} - -#[derive(Deserialize)] -pub struct NewGroup { - pub name: String, - pub summary: Option, -} - -#[derive(Serialize)] -pub struct GroupInfo { - pub workspace_id: String, - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - pub members: Vec, - pub extra_perms: serde_json::Value, -} - -#[derive(Deserialize)] -pub struct EditGroup { - pub summary: Option, -} - -#[derive(Deserialize)] -pub struct Username { - pub username: String, -} - -#[derive(Deserialize)] -pub struct Email { - pub email: String, -} - -async fn list_groups( - Extension(db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - - let rows = sqlx::query_as!( - Group, - "SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", - w_id, - per_page as i64, - offset as i64 - ) - .fetch_all(&db) - .await?; - - Ok(Json(rows)) -} - -#[derive(Deserialize)] -struct QueryListGroup { - pub only_member_of: Option, -} -async fn list_group_names( - ApiAuthed { username, email, .. }: ApiAuthed, - Extension(db): Extension, - Query(QueryListGroup { only_member_of }): Query, - Path(w_id): Path, -) -> JsonResult> { - let rows = if !only_member_of.unwrap_or(false) { - sqlx::query_scalar!( - "SELECT name FROM group_ WHERE workspace_id = $1 UNION SELECT name FROM instance_group ORDER BY name asc", - w_id - ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|x| x) - .collect() - } else { - get_groups_for_user(&w_id, &username, &email, &db).await? - }; - - Ok(Json(rows)) -} - -async fn check_name_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - name: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM group_ WHERE name = $1 AND workspace_id = $2)", - name, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(windmill_common::error::Error::BadRequest(format!( - "Group {} already exists", - name - ))); - } - return Ok(()); -} - -pub async fn is_owner( - ApiAuthed { username, is_admin, groups, .. }: ApiAuthed, - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - if is_admin { - Ok(Json(true)) - } else { - Ok(Json( - require_is_owner(&name, &username, &groups, &w_id, &db) - .await - .is_ok(), - )) - } -} - -pub async fn require_is_owner( - group_name: &str, - username: &str, - groups: &Vec, - w_id: &str, - db: &DB, -) -> Result<()> { - let is_owner = query_scalar!( - "SELECT EXISTS(SELECT 1 FROM group_ WHERE (group_.extra_perms ->> CONCAT('u/', $1::text))::boolean AND name = $2 AND workspace_id = $4) OR exists( - SELECT 1 FROM group_ g, jsonb_each_text(g.extra_perms) f - WHERE $2 = g.name AND $4 = g.workspace_id AND SPLIT_PART(key, '/', 1) = 'g' AND key = ANY($3::text[]) - AND value::boolean)", - username, - group_name, - groups, - w_id, - ).fetch_one(db) - .await? - .unwrap_or(false); - if !is_owner { - Err(Error::BadRequest(format!( - "{} is not an owner of {} and hence is not authorized to perform this operation", - username, group_name - ))) - } else { - Ok(()) - } -} - -async fn _check_nb_of_groups(db: &DB) -> Result<()> { - let nb_groups = sqlx::query_scalar!("SELECT COUNT(*) FROM group_ WHERE name != 'all' AND name != 'error_handler' AND name != 'slack'",) - .fetch_one(db) - .await?; - if nb_groups.unwrap_or(0) >= 3 { - return Err(Error::BadRequest( - "You have reached the maximum number of groups (3 outside of native groups 'all', 'slack' and 'error_handler') without an enterprise license" - .to_string(), - )); - } - return Ok(()); -} - -async fn create_group( - authed: ApiAuthed, - Extension(_db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(ng): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - check_name_conflict(&mut tx, &w_id, &ng.name).await?; - - #[cfg(not(feature = "enterprise"))] - _check_nb_of_groups(&_db).await?; - - sqlx::query!( - "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4)", - w_id, - ng.name, - ng.summary, - serde_json::json!({username_to_permissioned_as(&authed.username): true}) - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - &w_id, - &authed.username, - ng.name, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "group.create", - ActionKind::Create, - &w_id, - Some(&ng.name.to_string()), - None, - ) - .await?; - - log_group_permission_change( - &mut *tx, - &w_id, - &ng.name, - &authed.username, - "create", - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &_db, - &w_id, - windmill_git_sync::DeployedObject::Group { name: ng.name.clone() }, - Some(format!("Created group '{}'", &ng.name)), - true, - None, - ) - .await?; - - Ok(format!("Created group {}", ng.name)) -} - -async fn create_igroup( - authed: ApiAuthed, - Extension(db): Extension, - Json(ng): Json, -) -> Result { - use uuid::Uuid; - - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - - let normalized_name = convert_name(&ng.name); - - let id = Uuid::new_v4().to_string(); - sqlx::query!( - "INSERT INTO instance_group (name, summary, id) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - normalized_name, - ng.summary, - id, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "igroup.create", - ActionKind::Create, - "global", - Some(&normalized_name), - None, - ) - .await?; - - tx.commit().await?; - Ok(format!("Created group {}", normalized_name)) -} - -#[derive(Deserialize)] -struct IGroupUpdate { - new_summary: String, -} - -async fn update_igroup( - authed: ApiAuthed, - Extension(db): Extension, - Path(name): Path, - Json(igroup_update): Json, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - - let exists_opt = sqlx::query("SELECT 1 FROM instance_group WHERE name = $1") - .bind(name.clone()) - .fetch_optional(&mut *tx) - .await?; - not_found_if_none(exists_opt, "instance_group", name.clone())?; - - sqlx::query("UPDATE instance_group SET summary = $1 WHERE name = $2") - .bind(igroup_update.new_summary) - .bind(&name) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "igroup.updated", - ActionKind::Delete, - "global", - Some(&name.to_string()), - None, - ) - .await?; - - tx.commit().await?; - Ok(format!("Deleted group {}", name)) -} - -async fn delete_igroup( - authed: ApiAuthed, - Extension(db): Extension, - Path(name): Path, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - sqlx::query!("DELETE FROM instance_group WHERE name = $1", name) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM email_to_igroup WHERE igroup = $1", name) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "igroup.delete", - ActionKind::Delete, - "global", - Some(&name.to_string()), - None, - ) - .await?; - - tx.commit().await?; - Ok(format!("Deleted group {}", name)) -} - -pub async fn get_group_opt<'c>( - db: &mut Transaction<'c, Postgres>, - w_id: &str, - name: &str, -) -> Result> { - let group_opt = sqlx::query_as!( - Group, - "SELECT * FROM group_ WHERE name = $1 AND workspace_id = $2", - name, - w_id - ) - .fetch_optional(&mut **db) - .await?; - Ok(group_opt) -} - -async fn get_group( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - if *CLOUD_HOSTED && w_id == "demo" && name == "all" && !authed.is_admin { - return Ok(Json(GroupInfo { - workspace_id: w_id, - name: name, - summary: Some("The group that contains all users".to_string()), - members: vec!["redacted_in_demo_workspace".to_string()], - extra_perms: serde_json::json!({}), - })); - } - - let mut tx = user_db.begin(&authed).await?; - - let group = not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; - - let members = sqlx::query_scalar!( - "SELECT usr.username - FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2 - WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2", - name, - w_id - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - Ok(Json(GroupInfo { - workspace_id: group.workspace_id, - name: group.name, - summary: group.summary, - members, - extra_perms: group.extra_perms, - })) -} - -async fn delete_group( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - if name == "all" { - return Err(Error::BadRequest( - "The group 'all' is a special group that contains all users and cannot be deleted" - .to_string(), - )); - } - - if !authed.is_admin { - require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; - } - not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; - - sqlx::query!( - "DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2", - name, - w_id - ) - .execute(&mut *tx) - .await?; - sqlx::query!( - "DELETE FROM group_ WHERE name = $1 AND workspace_id = $2", - name, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "group.delete", - ActionKind::Delete, - &w_id, - Some(&name.to_string()), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Group { name: name.clone() }, - Some(format!("Deleted group '{}'", &name)), - true, - None, - ) - .await?; - - Ok(format!("delete group at name {}", name)) -} - -async fn update_group( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(eg): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - if !authed.is_admin { - require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; - } - not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; - - sqlx::query!( - "UPDATE group_ SET summary = $1 WHERE name = $2 AND workspace_id = $3", - eg.summary, - &name, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "group.edit", - ActionKind::Update, - &w_id, - Some(&name.to_string()), - None, - ) - .await?; - - log_group_permission_change( - &mut *tx, - &w_id, - &name, - &authed.username, - "update_summary", - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Group { name: name.clone() }, - Some(format!("Updated group '{}'", &name)), - true, - None, - ) - .await?; - - Ok(format!("Edited group {}", name)) -} - -async fn add_user( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(Username { username: user_username }): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - if !authed.is_admin { - require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; - } - - not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; - - let result = sqlx::query!( - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - &w_id, - user_username, - name, - ) - .execute(&mut *tx) - .await?; - - if result.rows_affected() == 0 { - return Ok(format!("{} is already a member of group {}", user_username, name)); - } - - audit_log( - &mut *tx, - &authed, - "group.adduser", - ActionKind::Update, - &w_id, - Some(&name.to_string()), - Some([("user", user_username.as_str())].into()), - ) - .await?; - - log_group_permission_change( - &mut *tx, - &w_id, - &name, - &authed.username, - "add_member", - Some(&user_username), - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Group { name: name.clone() }, - Some(format!("Added user to group '{}'", &name)), - true, - None, - ) - .await?; - - Ok(format!("Added {} to group {}", user_username, name)) -} - -async fn add_user_igroup( - authed: ApiAuthed, - Extension(db): Extension, - Path(name): Path, - Json(Email { email }): Json, -) -> Result { - require_super_admin(&db, &authed.email).await?; - - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - - let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name) - .fetch_optional(&mut *tx) - .await?; - - not_found_if_none(group_opt, "IGroup", &name)?; - - sqlx::query!( - "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2) ON CONFLICT DO NOTHING", - email, - name, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "igroup.adduser", - ActionKind::Update, - "global", - Some(&name.to_string()), - Some([("email", email.as_str())].into()), - ) - .await?; - - // Sync user to workspaces configured with this instance group - #[cfg(all(feature = "private", feature = "enterprise"))] - { - use crate::workspaces_ee::auto_add_user; - let workspaces = sqlx::query!( - r#" - SELECT workspace_id, auto_invite->'instance_groups_roles' as instance_groups_roles - FROM workspace_settings - WHERE auto_invite->'instance_groups' ? $1 - "#, - &name - ).fetch_all(&mut *tx).await?; - for ws in workspaces { - let role = ws.instance_groups_roles.and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))).unwrap_or_else(|| "developer".to_string()); - let (is_admin, is_operator) = match role.as_str() { "admin" => (true, false), "operator" => (false, true), _ => (false, false) }; - auto_add_user(&email, &ws.workspace_id, &is_operator, &mut tx, &authed, Some(serde_json::json!({"source": "instance_group", "group": &name}))).await?; - if is_admin { sqlx::query!("UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", &ws.workspace_id, &email).execute(&mut *tx).await?; } - } - } - - tx.commit().await?; - Ok(format!("Added {} to igroup {}", email, name)) -} - -#[derive(Serialize)] -struct IGroup { - name: String, - summary: Option, - emails: Option>, -} - -#[derive(Serialize)] -struct IGroupWithWorkspaces { - name: String, - summary: Option, - emails: Option>, - workspaces: Vec, -} - -#[derive(Serialize, Clone)] -struct WorkspaceInfo { - workspace_id: String, - workspace_name: String, - role: String, -} -async fn list_igroups(Extension(db): Extension) -> JsonResult> { - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - - let groups = sqlx::query_as!( - IGroup, - "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name" - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - return Ok(Json(groups)); -} - -async fn list_igroups_with_workspaces(Extension(db): Extension) -> JsonResult> { - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - - // Get all instance groups with their emails first - let groups = sqlx::query_as!( - IGroup, - "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name, summary" - ) - .fetch_all(&mut *tx) - .await?; - - // Get all workspace mappings for instance groups in a single query - let workspace_mappings = sqlx::query!( - r#" - SELECT - ig.name as group_name, - ws.workspace_id, - w.name as workspace_name, - ws.auto_invite->'instance_groups_roles'->ig.name as role - FROM instance_group ig - INNER JOIN workspace_settings ws ON ws.auto_invite->'instance_groups' IS NOT NULL - AND ws.auto_invite->'instance_groups' ? ig.name - INNER JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false - ORDER BY ig.name, ws.workspace_id - "# - ) - .fetch_all(&mut *tx) - .await?; - - // Create a map of group_name -> Vec - let mut workspaces_by_group: std::collections::HashMap> = std::collections::HashMap::new(); - for mapping in workspace_mappings { - let role = mapping.role - .and_then(|r| r.as_str().map(|s| s.to_string())) - .unwrap_or_else(|| "developer".to_string()); - - let workspace_info = WorkspaceInfo { - workspace_id: mapping.workspace_id.clone(), - workspace_name: mapping.workspace_name, - role, - }; - - workspaces_by_group - .entry(mapping.group_name) - .or_insert_with(Vec::new) - .push(workspace_info); - } - - let mut result = Vec::new(); - for group in groups { - let workspaces = workspaces_by_group.get(&group.name).cloned().unwrap_or_default(); - - result.push(IGroupWithWorkspaces { - name: group.name, - summary: group.summary, - emails: group.emails, - workspaces, - }); - } - - tx.commit().await?; - return Ok(Json(result)); -} - -async fn get_igroup(Path(name): Path, Extension(db): Extension) -> JsonResult { - let group = sqlx::query_as!( - IGroup, - "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup WHERE name = $1 GROUP BY name", - name - ) - .fetch_optional(&db) - .await?; - let group = not_found_if_none(group, "IGroup", &name)?; - return Ok(Json(group)); -} - -async fn remove_user_igroup( - authed: ApiAuthed, - Extension(db): Extension, - Path(name): Path, - Json(Email { email }): Json, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - - let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name,) - .fetch_optional(&mut *tx) - .await?; - - not_found_if_none(group_opt, "IGroup", &name)?; - - sqlx::query!( - "DELETE FROM email_to_igroup WHERE email = $1 AND igroup = $2", - email, - name, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "igroup.removeuser", - ActionKind::Update, - "global", - Some(&name.to_string()), - Some([("email", email.as_str())].into()), - ) - .await?; - - // Remove user from workspaces where they were added via this instance group - #[cfg(all(feature = "private", feature = "enterprise"))] - { - use crate::workspaces_ee::remove_users_from_instance_group_workspaces; - remove_users_from_instance_group_workspaces(&email, &name, &mut tx).await?; - } - - tx.commit().await?; - Ok(format!("Removed {} from igroup {}", email, name)) -} - -async fn remove_user( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(Username { username: user_username }): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - if !authed.is_admin { - require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?; - } - - not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?; - if &name == "all" { - return Err(Error::BadRequest(format!("Cannot delete users from all"))); - } - sqlx::query!( - "DELETE FROM usr_to_group WHERE usr = $1 AND group_ = $2 AND workspace_id = $3", - user_username, - name, - &w_id, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "group.removeuser", - ActionKind::Update, - &w_id, - Some(&name.to_string()), - Some([("user", user_username.as_str())].into()), - ) - .await?; - - log_group_permission_change( - &mut *tx, - &w_id, - &name, - &authed.username, - "remove_member", - Some(&user_username), - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Group { name: name.clone() }, - Some(format!("Removed user from group '{}'", &name)), - true, - None, - ) - .await?; - - Ok(format!("Removed {} to group {}", user_username, name)) -} - -#[cfg(feature = "enterprise")] -#[derive(Serialize, Deserialize)] -struct ExportedIGroup { - name: String, - #[serde(skip_serializing_if = "Option::is_none")] - summary: Option, - #[serde(skip_serializing_if = "Option::is_none")] - id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - scim_display_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - external_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - emails: Option>, -} - -#[cfg(feature = "enterprise")] -async fn export_igroups( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - let igroups = sqlx::query_as!( - ExportedIGroup, - "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails, id, scim_display_name, external_id FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name", - ).fetch_all(&mut *tx).await?; - - audit_log( - &mut *tx, - &authed, - "igroups.export", - ActionKind::Execute, - "global", - None, - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(igroups)) -} - -#[cfg(not(feature = "enterprise"))] -async fn export_igroups() -> JsonResult { - Err(Error::BadRequest( - "This feature is only available in the enterprise version".to_string(), - )) -} - -#[cfg(feature = "enterprise")] -async fn overwrite_igroups( - authed: ApiAuthed, - Extension(db): Extension, - Json(igroups): Json>, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - - sqlx::query!("DELETE FROM email_to_igroup") - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM instance_group") - .execute(&mut *tx) - .await?; - - for igroup in igroups.iter() { - sqlx::query!( - "INSERT INTO instance_group (name, summary, id, scim_display_name, external_id) VALUES ($1, $2, $3, $4, $5)", - igroup.name, - igroup.summary, - igroup.id, - igroup.scim_display_name, - igroup.external_id, - ) - .execute(&mut *tx) - .await?; - - if let Some(emails) = &igroup.emails { - for email in emails.iter() { - sqlx::query!( - "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)", - email, - igroup.name, - ) - .execute(&mut *tx) - .await?; - } - } - } - - audit_log( - &mut *tx, - &authed, - "igroups.import", - ActionKind::Create, - "global", - None, - None, - ) - .await?; - - tx.commit().await?; - Ok("Imported igroups".to_string()) -} - -#[cfg(not(feature = "enterprise"))] -async fn overwrite_igroups() -> JsonResult { - Err(Error::BadRequest( - "This feature is only available in the enterprise version".to_string(), - )) -} - -pub async fn log_group_permission_change<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - group_name: &str, - changed_by: &str, - change_type: &str, - member_affected: Option<&str>, -) -> Result<()> { - sqlx::query!( - "INSERT INTO group_permission_history - (workspace_id, group_name, changed_by, change_type, member_affected) - VALUES ($1, $2, $3, $4, $5)", - workspace_id, - group_name, - changed_by, - change_type, - member_affected - ) - .execute(db) - .await?; - Ok(()) -} +pub use windmill_api_groups::groups::*; diff --git a/backend/windmill-api/src/job_metrics.rs b/backend/windmill-api/src/job_metrics.rs index 70e68ba3ed..c78d8472b7 100644 --- a/backend/windmill-api/src/job_metrics.rs +++ b/backend/windmill-api/src/job_metrics.rs @@ -1,266 +1 @@ -use crate::db::DB; - -use axum::{ - extract::Path, - routing::{get, post}, - Extension, Json, Router, -}; -use serde::{Deserialize, Serialize}; -use tower_http::cors::{Any, CorsLayer}; -use uuid::Uuid; -use windmill_common::{ - error::{self, Error}, - job_metrics::{ - record_metric, register_metric_for_job, JobStatsRecord, MetricKind, MetricNumericValue, - }, -}; - -pub fn workspaced_service() -> Router { - let cors = CorsLayer::new() - .allow_methods([http::Method::GET, http::Method::POST]) - .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) - .allow_origin(Any); - - Router::new() - .route("/get/:id", post(get_job_metrics).layer(cors.clone())) - .route( - "/set_progress/:id", - post(set_job_progress).layer(cors.clone()), - ) - .route( - "/get_progress/:id", - get(get_job_progress).layer(cors.clone()), - ) -} - -#[derive(Deserialize)] -struct JobStatsRequest { - from_timestamp: Option>, - to_timestamp: Option>, - timeseries_max_datapoints: Option, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all -} - -#[derive(Serialize)] -struct JobStatsResponse { - metrics_metadata: Vec, - scalar_metrics: Vec, - timeseries_metrics: Vec, -} - -#[derive(Serialize)] -pub struct MetricsMetadata { - pub id: String, - pub name: Option, -} - -#[derive(Serialize)] -struct ScalarMetric { - metric_id: String, - value: f64, -} - -#[derive(Serialize)] -struct TimeseriesMetric { - metric_id: String, - values: Vec, -} - -#[derive(Debug, Clone, Serialize)] -pub struct DataPoint { - pub timestamp: chrono::DateTime, - pub value: f64, -} - -async fn get_job_metrics( - Extension(db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json< - JobStatsRequest, - >, -) -> error::JsonResult { - let records = sqlx::query_as::<_, JobStatsRecord>( - "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", - ) - .bind(w_id) - .bind(job_id) - .fetch_all(&db) - .await?; - - let mut metrics_metadata: Vec = vec![]; - let mut scalar_metrics: Vec = vec![]; - let mut timeseries_metrics: Vec = vec![]; - - for record in records { - let metric_id = record.metric_id; - match record.metric_kind { - MetricKind::ScalarInt => { - let value = record.scalar_int.unwrap_or_default() as f64; - scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); - } - MetricKind::ScalarFloat => { - let value = record.scalar_float.unwrap_or_default() as f64; - scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); - } - MetricKind::TimeseriesInt => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) - } - let (timestamps, timeseries_int) = timeseries_sample( - from_timestamp, - to_timestamp, - timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_int.unwrap_or_default(), - ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_int.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } - timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); - } - MetricKind::TimeseriesFloat => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) - } - let (timestamps, timeseries_float) = timeseries_sample( - from_timestamp, - to_timestamp, - timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_float.unwrap_or_default(), - ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_float.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } - timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); - } - }; - metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name }); - } - - let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; - Ok(Json(response)) -} -#[derive(Deserialize)] -struct JobProgressSetRequest { - percent: i32, - /// Optional parent flow id - /// Used to modify flow status - /// Specifically `progress` field in corresponding FlowStatusModule in `InProgress` state - flow_job_id: Option, -} - -async fn set_job_progress( - Extension(db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Json(JobProgressSetRequest { percent, flow_job_id }): Json, -) -> error::JsonResult<()> { - // If flow_job_id exists, than we should modify flow_status of corresponding module - // Individual jobs and flows are handled differently - if let Some(flow_job_id) = flow_job_id { - // TODO: Return error if trying to set completed job? - sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1) - WHERE id = $2", - serde_json::json!(percent.clamp(0, 99)), - flow_job_id - ) - .execute(&db) - .await?; - } - - let record_progress = || { - record_metric( - &db, - w_id.clone(), - job_id, - "progress_perc".to_owned(), - MetricNumericValue::Integer(percent), - ) - }; - - // Try to record - if let Err(err) = record_progress().await { - if matches!(err, Error::MetricNotFound(..)) { - // Register - // TODO: Reset progress after job is finished (in case it reruns same job)? - _ = register_metric_for_job( - &db, - w_id.clone(), - job_id, - "progress_perc".to_string(), - MetricKind::ScalarInt, - Some("Job Execution Progress (%)".to_owned()), - ) - .await?; - // Retry recording progress - record_progress().await.map_err(|err| { - // If for some reason it still returns same error, this error will be converted to BadRequest and returned - if let Error::MetricNotFound(body) = err { - Error::BadRequest(body) - } else { - err - } - })?; - } else { - return Err(err); - } - }; - return Ok(Json(())); -} - -async fn get_job_progress( - Extension(db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, -) -> error::JsonResult> { - let progress: Option> = sqlx::query_scalar!( - "SELECT (scalar_int)::int FROM job_stats WHERE job_id = $1 AND workspace_id = $2 AND metric_id = 'progress_perc'", - job_id, w_id) - .fetch_optional(&db) - .await?; - - let respond_value = if let Some(Some(progress)) = progress { - Some(progress.clamp(0, 99)) - } else { - None - }; - - Ok(Json(respond_value)) -} - -fn timeseries_sample( - from: Option>, - to: Option>, - _datapoints: Option, - timestamps: Vec>, - values: Vec, -) -> (Vec>, Vec) { - if timestamps.len() != values.len() { - tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len()); - return (vec![], vec![]); - } - let mut filtered_timestamp: Vec> = vec![]; - let mut filtered_values: Vec = vec![]; - for (idx, timestamp) in timestamps.iter().enumerate() { - if *timestamp > from.unwrap_or(chrono::DateTime::::MIN_UTC) - && *timestamp < to.unwrap_or(chrono::DateTime::::MAX_UTC) - { - filtered_timestamp.push(timestamps[idx]); - filtered_values.push(values[idx]); - } - } - // TODO: implement sampling - return (filtered_timestamp, filtered_values); -} +pub use windmill_api_jobs::job_metrics::*; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 15adf46caa..6974c5f244 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6,22 +6,20 @@ * LICENSE-AGPL for a copy of the license. */ +pub use windmill_api_jobs::execution::*; +pub use windmill_api_jobs::query::*; +pub use windmill_api_jobs::types::*; +pub use windmill_api_sse::*; + use axum::body::Body; -use axum::extract::Request; -use axum::http::HeaderValue; -#[cfg(feature = "deno_core")] -use deno_core::{op2, serde_v8, v8, JsRuntime, OpState}; use futures::{StreamExt, TryFutureExt}; -use http::{HeaderMap, HeaderName}; use itertools::Itertools; use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; -use sqlx::Pool; use std::collections::HashMap; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::ops::{Deref, DerefMut}; use std::str::FromStr; +use std::sync::Arc; use std::time::Instant; use tokio::io::AsyncReadExt; use tower::ServiceBuilder; @@ -29,18 +27,20 @@ use url::Url; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; +#[cfg(feature = "inline_preview")] use windmill_common::client::AuthedClient; use windmill_common::db::UserDbWithAuthed; use windmill_common::error::JsonResult; -use windmill_common::flow_conversations::add_message_to_conversation_tx; use windmill_common::flow_status::{JobResult, RestartedFrom}; +#[cfg(feature = "inline_preview")] +use windmill_common::jobs::RunInlinePreviewScriptFnParams; use windmill_common::jobs::{ - check_tag_available_for_workspace_internal, format_completed_job_result, format_result, - DynamicInput, JobTriggerKind, RunInlinePreviewScriptFnParams, ENTRYPOINT_OVERRIDE, + format_completed_job_result, format_result, DynamicInput, ENTRYPOINT_OVERRIDE, }; use windmill_common::runnable_settings::{ - ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettings, + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, }; +#[cfg(feature = "inline_preview")] use windmill_common::runtime_assets::{register_runtime_asset, InsertRuntimeAssetParams}; use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use windmill_common::scripts::ScriptRunnableSettingsInline; @@ -53,11 +53,14 @@ use windmill_common::workspace_dependencies::{ use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; +#[cfg(feature = "inline_preview")] use windmill_parser::asset_parser::AssetKind; +#[cfg(feature = "inline_preview")] use windmill_worker::get_worker_internal_server_inline_utils; use windmill_common::variables::get_workspace_key; +#[cfg(feature = "inline_preview")] use crate::db::OptJobAuthed; use crate::triggers::trigger_helpers::{FlowId, ScriptId}; use crate::{ @@ -74,16 +77,15 @@ use crate::{ }; use anyhow::Context; use axum::{ - extract::{FromRequest, Json, Path, Query}, + extract::{Json, Path, Query}, response::{IntoResponse, Response}, routing::{get, post}, Extension, Router, }; -use base64::Engine; use chrono::Utc; use hmac::Mac; use hyper::StatusCode; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use sql_builder::prelude::*; use sqlx::types::JsonRawValue; use sqlx::{types::Uuid, FromRow, Postgres, Transaction}; @@ -104,24 +106,15 @@ use windmill_common::{ oauth2::HmacSha256, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, - utils::{ - not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, - Pagination, StripPath, - }, + utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, }; use windmill_common::{ get_flow_version_info_from_version, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, get_script_info_for_hash, utils::empty_as_none, - FlowVersionInfo, ScriptHashInfo, BASE_URL, + ScriptHashInfo, BASE_URL, }; -use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs, - PushArgsOwned, PushIsolationLevel, -}; - -use crate::flow_conversations; -use windmill_common::flow_conversations::MessageType; +use windmill_queue::{job_is_complete, push, PushArgs, PushArgsOwned, PushIsolationLevel}; pub fn workspaced_service() -> Router { let cors = CorsLayer::new() @@ -1156,7 +1149,7 @@ impl<'a> GetQuery<'a> { #[cfg(all(feature = "smtp", feature = "enterprise"))] async fn send_workspace_trigger_failure_email_notification( - db: &Pool, + db: &DB, w_id: &str, job_id: &Uuid, trigger_path: Option<&str>, @@ -1726,337 +1719,6 @@ async fn get_started_at_by_ids( Ok(Json(r)) } -#[derive(Debug, sqlx::FromRow, Serialize)] -pub struct ListableCompletedJob { - pub r#type: String, - pub workspace_id: String, - pub id: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub duration_ms: i64, - pub success: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_path: Option, - pub deleted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_code: Option, - pub canceled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_reason: Option, - pub job_kind: JobKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub schedule_path: Option, - pub permissioned_as: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_flow: Option, - pub is_flow_step: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option, - pub is_skipped: bool, - pub email: String, - pub visible_to_owner: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - pub tag: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub labels: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub args: Option, -} - -#[derive(Debug, Deserialize, Clone, Default)] -pub struct RunJobQuery { - pub scheduled_for: Option>, - pub scheduled_in_secs: Option, - pub parent_job: Option, - pub root_job: Option, - pub invisible_to_owner: Option, - pub queue_limit: Option, - pub payload: Option, - pub job_id: Option, - pub tag: Option, - pub timeout: Option, - pub cache_ttl: Option, - pub cache_ignore_s3_path: Option, - pub skip_preprocessor: Option, - pub poll_delay_ms: Option, - pub memory_id: Option, - pub trigger_external_id: Option, - pub service_name: Option, - pub suspended_mode: Option, -} - -impl RunJobQuery { - async fn get_scheduled_for<'c>( - &self, - db: &DB, - ) -> error::Result>> { - if let Some(scheduled_for) = self.scheduled_for { - Ok(Some(scheduled_for)) - } else if let Some(scheduled_in_secs) = self.scheduled_in_secs { - let now = now_from_db(db).await?; - Ok(Some( - now + chrono::Duration::try_seconds(scheduled_in_secs).unwrap_or_default(), - )) - } else { - Ok(None) - } - } - - fn payload_as_args(&self) -> error::Result>> { - let payload_r = self.payload.clone().map(decode_payload).map(|x| { - x.map_err(|e| { - error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) - }) - }); - - let payload_as_args = if let Some(payload) = payload_r { - payload? - } else { - HashMap::new() - }; - - Ok(payload_as_args) - } -} - -#[derive(Deserialize, Clone)] -pub struct ListQueueQuery { - pub script_path_start: Option, - pub script_path_exact: Option, - pub script_hash: Option, - pub created_by: Option, - pub started_before: Option>, - pub started_after: Option>, - pub created_before: Option>, - pub created_after: Option>, - pub created_or_started_before: Option>, - pub created_or_started_after: Option>, - pub running: Option, - pub schedule_path: Option, - pub parent_job: Option, - pub order_desc: Option, - pub job_kinds: Option, - pub suspended: Option, - pub worker: Option, - // filter by matching a subset of the args using base64 encoded json subset - pub args: Option, - pub tag: Option, - pub scheduled_for_before_now: Option, - pub all_workspaces: Option, - pub is_flow_step: Option, - pub has_null_parent: Option, - pub is_not_schedule: Option, - pub concurrency_key: Option, - pub allow_wildcards: Option, - pub trigger_kind: Option, - pub trigger_path: Option, - pub include_args: Option, -} - -impl From for ListQueueQuery { - fn from(lcq: ListCompletedQuery) -> Self { - Self { - script_path_start: lcq.script_path_start, - script_path_exact: lcq.script_path_exact, - script_hash: lcq.script_hash, - created_by: lcq.created_by, - started_before: lcq.started_before, - started_after: lcq.started_after, - created_before: lcq.created_before_queue.or(lcq.created_before), - created_after: lcq.created_after_queue.or(lcq.created_after), - created_or_started_before: lcq.created_or_started_before, - created_or_started_after: lcq.created_or_started_after, - worker: lcq.worker, - running: lcq.running, - parent_job: lcq.parent_job, - order_desc: lcq.order_desc, - job_kinds: lcq.job_kinds, - suspended: lcq.suspended, - args: lcq.args, - tag: lcq.tag, - schedule_path: lcq.schedule_path, - scheduled_for_before_now: lcq.scheduled_for_before_now, - all_workspaces: lcq.all_workspaces, - is_flow_step: lcq.is_flow_step, - has_null_parent: lcq.has_null_parent, - is_not_schedule: lcq.is_not_schedule, - concurrency_key: lcq.concurrency_key, - allow_wildcards: lcq.allow_wildcards, - trigger_kind: lcq.trigger_kind, - trigger_path: lcq.trigger_path, - include_args: lcq.include_args, - } - } -} - -pub fn filter_list_queue_query( - mut sqlb: SqlBuilder, - lq: &ListQueueQuery, - w_id: &str, - join_outstanding_wait_times: bool, -) -> SqlBuilder { - sqlb.join("v2_job").on_eq("v2_job_queue.id", "v2_job.id"); - - if join_outstanding_wait_times { - sqlb.left() - .join("outstanding_wait_time") - .on_eq("v2_job_queue.id", "outstanding_wait_time.job_id"); - } - - if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { - sqlb.and_where_eq("v2_job_queue.workspace_id", "?".bind(&w_id)); - } - - if let Some(w) = &lq.worker { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job_queue.worker", w.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job_queue.worker", "?".bind(w)); - } - } - - if let Some(ps) = &lq.script_path_start { - sqlb.and_where_like_left("runnable_path", ps); - } - if let Some(p) = &lq.script_path_exact { - sqlb.and_where_eq("runnable_path", "?".bind(p)); - } - if let Some(p) = &lq.schedule_path { - sqlb.and_where_eq("trigger", "?".bind(p)); - sqlb.and_where_eq("trigger_kind", "'schedule'"); - } - if let Some(h) = &lq.script_hash { - sqlb.and_where_eq("runnable_id", "?".bind(h)); - } - if let Some(cb) = &lq.created_by { - sqlb.and_where_eq("created_by", "?".bind(cb)); - } - if let Some(t) = &lq.tag { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job.tag", "?".bind(t)); - } - } - - if let Some(r) = &lq.running { - sqlb.and_where_eq("running", &r); - } - if let Some(pj) = &lq.parent_job { - sqlb.and_where_eq("parent_job", "?".bind(pj)); - } - if let Some(dt) = &lq.started_before { - sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.started_after { - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(fs) = &lq.is_flow_step { - if *fs { - sqlb.and_where_is_not_null("flow_step_id"); - } else { - sqlb.and_where_is_null("flow_step_id"); - } - } - if let Some(fs) = &lq.has_null_parent { - if *fs { - sqlb.and_where_is_null("parent_job"); - } - } - - if let Some(dt) = &lq.created_before { - sqlb.and_where_le("v2_job.created_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.created_after { - sqlb.and_where_ge("v2_job.created_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.created_or_started_after { - let ts = dt.timestamp_millis(); - sqlb.and_where(format!("(started_at IS NOT NULL AND started_at >= to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at >= to_timestamp({} / 1000.0))", ts, ts)); - } - - if let Some(dt) = &lq.created_or_started_before { - let ts = dt.timestamp_millis(); - sqlb.and_where(format!("(started_at IS NOT NULL AND started_at < to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at < to_timestamp({} / 1000.0))", ts, ts)); - } - - if let Some(s) = &lq.suspended { - if *s { - sqlb.and_where_gt("suspend", 0); - } else { - sqlb.and_where_is_null("suspend_until"); - } - } - - if let Some(jk) = &lq.job_kinds { - sqlb.and_where_in( - "kind", - &jk.split(',').into_iter().map(quote).collect::>(), - ); - } - - if let Some(args) = &lq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); - } - - if lq.scheduled_for_before_now.is_some_and(|x| x) { - sqlb.and_where_le("scheduled_for", "now()"); - } - - if lq.is_not_schedule.unwrap_or(false) { - sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); - } - - if let Some(tk) = &lq.trigger_kind { - sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); - } - - if let Some(tp) = &lq.trigger_path { - sqlb.and_where_eq("trigger", "?".bind(tp)); - } - - sqlb -} - -pub fn list_queue_jobs_query( - w_id: &str, - lq: &ListQueueQuery, - fields: &[&str], - pagination: Pagination, - join_outstanding_wait_times: bool, - tags: Option>, -) -> SqlBuilder { - let (limit, offset) = paginate_without_limits(pagination); - let mut sqlb = SqlBuilder::select_from("v2_job_queue") - .fields(fields) - .order_by("v2_job.created_at", lq.order_desc.unwrap_or(true)) - .limit(limit) - .offset(offset) - .clone(); - - if let Some(tags) = tags { - sqlb.and_where_in( - "v2_job.tag", - &tags.iter().map(|x| quote(x)).collect::>(), - ); - } - - filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times) -} - #[derive(Serialize, FromRow)] struct ListableQueuedJob { pub id: Uuid, @@ -2137,103 +1799,6 @@ async fn list_queue_jobs( Ok(Json(jobs)) } -pub async fn cancel_jobs( - jobs: Vec, - db: &DB, - username: &str, - w_id: &str, - force_cancel: bool, -) -> error::JsonResult> { - let mut uuids = vec![]; - tracing::info!("Cancelling jobs: {:?}", jobs); - let mut tx = db.begin().await?; - let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj - ( workspace_id - , id - , duration_ms - , result - , canceled_by - , canceled_reason - , flow_status - , status - , worker - ) - SELECT q.workspace_id - , q.id - , 0 - , $4 - , $1 - , 'cancel all' - , (SELECT flow_status FROM v2_job_status WHERE id = q.id) - , 'canceled'::job_status - , worker - FROM v2_job_queue q - JOIN v2_job USING (id) - WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule' - FOR UPDATE SKIP LOCKED - ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) - .fetch_all(&mut *tx) - .await?.into_iter().map(|x| x.id).collect::>(); - - sqlx::query!( - "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", - &trivial_jobs, - w_id - ) - .execute(&mut *tx) - .await?; - tx.commit().await?; - - // sqlx::query!( - // "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = 'cancelled all by user' WHERE id IN (SELECT id FROM queue where id = any($2) AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED) RETURNING id", - // username, - // &jobs, - // w_id - // ).execute(db).await?; - for job_id in jobs.into_iter() { - if trivial_jobs.contains(&job_id) { - continue; - } - match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move { - let tx = db.begin().await?; - let (tx, _) = windmill_queue::cancel_job( - username, - None, - job_id.clone(), - w_id, - tx, - db, - force_cancel, - false, - ) - .await?; - tx.commit().await?; - Ok::<_, anyhow::Error>(()) - }) - .await - { - Ok(result) => match result { - Ok(_) => { - uuids.push(job_id); - } - Err(e) => { - tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e); - } - }, - Err(_) => { - tracing::error!( - "Timeout while trying to cancel job {:?} after 5 seconds", - job_id - ); - } - } - } - - uuids.extend(trivial_jobs); - - Ok(Json(uuids)) -} - #[derive(Deserialize)] pub struct CancelSelectionQuery { force_cancel: Option, @@ -2948,14 +2513,6 @@ pub struct SuspendedJobFlow { pub approvers: Vec, } -#[derive(Deserialize, Debug)] -pub struct QueryApprover { - pub approver: Option, - /// If true, generate/verify resume URLs for the parent flow instead of the specific step. - /// This allows pre-approvals that can be consumed by any later suspend step in the same flow. - pub flow_level: Option, -} - pub async fn get_suspended_job_flow( authed: Option, opt_tokened: OptTokened, @@ -3170,14 +2727,6 @@ fn create_signature( Ok(hex::encode(mac.finalize().into_bytes())) } -#[allow(non_snake_case)] -#[derive(Serialize, Debug)] -pub struct ResumeUrls { - pub approvalPage: String, - pub cancel: String, - pub resume: String, -} - fn build_resume_url( op: &str, w_id: &str, @@ -3282,466 +2831,6 @@ async fn get_flow_id_for_job(db: &DB, job_id: Uuid) -> error::Result { .ok_or_else(|| anyhow::anyhow!("job {} has no parent flow", job_id).into()) } -#[derive(sqlx::FromRow, Debug, Serialize)] -pub struct JobExtended { - #[sqlx(flatten)] - #[serde(flatten)] - inner: T, - - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_lock: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_flow: Option>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub worker: Option, - - #[sqlx(skip)] - #[serde(skip_serializing_if = "Option::is_none")] - pub self_wait_time_ms: Option, - #[sqlx(skip)] - #[serde(skip_serializing_if = "Option::is_none")] - pub aggregate_wait_time_ms: Option, -} - -pub trait JobCommon { - fn job_kind(&self) -> &JobKind; -} - -impl JobCommon for QueuedJob { - fn job_kind(&self) -> &JobKind { - &self.job_kind - } -} - -impl JobCommon for CompletedJob { - fn job_kind(&self) -> &JobKind { - &self.job_kind - } -} - -impl JobExtended { - pub fn new( - self_wait_time_ms: Option, - aggregate_wait_time_ms: Option, - inner: T, - ) -> Self { - Self { - inner, - raw_code: None, - raw_lock: None, - raw_flow: None, - worker: None, - self_wait_time_ms, - aggregate_wait_time_ms, - } - } -} - -impl Deref for JobExtended { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl DerefMut for JobExtended { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -#[derive(Serialize, Debug)] -#[serde(tag = "type")] -pub enum Job { - QueuedJob(JobExtended), - CompletedJob(JobExtended), -} - -impl Job { - pub fn created_by(&self) -> &str { - match self { - Job::QueuedJob(job) => &job.created_by, - Job::CompletedJob(job) => &job.created_by, - } - } - - pub fn append_to_logs(&mut self, logs: &str) { - match self { - Job::QueuedJob(job) => { - if let Some(ref mut l) = job.logs { - l.push_str(logs); - } else { - job.logs = Some(logs.to_string()); - } - } - Job::CompletedJob(job) => { - if let Some(ref mut l) = job.logs { - l.push_str(logs); - } else { - job.logs = Some(logs.to_string()); - } - } - } - } - - pub fn log_len(&self) -> Option { - match self { - Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()), - Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()), - } - } - - pub fn logs(&self) -> Option { - match self { - Job::QueuedJob(job) => job.logs.clone(), - Job::CompletedJob(job) => job.logs.clone(), - } - } - pub fn flow_status(&self) -> Option { - match self { - Job::QueuedJob(job) => job - .flow_status - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), - Job::CompletedJob(job) => job - .flow_status - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), - } - } - pub fn is_flow_step(&self) -> bool { - match self { - Job::QueuedJob(job) => job.is_flow_step, - Job::CompletedJob(job) => job.is_flow_step, - } - } - - pub fn is_flow(&self) -> bool { - self.job_kind().is_flow() - } - - pub fn job_kind(&self) -> &JobKind { - match self { - Job::QueuedJob(job) => &job.job_kind, - Job::CompletedJob(job) => &job.job_kind, - } - } - - pub fn id(&self) -> Uuid { - match self { - Job::QueuedJob(job) => job.id, - Job::CompletedJob(job) => job.id, - } - } - - pub fn workspace_id(&self) -> &String { - match self { - Job::QueuedJob(job) => &job.workspace_id, - Job::CompletedJob(job) => &job.workspace_id, - } - } - - pub fn script_path(&self) -> &str { - match self { - Job::QueuedJob(job) => job.script_path.as_ref(), - Job::CompletedJob(job) => job.script_path.as_ref(), - } - .map(String::as_str) - .unwrap_or("tmp/main") - } - - pub fn args(&self) -> Option<&sqlx::types::Json>>> { - match self { - Job::QueuedJob(job) => job.args.as_ref(), - Job::CompletedJob(job) => job.args.as_ref(), - } - } - - pub fn full_path_with_workspace(&self) -> String { - format!( - "{}/{}/{}", - self.workspace_id(), - if self.is_flow() { "flow" } else { "script" }, - self.script_path(), - ) - } - - pub async fn concurrency_key( - &self, - db: &Pool, - ) -> Result, sqlx::Error> { - sqlx::query_scalar!( - "SELECT key FROM concurrency_key WHERE job_id = $1", - self.id() - ) - .fetch_optional(db) - .await - } - - pub async fn fetch_outstanding_wait_time( - &mut self, - db: &Pool, - ) -> Result<(), sqlx::Error> { - let r = sqlx::query!( - "SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1", - self.id() - ) - .fetch_optional(db) - .await?; - - let (self_wait_time, aggregate_wait_time) = r - .map(|x| (x.self_wait_time_ms, x.aggregate_wait_time_ms)) - .unwrap_or((None, None)); - - match self { - Job::QueuedJob(job) => { - job.self_wait_time_ms = self_wait_time; - job.aggregate_wait_time_ms = aggregate_wait_time; - } - Job::CompletedJob(job) => { - job.self_wait_time_ms = self_wait_time; - job.aggregate_wait_time_ms = aggregate_wait_time; - } - } - Ok(()) - } -} - -#[derive(sqlx::FromRow)] -pub struct UnifiedJob { - pub workspace_id: String, - pub typ: String, - pub id: Uuid, - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub completed_at: Option>, - pub scheduled_for: Option>, - pub running: Option, - pub script_hash: Option, - pub script_path: Option, - pub args: Option, - pub duration_ms: Option, - pub success: Option, - pub deleted: bool, - pub canceled: bool, - pub canceled_by: Option, - pub job_kind: JobKind, - pub schedule_path: Option, - pub permissioned_as: String, - pub is_flow_step: bool, - pub language: Option, - pub is_skipped: bool, - pub email: String, - pub visible_to_owner: bool, - pub suspend: Option, - pub mem_peak: Option, - pub tag: String, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub priority: Option, - pub labels: Option, - pub self_wait_time_ms: Option, - pub aggregate_wait_time_ms: Option, - pub preprocessed: Option, - pub worker: Option, - pub runnable_settings_handle: Option, -} - -const CJ_FIELDS: &[&str] = &[ - "'CompletedJob' as typ", - "v2_job_completed.id", - "v2_job_completed.workspace_id", - "v2_job.parent_job", - "v2_job.created_by", - "v2_job.created_at", - "v2_job_completed.started_at", - "null as scheduled_for", - "v2_job_completed.completed_at", - "null as running", - "v2_job.runnable_id as script_hash", - "v2_job.runnable_path as script_path", - "null as args", - "v2_job_completed.duration_ms", - "v2_job_completed.status = 'success' OR v2_job_completed.status = 'skipped' as success", - "false as deleted", - "v2_job_completed.status = 'canceled' as canceled", - "v2_job_completed.canceled_by", - "v2_job.kind as job_kind", - "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", - "v2_job.permissioned_as", - "v2_job.flow_step_id IS NOT NULL as is_flow_step", - "v2_job.script_lang as language", - "v2_job_completed.status = 'skipped' as is_skipped", - "v2_job.permissioned_as_email as email", - "v2_job.visible_to_owner", - "null as suspend", - "v2_job_completed.memory_peak as mem_peak", - "v2_job.tag", - "null as concurrent_limit", - "null as concurrency_time_window_s", - "v2_job.priority", - "v2_job_completed.result->'wm_labels' as labels", - "self_wait_time_ms", - "aggregate_wait_time_ms", - "v2_job.preprocessed", - "v2_job_completed.worker", - "null as runnable_settings_handle", -]; - -const QJ_FIELDS: &[&str] = &[ - "'QueuedJob' as typ", - "v2_job_queue.id", - "v2_job_queue.workspace_id", - "v2_job.parent_job", - "v2_job.created_by", - "v2_job_queue.created_at", - "v2_job_queue.started_at", - "v2_job_queue.scheduled_for", - "null as completed_at", - "v2_job_queue.running", - "v2_job.runnable_id as script_hash", - "v2_job.runnable_path as script_path", - "null as args", - "null as duration_ms", - "null as success", - "false as deleted", - "v2_job_queue.canceled_by IS NOT NULL as canceled", - "v2_job_queue.canceled_by", - "v2_job.kind as job_kind", - "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", - "v2_job.permissioned_as", - "v2_job.flow_step_id IS NOT NULL as is_flow_step", - "v2_job.script_lang as language", - "false as is_skipped", - "v2_job.permissioned_as_email as email", - "v2_job.visible_to_owner", - "v2_job_queue.suspend", - "null as mem_peak", - "v2_job.tag", - "v2_job.concurrent_limit", - "v2_job.concurrency_time_window_s", - "v2_job.priority", - "null as labels", - "self_wait_time_ms", - "aggregate_wait_time_ms", - "v2_job.preprocessed", - "v2_job_queue.worker", - "v2_job_queue.runnable_settings_handle", -]; - -impl UnifiedJob { - pub fn completed_job_fields() -> &'static [&'static str] { - CJ_FIELDS - } - pub fn queued_job_fields() -> &'static [&'static str] { - QJ_FIELDS - } -} - -impl<'a> From for Job { - fn from(uj: UnifiedJob) -> Self { - let args = uj.args.and_then(|v| serde_json::from_value(v).ok()); - match uj.typ.as_ref() { - "CompletedJob" => Job::CompletedJob(JobExtended::new( - uj.self_wait_time_ms, - uj.aggregate_wait_time_ms, - CompletedJob { - workspace_id: uj.workspace_id, - id: uj.id, - parent_job: uj.parent_job, - created_by: uj.created_by, - created_at: uj.created_at, - started_at: uj.started_at, - completed_at: uj.completed_at, - duration_ms: uj.duration_ms.unwrap(), - success: uj.success.unwrap(), - script_hash: uj.script_hash, - script_path: uj.script_path, - args: args.clone(), - result: None, - result_columns: None, - logs: None, - flow_status: None, - workflow_as_code_status: None, - deleted: uj.deleted, - canceled: uj.canceled, - canceled_by: uj.canceled_by, - canceled_reason: None, - job_kind: uj.job_kind, - schedule_path: uj.schedule_path, - permissioned_as: uj.permissioned_as, - is_flow_step: uj.is_flow_step, - language: uj.language, - is_skipped: uj.is_skipped, - email: uj.email, - visible_to_owner: uj.visible_to_owner, - mem_peak: uj.mem_peak, - tag: uj.tag, - priority: uj.priority, - labels: uj.labels, - preprocessed: uj.preprocessed, - }, - )), - "QueuedJob" => Job::QueuedJob(JobExtended::new( - uj.self_wait_time_ms, - uj.aggregate_wait_time_ms, - QueuedJob { - workspace_id: uj.workspace_id, - id: uj.id, - parent_job: uj.parent_job, - created_by: uj.created_by, - created_at: uj.created_at, - started_at: uj.started_at, - scheduled_for: uj.scheduled_for.unwrap(), - running: uj.running.unwrap(), - script_hash: uj.script_hash, - script_path: uj.script_path, - script_entrypoint_override: None, - args, - logs: None, - canceled: uj.canceled, - canceled_by: uj.canceled_by, - canceled_reason: None, - last_ping: None, - job_kind: uj.job_kind, - schedule_path: uj.schedule_path, - permissioned_as: uj.permissioned_as, - flow_status: None, - workflow_as_code_status: None, - is_flow_step: uj.is_flow_step, - language: uj.language, - same_worker: false, - pre_run_error: None, - email: uj.email, - visible_to_owner: uj.visible_to_owner, - suspend: uj.suspend, - mem_peak: uj.mem_peak, - root_job: None, - leaf_jobs: None, - tag: uj.tag, - concurrent_limit: uj.concurrent_limit, - concurrency_time_window_s: uj.concurrency_time_window_s, - timeout: None, - flow_step_id: None, - cache_ttl: None, - cache_ignore_s3_path: None, - priority: uj.priority, - preprocessed: uj.preprocessed, - runnable_settings_handle: uj.runnable_settings_handle, - }, - )), - t => panic!("job type {} not valid", t), - } - } -} #[derive(Deserialize)] struct CancelJob { reason: Option, @@ -3772,6 +2861,7 @@ struct Preview { format: Option, } +#[cfg(feature = "inline_preview")] #[derive(Debug, Deserialize)] struct PreviewInline { content: String, @@ -3809,94 +2899,6 @@ pub enum DynamicSelectRunnableRef { Inline { code: String, lang: Option }, } -pub struct QueryOrBody(pub Option); - -#[axum::async_trait] -impl FromRequest for QueryOrBody -where - D: DeserializeOwned, - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request( - req: Request, - state: &S, - ) -> std::result::Result { - return if req.method() == axum::http::Method::GET { - let Query(InPayload { payload }) = Query::from_request(req, state) - .await - .map_err(IntoResponse::into_response)?; - payload - .map(|p| { - decode_payload(p) - .map(QueryOrBody) - .map_err(|err| (StatusCode::BAD_REQUEST, format!("{err:#?}"))) - .map_err(IntoResponse::into_response) - }) - .unwrap_or(Ok(QueryOrBody(None))) - } else { - Json::from_request(req, state) - .await - .map(|Json(v)| QueryOrBody(Some(v))) - .map_err(IntoResponse::into_response) - }; - - #[derive(Deserialize)] - struct InPayload { - payload: Option, - } - } -} - -fn decode_payload(t: String) -> anyhow::Result { - let vec = base64::engine::general_purpose::STANDARD - .decode(t) - .context("invalid base64")?; - serde_json::from_slice(vec.as_slice()).context("invalid json") -} - -pub fn add_raw_string( - raw_string: Option, - mut args: serde_json::Map, -) -> serde_json::Map { - if let Some(raw_string) = raw_string { - args.insert( - "raw_string".to_string(), - serde_json::Value::String(raw_string), - ); - } - return args; -} - -pub async fn check_tag_available_for_workspace( - db: &DB, - w_id: &str, - tag: &Option, - authed: &ApiAuthed, -) -> error::Result<()> { - if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { - let tags = get_scope_tags(authed); - check_tag_available_for_workspace_internal(&db, w_id, tag, &authed.email, tags).await - } else { - Ok(()) - } -} - -#[cfg(feature = "enterprise")] -pub async fn check_license_key_valid() -> error::Result<()> { - use windmill_common::ee_oss::LICENSE_KEY_VALID; - - let valid = *LICENSE_KEY_VALID.read().await; - if !valid { - return Err(Error::BadRequest( - "License key is not valid. Go to your superadmin settings to update your license key." - .to_string(), - )); - } - Ok(()) -} - use windmill_common::flows::InputTransform; #[derive(Deserialize)] @@ -3924,50 +2926,18 @@ struct BatchReRunQueryReturnType { schema: Option, } -#[cfg(feature = "deno_core")] -#[op2] -#[string] -fn get_deno_core_job_value(state: &mut OpState) -> Option { - let obj = state.borrow::(); - let str = serde_json::to_string(&obj).ok()?; - Some(str) -} - -#[cfg(feature = "deno_core")] async fn batch_rerun_compute_js_expression( expr: String, job: BatchReRunQueryReturnType, ) -> error::Result> { - let ext = deno_core::Extension { - name: "batch_rerun_arg_transform_ext", - ops: vec![get_deno_core_job_value()].into(), - ..Default::default() - }; - let mut isolate = - JsRuntime::new(deno_core::RuntimeOptions { extensions: vec![ext], ..Default::default() }); - - { - let op_state = isolate.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(BatchReRunQueryReturnType { schema: None, ..job }); - } - isolate - .execute_script( - "", - "let job = JSON.parse(Deno.core.ops.get_deno_core_job_value());", - ) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - - // Run user expr - let result = isolate - .execute_script("", expr) - .map_err(|e| Error::ExecutionErr(e.to_string()))?; - let mut scope = isolate.handle_scope(); - let result = v8::Local::new(&mut scope, result); - let result: serde_json::Value = - serde_v8::from_v8(&mut scope, result).map_err(|e| Error::ExecutionErr(e.to_string()))?; - let result = JsonRawValue::from_string(result.to_string())?; - Ok(result) + let job_no_schema = BatchReRunQueryReturnType { schema: None, ..job }; + let job_value = + serde_json::to_value(&job_no_schema).map_err(|e| Error::ExecutionErr(e.to_string()))?; + let mut globals = std::collections::HashMap::new(); + globals.insert("job".to_string(), job_value); + windmill_jseval::eval_simple_js(expr, globals) + .await + .map_err(|e| Error::ExecutionErr(e.to_string())) } async fn batch_rerun_jobs( @@ -4094,13 +3064,6 @@ async fn batch_rerun_handle_job( args.insert(property_name.clone(), value.clone()); } InputTransform::Javascript { expr } => { - #[cfg(not(feature = "deno_core"))] - Err(error::Error::ExecutionErr( - format!("deno_core feature is not activated, cannot evaluate: {expr}") - .to_string(), - ))?; - - #[cfg(feature = "deno_core")] args.insert( property_name.clone(), batch_rerun_compute_js_expression(expr.clone(), job.clone()).await?, @@ -4169,95 +3132,6 @@ async fn batch_rerun_handle_job( )) } -/// Set the memory_id in flow_status for agent memory persistence -async fn set_flow_memory_id( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - job_id: Uuid, - memory_id: Uuid, -) -> error::Result<()> { - sqlx::query!( - "UPDATE v2_job_status - SET flow_status = jsonb_set( - flow_status, - '{memory_id}', - to_jsonb($2::uuid) - ) - WHERE id = $1", - job_id, - memory_id - ) - .execute(&mut **tx) - .await?; - Ok(()) -} - -/// Apply flow-specific query parameters after job creation -pub async fn process_flow_run_query_params( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - job_id: Uuid, - run_query: &RunJobQuery, -) -> error::Result<()> { - // Set memory_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { - set_flow_memory_id(tx, job_id, memory_id).await?; - } - Ok(()) -} - -async fn handle_chat_conversation_messages( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - authed: &ApiAuthed, - w_id: &str, - flow_path: &str, - run_query: &RunJobQuery, - user_message_raw: Option<&Box>, -) -> error::Result<()> { - let memory_id = run_query.memory_id.ok_or_else(|| { - windmill_common::error::Error::BadRequest( - "memory_id is required for chat-enabled flows".to_string(), - ) - })?; - - let user_message_raw = user_message_raw.ok_or_else(|| { - windmill_common::error::Error::BadRequest( - "user_message argument is required for chat-enabled flows".to_string(), - ) - })?; - - // Deserialize the RawValue to get the actual string without quotes - let user_message: String = serde_json::from_str(user_message_raw.get()).map_err(|e| { - windmill_common::error::Error::BadRequest(format!( - "Failed to deserialize user_message: {}", - e - )) - })?; - - // Create conversation with provided ID (or get existing one) - flow_conversations::get_or_create_conversation_with_id( - tx, - w_id, - flow_path, - &authed.username, - &user_message, - memory_id, - ) - .await?; - - // Create user message - add_message_to_conversation_tx( - tx, - memory_id, - None, - &user_message, - MessageType::User, - None, - true, - ) - .await?; - - Ok(()) -} - pub async fn run_flow_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -4292,199 +3166,6 @@ pub async fn run_flow_by_path( Ok((StatusCode::CREATED, uuid.to_string())) } -pub async fn run_flow<'c>( - authed: &ApiAuthed, - db: &DB, - tx_o: Option>, - user_db: UserDB, - w_id: &str, - flow_path: &str, - flow_version_info: FlowVersionInfo, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result<( - Uuid, - Option, - Option>, -)> { - let FlowVersionInfo { - version, - tag, - dedicated_worker, - has_preprocessor, - chat_input_enabled, - on_behalf_of_email, - edited_by, - early_return, - .. - } = flow_version_info; - - let tag = run_query.tag.clone().or(tag); - - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - let scheduled_for = run_query.get_scheduled_for(&db).await?; - - let return_tx = tx_o.is_some(); - - let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { - ( - &authed.email, - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Transaction(tx), - ) - } else if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { - ( - on_behalf_of_email, - username_to_permissioned_as(&edited_by), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - &authed.email, - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), - ) - }; - - let (uuid, mut tx) = push( - &db, - tx, - &w_id, - JobPayload::Flow { - path: flow_path.to_string(), - dedicated_worker, - version, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - }, - PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - scheduled_for, - None, - run_query.parent_job, - None, - run_query.root_job, - run_query.job_id, - false, - false, - None, - !run_query.invisible_to_owner.unwrap_or(false), - tag, - None, - None, - None, - push_authed.as_ref(), - false, - None, - trigger, - run_query.suspended_mode, - ) - .await?; - - // Set memory_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { - set_flow_memory_id(&mut tx, uuid, memory_id).await?; - } - - // Handle conversation messages for chat-enabled flows - if chat_input_enabled.unwrap_or(false) { - handle_chat_conversation_messages( - &mut tx, - &authed, - &w_id, - &flow_path.to_string(), - &run_query, - args.args.get("user_message"), - ) - .await?; - } - - // If we were given a transaction, return it; otherwise commit it - if return_tx { - Ok((uuid, early_return, Some(tx))) - } else { - tx.commit().await?; - Ok((uuid, early_return, None)) - } -} - -pub async fn run_flow_and_wait_result( - authed: &ApiAuthed, - db: &DB, - user_db: UserDB, - w_id: &str, - flow_path: &str, - flow_version_info: FlowVersionInfo, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result { - let (uuid, early_return, _) = run_flow( - authed, - db, - None, - user_db, - w_id, - flow_path, - flow_version_info, - run_query, - args, - trigger, - ) - .await?; - - run_wait_result(&db, uuid, w_id, early_return, &authed.username).await -} - -pub async fn push_flow_job_by_path_into_queue<'c>( - authed: ApiAuthed, - db: DB, - tx_o: Option>, - user_db: UserDB, - w_id: String, - flow_path: StripPath, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result<( - Uuid, - Option, - Option>, -)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - let flow_path = flow_path.to_path(); - check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; - - let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - - let flow_version_info = - get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) - .await?; - - run_flow( - &authed, - &db, - tx_o, - user_db, - &w_id, - flow_path, - flow_version_info, - run_query, - args, - trigger, - ) - .await -} - pub async fn run_flow_by_version( authed: ApiAuthed, Extension(db): Extension, @@ -4745,112 +3426,6 @@ pub async fn get_args_and_trigger_metadata( Ok((args, trigger_metadata)) } -pub async fn push_script_job_by_path_into_queue<'c>( - authed: ApiAuthed, - db: DB, - tx_o: Option>, - user_db: UserDB, - w_id: String, - script_path: StripPath, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result<( - Uuid, - Option, - Option>, -)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - let script_path = script_path.to_path(); - check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; - - let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload( - script_path, - Some(userdb_authed), - db.clone(), - &w_id, - run_query.skip_preprocessor, - ) - .await?; - let scheduled_for = run_query.get_scheduled_for(&db).await?; - - let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - - let return_tx = tx_o.is_some(); - - let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Transaction(tx), - ) - } else if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), - ) - }; - - let (uuid, tx) = push( - &db, - tx, - &w_id, - job_payload, - PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - scheduled_for, - None, - run_query.parent_job, - None, - run_query.root_job, - run_query.job_id, - false, - false, - None, - !run_query.invisible_to_owner.unwrap_or(false), - tag, - timeout, - None, - // If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher. - if run_query.parent_job.is_some() || run_query.root_job.is_some() { - Some(2) - } else { - None - }, - push_authed.as_ref(), - false, - None, - trigger, - run_query.suspended_mode, - ) - .await?; - - // If we were given a transaction, return it; otherwise commit it - if return_tx { - Ok((uuid, delete_after_use, Some(tx))) - } else { - tx.commit().await?; - Ok((uuid, delete_after_use, None)) - } -} - #[derive(Deserialize)] pub struct WorkflowAsCodeQuery { pub skip_update: Option, @@ -4896,9 +3471,7 @@ pub async fn run_workflow_as_code( let JobExtended { inner: job, raw_code, raw_lock, .. } = job; let (_debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle(job.runnable_settings_handle, &db) - .await? - .prefetch_cached(&db) + windmill_common::runnable_settings::prefetch_cached_from_handle(job.runnable_settings_handle, &db) .await?; let (job_payload, tag, _delete_after_use, timeout, on_behalf_of) = match job.job_kind { @@ -5062,327 +3635,7 @@ pub async fn run_workflow_as_code( Ok((StatusCode::CREATED, uuid.to_string())) } -struct Guard { - done: bool, - id: Uuid, - w_id: String, - db: DB, - username: String, -} - -impl Drop for Guard { - fn drop(&mut self) { - if !&self.done { - let id = self.id; - let w_id = self.w_id.clone(); - let db = self.db.clone(); - let username = self.username.clone(); - - tracing::info!("http connection broke, marking job {id} as canceled"); - tokio::spawn(async move { - let cancel_f = async { - let tx = db.begin().await?; - let (tx, _) = cancel_job( - &username, - Some("http connection broke".to_string()), - id, - &w_id, - tx, - &db, - false, - false, - ) - .await?; - tx.commit().await?; - Ok::<_, anyhow::Error>(()) - }; - - if let Err(e) = cancel_f.await { - tracing::error!( - "Error marking job as canceled after http connection broke: {e}" - ); - } - }); - } - } -} - -use std::sync::Arc; -use tokio::sync::RwLock; - lazy_static::lazy_static! { - pub static ref TIMEOUT_WAIT_RESULT: Arc>> = Arc::new(RwLock::new( - std::env::var("TIMEOUT_WAIT_RESULT") - .ok() - .and_then(|x| x.parse::().ok()) - )); -} - -#[derive(Deserialize)] -pub struct WindmillCompositeResult { - windmill_status_code: Option, - windmill_content_type: Option, - windmill_headers: Option>, - result: Option>, -} - -pub async fn run_wait_result_internal( - db: &DB, - uuid: Uuid, - w_id: &str, - node_id_for_empty_return: Option, - username: &str, -) -> error::Result<(Box, bool)> { - let mut result = None; - let mut success = false; - let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600); - let timeout_ms = if timeout <= 0 { - 2000 - } else { - (timeout * 1000) as u64 - }; - - let mut g = Guard { - done: false, - id: uuid, - w_id: w_id.to_string(), - db: db.clone(), - username: username.to_string(), - }; - - let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000; - let mut accumulated_delay = 0 as u64; - - loop { - if let Some(node_id_for_empty_return) = node_id_for_empty_return.as_ref() { - let result_and_success = get_result_and_success_by_id_from_flow( - &db, - w_id, - &uuid, - node_id_for_empty_return, - None, - ) - .await - .ok(); - if let Some((r, s)) = result_and_success { - result = Some(r); - success = s; - } - } - - if result.is_none() { - let row = sqlx::query!( - " - SELECT - result AS \"result: sqlx::types::Json>\", - result_columns, - status = 'success' AS \"success!\" - FROM - v2_job_completed - WHERE - id = $1 AND - workspace_id = $2 - ", - uuid, - &w_id - ) - .fetch_optional(db) - .await?; - if let Some(mut raw_result) = row { - format_result( - raw_result.result_columns.as_ref(), - raw_result.result.as_mut(), - ); - result = raw_result.result.map(|x| x.0); - success = raw_result.success; - } - } - - if result.is_some() { - break; - } - - let delay = if accumulated_delay <= fast_poll_duration { - *WAIT_RESULT_FAST_POLL_INTERVAL_MS - } else { - *WAIT_RESULT_SLOW_POLL_INTERVAL_MS - }; - accumulated_delay += delay; - if accumulated_delay > timeout_ms { - break; - }; - tokio::time::sleep(core::time::Duration::from_millis(delay)).await; - } - - if let Some(result) = result { - g.done = true; - Ok((result, success)) - } else { - Err(Error::ExecutionErr(format!("timeout after {}s", timeout))) - } -} - -pub fn result_to_response(result: Box, success: bool) -> error::Result { - let composite_result = serde_json::from_str::(result.get()); - match composite_result { - Ok(WindmillCompositeResult { - windmill_status_code, - windmill_content_type, - windmill_headers, - result: result_value, - }) => { - if windmill_content_type.is_none() - && windmill_status_code.is_none() - && windmill_headers.is_none() - { - return Ok(( - if success { - StatusCode::OK - } else { - StatusCode::UNPROCESSABLE_ENTITY - }, - Json(result), - ) - .into_response()); - } - - let status_code_or_default = windmill_status_code - .map(|val| match StatusCode::from_u16(val) { - Ok(sc) => Ok(sc), - Err(_) => Err(Error::ExecutionErr("Invalid status code".to_string())), - }) - .unwrap_or_else(|| { - if !success { - Ok(StatusCode::UNPROCESSABLE_ENTITY) - } else if result_value.is_some() { - Ok(StatusCode::OK) - } else { - Ok(StatusCode::NO_CONTENT) - } - })?; - - let mut headers = HeaderMap::new(); - - if let Some(windmill_headers) = windmill_headers { - for (k, v) in windmill_headers { - let k = HeaderName::from_str(k.as_str()).map_err(|err| { - Error::internal_err(format!("Invalid header name {k}: {err}")) - })?; - let v = HeaderValue::from_str(v.as_str()).map_err(|err| { - Error::internal_err(format!("Invalid header value {v}: {err}")) - })?; - headers.insert(k, v); - } - } - - if let Some(content_type) = windmill_content_type { - let serialized_json_result = result_value - .map(|val| val.get().to_owned()) - .unwrap_or_else(String::new); - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let serialized_result = - serde_json::from_str::(serialized_json_result.as_str()) - .ok() - .unwrap_or(serialized_json_result); - headers.insert( - http::header::CONTENT_TYPE, - HeaderValue::from_str(content_type.as_str()).map_err(|err| { - Error::internal_err(format!("Invalid content type {content_type}: {err}")) - })?, - ); - return Ok((status_code_or_default, headers, serialized_result).into_response()); - } - if let Some(result_value) = result_value { - return Ok((status_code_or_default, headers, Json(result_value)).into_response()); - } else { - Ok((status_code_or_default, headers).into_response()) - } - } - _ => Ok(( - if success { - StatusCode::OK - } else { - StatusCode::UNPROCESSABLE_ENTITY - }, - Json(result), - ) - .into_response()), - } -} - -pub async fn run_wait_result( - db: &DB, - uuid: Uuid, - w_id: &str, - node_id_for_empty_return: Option, - username: &str, -) -> error::Result { - let (result, success) = - run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username).await?; - - result_to_response(result, success) -} - -pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> { - sqlx::query!( - "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", - job_uuid, - ) - .execute(db) - .await?; - sqlx::query!( - "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", - job_uuid, - ) - .execute(db) - .await?; - sqlx::query!( - "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", - job_uuid, - ) - .execute(db) - .await?; - Ok(()) -} - -pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { - if let Some(limit) = queue_limit { - let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() - OR (q.suspend_until IS NOT NULL - AND (q.suspend <= 0 OR q.suspend_until <= now())))", - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - if count > queue_limit.unwrap() { - return Err(Error::internal_err(format!( - "Number of queued job is too high: {count} > {limit}" - ))); - } - } - Ok(()) -} - -lazy_static::lazy_static! { - pub static ref QUEUE_LIMIT_WAIT_RESULT: Option = std::env::var("QUEUE_LIMIT_WAIT_RESULT") - .ok() - .and_then(|x| x.parse().ok()); - pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(50); - pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(2); - pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(200); - static ref JOB_VIEW_AUDIT_LOGS: bool = std::env::var("JOB_VIEW_AUDIT_LOGS") .ok() .and_then(|x| x.parse().ok()) @@ -6302,6 +4555,7 @@ async fn run_preview_script( Ok((StatusCode::CREATED, uuid.to_string())) } +#[cfg(feature = "inline_preview")] async fn run_inline_preview_script( OptJobAuthed { authed, job_id }: OptJobAuthed, Tokened { token }: Tokened, @@ -6338,6 +4592,14 @@ async fn run_inline_preview_script( Ok(Json(to_raw_value(&result)).into_response()) } +#[cfg(not(feature = "inline_preview"))] +async fn run_inline_preview_script() -> error::Result { + Err(error::Error::InternalErr( + "inline preview requires the worker feature".to_string(), + )) +} + +#[cfg(feature = "inline_preview")] fn register_potential_assets_on_inline_execution( job_id: Uuid, w_id: &str, @@ -6381,13 +4643,13 @@ fn register_potential_assets_on_inline_execution( let columns = asset.columns.as_ref().map(|cols| { cols.iter() .map(|(col_name, col_access_type)| { - (col_name.clone(), (*col_access_type).into()) + (col_name.clone(), windmill_common::assets::asset_access_type_from_parser(*col_access_type)) }) .collect() }); register_runtime_asset(InsertRuntimeAssetParams { - access_type: asset.access_type.map(|a| a.into()), - asset_kind: asset.kind.into(), + access_type: asset.access_type.map(windmill_common::assets::asset_access_type_from_parser), + asset_kind: windmill_common::assets::asset_kind_from_parser(asset.kind), asset_path: asset.path, columns, job_id, @@ -7457,75 +5719,6 @@ pub async fn run_job_by_hash_inner( Ok((uuid, delete_after_use)) } -#[derive(Deserialize)] -pub struct JobUpdateQuery { - pub running: Option, - pub log_offset: Option, - pub stream_offset: Option, - pub get_progress: Option, - pub no_logs: Option, - pub only_result: Option, - pub fast: Option, - pub is_flow: Option, - pub poll_delay_ms: Option, -} - -#[derive(Serialize, Debug)] -pub struct JobUpdate { - #[serde(skip_serializing_if = "Option::is_none")] - pub running: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub completed: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub new_logs: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub new_result_stream: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub log_offset: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_offset: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub progress: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub workflow_as_code_status: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub job: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub only_result: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_stream_job_id: Option, -} - -impl JobUpdate { - pub fn hash_str(&self) -> String { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - format!("{:x}", hasher.finish()) - } -} - -impl Hash for JobUpdate { - fn hash(&self, state: &mut H) { - self.running.hash(state); - self.completed.hash(state); - self.log_offset.hash(state); - self.mem_peak.hash(state); - self.progress.hash(state); - self.stream_offset.hash(state); - self.flow_stream_job_id.hash(state); - if !self.completed.unwrap_or(false) { - self.flow_status.as_ref().map(|x| x.get().hash(state)); - self.workflow_as_code_status - .as_ref() - .map(|x| x.get().hash(state)); - } - } -} - async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result { if file_p.contains("..") { return Err(error::Error::BadRequest("Invalid path".to_string())); @@ -7693,21 +5886,6 @@ async fn get_job_update_sse( .unwrap()) } -#[derive(Serialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum JobUpdateSSEStream { - Update(JobUpdate), - Error { error: String }, - NotFound, - Timeout, - Ping, -} - -lazy_static::lazy_static! { - pub static ref TIMEOUT_SSE_STREAM: u64 = - std::env::var("TIMEOUT_SSE_STREAM").unwrap_or("60".to_string()).parse::().unwrap_or(60); -} - pub fn start_job_update_sse_stream( opt_authed: Option, opt_tokened: OptTokened, @@ -8285,251 +6463,6 @@ async fn get_job_update_data( } } -pub fn filter_list_completed_query( - mut sqlb: SqlBuilder, - lq: &ListCompletedQuery, - w_id: &str, - join_outstanding_wait_times: bool, -) -> SqlBuilder { - sqlb.join("v2_job") - .on_eq("v2_job_completed.id", "v2_job.id"); - - if join_outstanding_wait_times { - sqlb.left() - .join("outstanding_wait_time") - .on_eq("v2_job_completed.id", "outstanding_wait_time.job_id"); - } - - if let Some(label) = &lq.label { - if lq.allow_wildcards.unwrap_or(false) { - let wh = format!( - "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", - &label.replace("*", "%").replace("'", "''") - ); - sqlb.and_where("result ? 'wm_labels'"); - sqlb.and_where(&wh); - } else { - let mut wh = format!("result->'wm_labels' ? "); - wh.push_str(&format!("'{}'", &label.replace("'", "''"))); - sqlb.and_where("result ? 'wm_labels'"); - sqlb.and_where(&wh); - } - } - - if let Some(worker) = &lq.worker { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job_completed.worker", worker.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job_completed.worker", "?".bind(worker)); - } - } - - if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { - sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)) - .and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); - } - - if let Some(p) = &lq.schedule_path { - sqlb.and_where_eq("trigger", "?".bind(p)); - sqlb.and_where_eq("trigger_kind", "'schedule'"); - } - - if let Some(ps) = &lq.script_path_start { - sqlb.and_where_like_left("runnable_path", ps); - } - if let Some(p) = &lq.script_path_exact { - sqlb.and_where_eq("runnable_path", "?".bind(p)); - } - if let Some(h) = &lq.script_hash { - sqlb.and_where_eq("runnable_id", "?".bind(h)); - } - if let Some(t) = &lq.tag { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job.tag", "?".bind(t)); - } - } - - if let Some(cb) = &lq.created_by { - sqlb.and_where_eq("created_by", "?".bind(cb)); - } - if let Some(r) = &lq.success { - if *r { - sqlb.and_where_eq("status", "'success'") - .or_where_eq("status", "'skipped'"); - } else { - sqlb.and_where_eq("status", "'failure'") - .or_where_eq("status", "'canceled'"); - } - } - if let Some(pj) = &lq.parent_job { - sqlb.and_where_eq("parent_job", "?".bind(pj)); - } - if let Some(dt) = &lq.started_before { - sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.started_after { - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.created_or_started_before { - sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.created_or_started_after { - let ts = dt.to_rfc3339(); - sqlb.and_where(format!( - "(created_at >= '{}' OR started_at >= '{}')", - ts.replace("'", "''"), - ts.replace("'", "''") - )); - } - - if let Some(dt) = &lq.created_before { - sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.created_after { - sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.created_or_started_after_completed_jobs { - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.completed_after { - sqlb.and_where_ge("completed_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.completed_before { - sqlb.and_where_le("completed_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(sk) = &lq.is_skipped { - if *sk { - sqlb.and_where_eq("status", "'skipped'"); - } else { - sqlb.and_where_ne("status", "'skipped'"); - } - } - if let Some(fs) = &lq.is_flow_step { - if *fs { - sqlb.and_where_is_not_null("flow_step_id"); - } else { - sqlb.and_where_is_null("flow_step_id"); - } - } - if let Some(fs) = &lq.has_null_parent { - if *fs { - sqlb.and_where_is_null("parent_job"); - } - } - if let Some(jk) = &lq.job_kinds { - sqlb.and_where_in( - "kind", - &jk.split(',').into_iter().map(quote).collect::>(), - ); - } - - if let Some(args) = &lq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); - } - - if let Some(result) = &lq.result { - sqlb.and_where("result @> ?".bind(&result.replace("'", "''"))); - } - - if lq.is_not_schedule.unwrap_or(false) { - sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); - } - - if let Some(tk) = &lq.trigger_kind { - sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); - } - - if let Some(tp) = &lq.trigger_path { - sqlb.and_where_eq("trigger", "?".bind(tp)); - } - - sqlb -} - -pub fn list_completed_jobs_query( - w_id: &str, - per_page: Option, - offset: usize, - lq: &ListCompletedQuery, - fields: &[&str], - join_outstanding_wait_times: bool, - tags: Option>, -) -> SqlBuilder { - let mut sqlb = SqlBuilder::select_from("v2_job_completed") - .fields(fields) - .order_by( - if lq.completed_before.is_some() || lq.completed_after.is_some() { - "v2_job_completed.completed_at" - } else { - "v2_job.created_at" - }, - lq.order_desc.unwrap_or(true), - ) - .offset(offset) - .clone(); - if let Some(per_page) = per_page { - sqlb.limit(per_page); - } - - if let Some(tags) = tags { - sqlb.and_where_in( - "v2_job.tag", - &tags.iter().map(|x| quote(x)).collect::>(), - ); - } - - filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times) -} -#[derive(Deserialize, Clone)] -pub struct ListCompletedQuery { - pub script_path_start: Option, - pub script_path_exact: Option, - pub script_hash: Option, - pub created_by: Option, - pub started_before: Option>, - pub started_after: Option>, - pub created_before: Option>, - pub created_after: Option>, - pub created_or_started_before: Option>, - pub created_or_started_after: Option>, - pub created_or_started_after_completed_jobs: Option>, - pub created_before_queue: Option>, - pub created_after_queue: Option>, - pub completed_after: Option>, - pub completed_before: Option>, - pub success: Option, - pub running: Option, - pub parent_job: Option, - pub order_desc: Option, - pub job_kinds: Option, - pub is_skipped: Option, - pub is_flow_step: Option, - pub suspended: Option, - pub schedule_path: Option, - // filter by matching a subset of the args using base64 encoded json subset - pub args: Option, - // filter by matching a subset of the result using base64 encoded json subset - pub result: Option, - pub tag: Option, - pub scheduled_for_before_now: Option, - pub all_workspaces: Option, - pub has_null_parent: Option, - pub label: Option, - pub is_not_schedule: Option, - pub concurrency_key: Option, - pub worker: Option, - pub allow_wildcards: Option, - pub trigger_kind: Option, - pub trigger_path: Option, - pub include_args: Option, -} - async fn list_completed_jobs( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/jobs_export.rs b/backend/windmill-api/src/jobs_export.rs index f1b83cba05..bb7e62cfa9 100644 --- a/backend/windmill-api/src/jobs_export.rs +++ b/backend/windmill-api/src/jobs_export.rs @@ -1,827 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use axum::{ - extract::{Path, Query}, - Extension, Json, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use uuid::Uuid; -use windmill_common::{ - db::UserDB, - error, - jobs::{JobKind, JobStatus, JobTriggerKind}, - scripts::ScriptLang, - utils::{paginate, paginate_without_limits, require_admin, Pagination}, -}; - -use crate::db::ApiAuthed; - -#[derive(Serialize, Deserialize)] -pub struct ExportableCompletedJob { - pub id: Uuid, - pub raw_code: Option, - pub raw_lock: Option, - pub raw_flow: Option>>, - pub tag: String, - pub created_at: chrono::DateTime, - pub created_by: String, - pub permissioned_as: String, - pub permissioned_as_email: String, - pub kind: JobKind, - pub runnable_id: Option, - pub runnable_path: Option, - pub parent_job: Option, - pub root_job: Option, - pub script_lang: Option, - pub script_entrypoint_override: Option, - pub flow_step: Option, - pub flow_step_id: Option, - pub flow_innermost_root_job: Option, - pub trigger: Option, - pub trigger_kind: Option, - pub same_worker: bool, - pub visible_to_owner: bool, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub cache_ttl: Option, - pub timeout: Option, - pub priority: Option, - pub preprocessed: Option, - pub args: Option>>, - pub labels: Option>, - pub pre_run_error: Option, - pub duration_ms: i64, - pub result: Option>>, - pub deleted: bool, - pub canceled_by: Option, - pub canceled_reason: Option, - pub flow_status: Option>>, - pub started_at: Option>, - pub memory_peak: Option, - pub status: JobStatus, - pub completed_at: chrono::DateTime, - pub worker: Option, - pub workflow_as_code_status: Option>>, - pub result_columns: Option>, - pub retries: Option>, - pub extras: Option>>, - pub logs: Option, - pub log_offset: Option, - pub log_file_index: Option>, -} - -#[derive(Serialize, Deserialize)] -pub struct ExportableQueuedJob { - // v2_job columns (excluding workspace_id) - pub id: Uuid, - pub raw_code: Option, - pub raw_lock: Option, - pub raw_flow: Option>>, - pub tag: String, - pub created_at: chrono::DateTime, - pub created_by: String, - pub permissioned_as: String, - pub permissioned_as_email: String, - pub kind: JobKind, - pub runnable_id: Option, - pub runnable_path: Option, - pub parent_job: Option, - pub root_job: Option, - pub script_lang: Option, - pub script_entrypoint_override: Option, - pub flow_step: Option, - pub flow_step_id: Option, - pub flow_innermost_root_job: Option, - pub trigger: Option, - pub trigger_kind: Option, - pub same_worker: bool, - pub visible_to_owner: bool, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub cache_ttl: Option, - pub timeout: Option, - pub priority: Option, - pub preprocessed: Option, - pub args: Option>>, - pub labels: Option>, - pub pre_run_error: Option, - - // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) - pub started_at: Option>, - pub scheduled_for: chrono::DateTime, - pub running: bool, - pub canceled_by: Option, - pub canceled_reason: Option, - pub suspend: Option, - pub suspend_until: Option>, - pub worker: Option, - pub extras: Option>>, - - // v2_job_runtime columns (excluding id) - pub ping: Option>, - pub memory_peak: Option, - - // v2_job_status columns (excluding id) - pub flow_status: Option>>, - pub flow_leaf_jobs: Option>>, - pub workflow_as_code_status: Option>>, - - // concurrency_key table - pub concurrency_key: Option, -} - -pub async fn export_completed_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> error::JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let (per_page, offset) = paginate(pagination); - let per_page = per_page as i64; - let offset = offset as i64; - - let mut tx = user_db.begin(&authed).await?; - - let jobs = sqlx::query_as!( - ExportableCompletedJob, - r#" - SELECT - v2_job.id, - v2_job.raw_code, - v2_job.raw_lock, - v2_job.raw_flow as "raw_flow: _", - v2_job.tag, - v2_job.created_at, - v2_job.created_by, - v2_job.permissioned_as, - v2_job.permissioned_as_email, - v2_job.kind as "kind: _", - v2_job.runnable_id, - v2_job.runnable_path, - v2_job.parent_job, - v2_job.root_job, - v2_job.script_lang as "script_lang: _", - v2_job.script_entrypoint_override, - v2_job.flow_step, - v2_job.flow_step_id, - v2_job.flow_innermost_root_job, - v2_job.trigger, - v2_job.trigger_kind as "trigger_kind: _", - v2_job.same_worker, - v2_job.visible_to_owner, - v2_job.concurrent_limit, - v2_job.concurrency_time_window_s, - v2_job.cache_ttl, - v2_job.timeout, - v2_job.priority, - v2_job.preprocessed, - v2_job.args as "args: _", - v2_job.labels, - v2_job.pre_run_error, - v2_job_completed.duration_ms, - v2_job_completed.result as "result: _", - v2_job_completed.deleted, - v2_job_completed.canceled_by, - v2_job_completed.canceled_reason, - v2_job_completed.flow_status as "flow_status: _", - v2_job_completed.started_at, - v2_job_completed.memory_peak, - v2_job_completed.status as "status: _", - v2_job_completed.completed_at, - v2_job_completed.worker, - v2_job_completed.workflow_as_code_status as "workflow_as_code_status: _", - v2_job_completed.result_columns, - v2_job_completed.retries, - v2_job_completed.extras as "extras: _", - job_logs.logs AS logs, - job_logs.log_offset, - job_logs.log_file_index - FROM v2_job_completed - INNER JOIN v2_job ON v2_job.id = v2_job_completed.id - LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id - LEFT JOIN job_logs ON job_logs.job_id = v2_job.id - WHERE v2_job_completed.workspace_id = $1 - ORDER BY v2_job.created_at DESC - LIMIT $2 - OFFSET $3 - "#, - w_id, - per_page, - offset - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(jobs)) -} - -pub async fn export_queued_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> error::JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let (per_page, offset) = paginate_without_limits(pagination); - let per_page = per_page as i64; - let offset = offset as i64; - - let mut tx = user_db.begin(&authed).await?; - - let jobs = sqlx::query_as!( - ExportableQueuedJob, - r#" - SELECT - v2_job.id, - v2_job.raw_code, - v2_job.raw_lock, - v2_job.raw_flow as "raw_flow: _", - v2_job.tag, - v2_job.created_at, - v2_job.created_by, - v2_job.permissioned_as, - v2_job.permissioned_as_email, - v2_job.kind as "kind: _", - v2_job.runnable_id, - v2_job.runnable_path, - v2_job.parent_job, - v2_job.root_job, - v2_job.script_lang as "script_lang: _", - v2_job.script_entrypoint_override, - v2_job.flow_step, - v2_job.flow_step_id, - v2_job.flow_innermost_root_job, - v2_job."trigger", - v2_job.trigger_kind as "trigger_kind: _", - v2_job.same_worker, - v2_job.visible_to_owner, - v2_job.concurrent_limit, - v2_job.concurrency_time_window_s, - v2_job.cache_ttl, - v2_job.timeout, - v2_job.priority, - v2_job.preprocessed, - v2_job.args as "args: _", - v2_job.labels, - v2_job.pre_run_error, - - v2_job_queue.started_at, - v2_job_queue.scheduled_for, - v2_job_queue.running, - v2_job_queue.canceled_by, - v2_job_queue.canceled_reason, - v2_job_queue.suspend, - v2_job_queue.suspend_until, - v2_job_queue.worker, - v2_job_queue.extras as "extras: _", - - v2_job_runtime.ping, - v2_job_runtime.memory_peak, - - v2_job_status.flow_status as "flow_status: _", - v2_job_status.flow_leaf_jobs as "flow_leaf_jobs: _", - v2_job_status.workflow_as_code_status as "workflow_as_code_status: _", - - concurrency_key.key as "concurrency_key?" - FROM v2_job_queue - INNER JOIN v2_job ON v2_job.id = v2_job_queue.id - LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id - LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id - LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id - WHERE v2_job_queue.workspace_id = $1 - AND v2_job_queue.running = false - AND v2_job.parent_job IS NULL - AND v2_job.trigger_kind IS DISTINCT FROM 'schedule' - ORDER BY v2_job.created_at DESC - LIMIT $2 - OFFSET $3 - "#, - w_id, - per_page, - offset - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(jobs)) -} - -pub async fn import_completed_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(jobs): Json>, -) -> error::Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - for job in jobs { - // Insert into v2_job table - sqlx::query!( - r#" - INSERT INTO v2_job ( - id, - raw_code, - raw_lock, - raw_flow, - tag, - workspace_id, - created_at, - created_by, - permissioned_as, - permissioned_as_email, - kind, - runnable_id, - runnable_path, - parent_job, - root_job, - script_lang, - script_entrypoint_override, - flow_step, - flow_step_id, - flow_innermost_root_job, - trigger, - trigger_kind, - same_worker, - visible_to_owner, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - timeout, - priority, - preprocessed, - args, - labels, - pre_run_error - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.raw_code, - job.raw_lock, - job.raw_flow as _, - &job.tag, - &w_id, - job.created_at, - &job.created_by, - &job.permissioned_as, - job.permissioned_as_email, - job.kind as _, - job.runnable_id, - job.runnable_path, - job.parent_job, - job.root_job, - job.script_lang as _, - job.script_entrypoint_override, - job.flow_step, - job.flow_step_id, - job.flow_innermost_root_job, - job.trigger, - job.trigger_kind as _, - job.same_worker, - job.visible_to_owner, - job.concurrent_limit, - job.concurrency_time_window_s, - job.cache_ttl, - job.timeout, - job.priority, - job.preprocessed, - job.args as _, - job.labels as _, - job.pre_run_error - ) - .execute(&mut *tx) - .await?; - - // Insert into v2_job_completed table - sqlx::query!( - r#" - INSERT INTO v2_job_completed ( - id, - workspace_id, - started_at, - completed_at, - duration_ms, - result, - deleted, - canceled_by, - canceled_reason, - flow_status, - memory_peak, - status, - worker, - workflow_as_code_status, - result_columns, - retries, - extras - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - &w_id, - job.started_at, - job.completed_at, - job.duration_ms, - job.result as _, - job.deleted, - job.canceled_by, - job.canceled_reason, - job.flow_status as _, - job.memory_peak, - job.status as _, - job.worker, - job.workflow_as_code_status as _, - job.result_columns as _, - job.retries as _, - job.extras as _ - ) - .execute(&mut *tx) - .await?; - - // Insert into job_logs table - if let Some(logs) = &job.logs { - sqlx::query!( - r#" - INSERT INTO job_logs ( - job_id, - workspace_id, - logs, - log_offset, - log_file_index - ) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (job_id) DO NOTHING - "#, - job.id, - &w_id, - logs, - job.log_offset, - job.log_file_index as _ - ) - .execute(&mut *tx) - .await?; - } - } - - tx.commit().await?; - Ok(format!("Successfully imported jobs")) -} - -pub async fn import_queued_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(jobs): Json>, -) -> error::Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - for job in jobs { - // Insert into v2_job table - // Insert into v2_job table, fields and order adapted to match ExportableQueuedJob - sqlx::query!( - r#" - INSERT INTO v2_job ( - id, - raw_code, - raw_lock, - raw_flow, - tag, - workspace_id, - created_at, - created_by, - permissioned_as, - permissioned_as_email, - kind, - runnable_id, - runnable_path, - parent_job, - root_job, - script_lang, - script_entrypoint_override, - flow_step, - flow_step_id, - flow_innermost_root_job, - trigger, - trigger_kind, - same_worker, - visible_to_owner, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - timeout, - priority, - preprocessed, - args, - labels, - pre_run_error - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, - $31, $32, $33 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.raw_code, - job.raw_lock, - job.raw_flow as _, - &job.tag, - &w_id, - job.created_at, - &job.created_by, - &job.permissioned_as, - &job.permissioned_as_email, - job.kind as _, - job.runnable_id, - job.runnable_path, - job.parent_job, - job.root_job, - job.script_lang as _, - job.script_entrypoint_override, - job.flow_step, - job.flow_step_id, - job.flow_innermost_root_job, - job.trigger, - job.trigger_kind as _, - job.same_worker, - job.visible_to_owner, - job.concurrent_limit, - job.concurrency_time_window_s, - job.cache_ttl, - job.timeout, - job.priority, - job.preprocessed, - job.args as _, - job.labels as _, - job.pre_run_error, - ) - .execute(&mut *tx) - .await?; - - // Insert into v2_job_queue table - sqlx::query!( - r#" - INSERT INTO v2_job_queue ( - id, - workspace_id, - started_at, - scheduled_for, - running, - canceled_by, - canceled_reason, - suspend, - suspend_until, - worker, - extras, - tag, - priority - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - &w_id, - job.started_at, - job.scheduled_for, - job.running, - job.canceled_by, - job.canceled_reason, - job.suspend, - job.suspend_until, - job.worker, - job.extras as _, - &job.tag, - job.priority - ) - .execute(&mut *tx) - .await?; - - // Insert into v2_job_runtime if any present - if job.ping.is_some() || job.memory_peak.is_some() { - sqlx::query!( - r#" - INSERT INTO v2_job_runtime ( - id, ping, memory_peak - ) VALUES ( - $1, $2, $3 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.ping, - job.memory_peak - ) - .execute(&mut *tx) - .await?; - } - - // Insert into v2_job_status if present - if job.flow_status.is_some() - || job.flow_leaf_jobs.is_some() - || job.workflow_as_code_status.is_some() - { - sqlx::query!( - r#" - INSERT INTO v2_job_status ( - id, flow_status, flow_leaf_jobs, workflow_as_code_status - ) VALUES ( - $1, $2, $3, $4 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.flow_status as _, - job.flow_leaf_jobs as _, - job.workflow_as_code_status as _ - ) - .execute(&mut *tx) - .await?; - } - - // Insert concurrency_key if present - if let Some(ref concurrency_key) = job.concurrency_key { - sqlx::query!( - r#" - WITH inserted_concurrency_counter AS ( - INSERT INTO concurrency_counter (concurrency_id, job_uuids) - VALUES ($1, '{}'::jsonb) - ON CONFLICT DO NOTHING - ) - INSERT INTO concurrency_key(key, job_id) - VALUES ($1, $2) - ON CONFLICT (job_id) DO NOTHING - "#, - concurrency_key, - job.id - ) - .execute(&mut *tx) - .await?; - } - } - - tx.commit().await?; - Ok(format!("Successfully imported jobs")) -} - -pub async fn delete_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(job_ids): Json>, -) -> error::Result { - require_admin(authed.is_admin, &authed.username)?; - - if job_ids.is_empty() { - return Ok("No jobs to delete".to_string()); - } - - let mut tx = user_db.begin(&authed).await?; - - // Delete from all job-related tables - // Order matters due to foreign key constraints - - // 1. Delete job logs - let logs_deleted = sqlx::query!( - "DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 2. Delete job permissions - let perms_deleted = sqlx::query!( - "DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 3. Delete job stats - let stats_deleted = sqlx::query!( - "DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 4. Delete resume jobs - let resume_deleted = sqlx::query!("DELETE FROM resume_job WHERE job = ANY($1)", &job_ids) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 5. Delete from v2_job_runtime - let runtime_deleted = sqlx::query!("DELETE FROM v2_job_runtime WHERE id = ANY($1)", &job_ids) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 6. Delete from v2_job_status - let status_deleted = sqlx::query!("DELETE FROM v2_job_status WHERE id = ANY($1)", &job_ids) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 7. Delete from concurrency_key - let concurrency_key_deleted = sqlx::query!( - "DELETE FROM concurrency_key WHERE job_id = ANY($1)", - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 8. Delete from v2_job_queue (for queued jobs) - let queue_deleted = sqlx::query!( - "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 9. Delete from v2_job_completed (for completed jobs) - let completed_deleted = sqlx::query!( - "DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 10. Delete zombie job counter - let zombie_deleted = sqlx::query!( - "DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 11. Finally, delete from v2_job (main table) - let jobs_deleted = sqlx::query!( - "DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - tx.commit().await?; - - let total_rows_deleted = logs_deleted - + perms_deleted - + stats_deleted - + resume_deleted - + runtime_deleted - + status_deleted - + concurrency_key_deleted - + queue_deleted - + completed_deleted - + zombie_deleted - + jobs_deleted; - - tracing::info!( - "Successfully deleted {} jobs ({} total rows across all tables) from workspace {}", - job_ids.len(), - total_rows_deleted, - w_id - ); - - Ok(format!( - "Successfully deleted {} jobs ({} total rows across all tables)", - jobs_deleted, total_rows_deleted - )) -} +pub use windmill_api_jobs::jobs_export::*; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index a78b6fa2a3..369d64e0e7 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -7,16 +7,14 @@ */ use crate::db::ApiAuthed; -#[cfg(feature = "enterprise")] -use crate::ee_oss::ExternalJwks; #[cfg(feature = "embedding")] use crate::embeddings::load_embeddings_db; #[cfg(feature = "oauth2")] -use crate::oauth2_oss::AllClients; -#[cfg(feature = "oauth2")] use crate::oauth2_oss::SlackVerifier; #[cfg(feature = "smtp")] use crate::smtp_server_oss::SmtpServer; +#[cfg(feature = "enterprise")] +use windmill_api_auth::ee_oss::ExternalJwks; #[cfg(feature = "mcp")] use crate::mcp::{extract_and_store_workspace_id, setup_mcp_server}; @@ -41,8 +39,6 @@ use axum::response::Response; use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router}; use db::DB; use reqwest::Client; -#[cfg(feature = "oauth2")] -use std::collections::HashMap; use tokio::task::JoinHandle; use windmill_common::global_settings::load_value_from_global_settings; use windmill_common::global_settings::EMAIL_DOMAIN_SETTING; @@ -60,9 +56,11 @@ use tower_http::{ }; use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; +#[allow(unused_imports)] +pub(crate) use windmill_common::BASE_URL; use windmill_common::{ utils::{configure_client, GIT_VERSION}, - BASE_URL, INSTANCE_NAME, + INSTANCE_NAME, }; use crate::scim_oss::has_scim_token; @@ -150,7 +148,6 @@ mod schedule; #[cfg(feature = "private")] pub mod scim_ee; mod scim_oss; -mod scopes; mod scripts; mod secret_backend_ext; mod service_logs; @@ -187,7 +184,6 @@ mod users; pub mod users_ee; mod users_oss; mod utils; -pub mod var_resource_cache; mod variables; pub mod webhook_util; mod workers; @@ -195,8 +191,6 @@ mod workspaces; #[cfg(feature = "private")] pub mod workspaces_ee; mod workspaces_export; -mod workspaces_extra; -mod workspaces_oss; #[cfg(feature = "mcp")] mod mcp; @@ -216,9 +210,7 @@ lazy_static::lazy_static! { pub static ref SAML_METADATA: Arc>> = Arc::new(RwLock::new(None)); - pub static ref COOKIE_DOMAIN: Option = std::env::var("COOKIE_DOMAIN").ok(); - - pub static ref IS_SECURE: Arc> = Arc::new(RwLock::new(false)); + // COOKIE_DOMAIN and IS_SECURE are now in windmill_common::utils pub static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .user_agent("windmill/beta") @@ -230,15 +222,13 @@ lazy_static::lazy_static! { } +pub use windmill_common::utils::{COOKIE_DOMAIN, IS_SECURE}; + +#[cfg(feature = "oauth2")] +pub use windmill_oauth::OAUTH_CLIENTS; + #[cfg(feature = "oauth2")] lazy_static::lazy_static! { - pub static ref OAUTH_CLIENTS: Arc> = Arc::new(RwLock::new(AllClients { - logins: HashMap::new(), - connects: HashMap::new(), - slack: None - })); - - pub static ref SLACK_SIGNING_SECRET: Option = std::env::var("SLACK_SIGNING_SECRET") .ok() .map(|x| SlackVerifier::new(x).unwrap()); diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 5b3eb0e656..e7fa3eb13f 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -18,13 +18,6 @@ pub async fn custom_migrations(migrator: &mut CustomMigrator, db: &DB) -> Result tracing::error!("Could not apply flow versioning fix migration: {err:#}"); } - let db2 = db.clone(); - let _ = tokio::task::spawn(async move { - if let Err(err) = fix_job_completed_index(&db2).await { - tracing::error!("Could not apply job completed index fix migration: {err:#}"); - } - }); - Ok(()) } @@ -74,377 +67,3 @@ async fn fix_flow_versioning_migration( } Ok(()) } - -async fn has_done_migration(db: &DB, migration_job_name: &str) -> bool { - sqlx::query_scalar!( - "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)", - migration_job_name - ) - .fetch_one(db) - .await - .ok() - .flatten() - .unwrap_or(false) -} - -use sqlx::Pool; - -macro_rules! run_windmill_migration { - ($migration_job_name:expr, $db:expr, |$tx:ident| $code:block) => { - { - let migration_job_name = $migration_job_name; - let db: &Pool = $db; - - let has_done = has_done_migration(db, migration_job_name).await; - if !has_done { - tracing::info!("Applying {migration_job_name} migration"); - let mut $tx = db.begin().await?; - let mut r = false; - while !r { - r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)") - .fetch_one(&mut *$tx) - .await - .map_err(|e| { - tracing::error!("Error acquiring {migration_job_name} lock: {e:#}"); - sqlx::migrate::MigrateError::Execute(e) - })? - .unwrap_or(false); - - if !r { - tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)"); - drop($tx); - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - $tx = db.begin().await?; - } - } - tracing::info!("acquired lock for {migration_job_name}"); - - let has_done = has_done_migration(db, migration_job_name).await; - - if !has_done { - - $code - - sqlx::query!( - "INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING", - migration_job_name - ) - .execute(&mut *$tx) - .await?; - tracing::info!("Finished applying {migration_job_name} migration"); - } else { - tracing::debug!("migration {migration_job_name} already done"); - } - - let _ = sqlx::query("SELECT pg_advisory_unlock(4242)") - .execute(&mut *$tx) - .await?; - $tx.commit().await?; - tracing::info!("released lock for {migration_job_name}"); - } else { - tracing::debug!("migration {migration_job_name} already done"); - - } - } - }; -} - -async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { - // let has_done_migration = sqlx::query_scalar!( - // "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_job_completed_index')" - // ) - // .fetch_one(db) - // .await? - // .unwrap_or(false); - // if !has_done_migration { - // tracing::info!("Applying fix_job_completed_index migration"); - // let mut tx = db.begin().await?; - // let mut r = false; - // while !r { - // r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)") - // .fetch_one(&mut *tx) - // .await - // .map_err(|e| { - // tracing::error!("Error acquiring fix_job_completed_index lock: {e:#}"); - // sqlx::migrate::MigrateError::Execute(e) - // })? - // .unwrap_or(false); - // if !r { - // tracing::info!("PG fix_job_completed_index_migration lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)"); - // tokio::time::sleep(std::time::Duration::from_secs(5)).await; - // } - // } - // // sqlx::query( - // // "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_created_at_new ON completed_job (workspace_id, job_kind, is_skipped, is_flow_step, created_at DESC, started_at DESC)" - // // ).execute(db).await?; - - // sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at") - // .execute(db) - // .await?; - - // sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('fix_job_completed_index') ON CONFLICT DO NOTHING") - // .execute(&mut *tx) - // .await?; - // let _ = sqlx::query("SELECT pg_advisory_unlock(4242)") - // .execute(&mut *tx) - // .await?; - // tx.commit().await?; - // } - - run_windmill_migration!("fix_job_completed_index_2", &db, |tx| { - // sqlx::query( - // "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_created_at_new_2 ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, created_at DESC)" - // ).execute(db).await?; - - // sqlx::query( - // "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_started_at_new ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, started_at DESC)" - // ).execute(db).await?; - - sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at") - .execute(db) - .await?; - - sqlx::query( - "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new", - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("fix_job_completed_index_3", &db, |tx| { - sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_job_on_schedule_path") - .execute(db) - .await?; - - sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS concurrency_limit_stats_queue") - .execute(db) - .await?; - - sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index") - .execute(db) - .await?; - - sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_on_created") - .execute(db) - .await?; - }); - - run_windmill_migration!("fix_job_index_1_II", &db, |tx| { - let migration_job_name = "fix_job_index_1_II"; - let mut i = 1; - tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_3 ON v2_job (workspace_id, created_at DESC)") - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_8 ON v2_job (workspace_id, created_at DESC) where kind in ('deploymentcallback') AND parent_job IS NULL") - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_9 ON v2_job (workspace_id, created_at DESC) where kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL") - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("create index concurrently if not exists ix_job_workspace_id_created_at_new_5 ON v2_job (workspace_id, created_at DESC) where kind in ('preview', 'flowpreview') AND parent_job IS NULL") - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path_2") - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query( - "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new_2", - ) - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query( - "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new", - ) - .execute(db) - .await?; - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path") - .execute(db) - .await?; - }); - - run_windmill_migration!("fix_labeled_jobs_index", &db, |tx| { - tracing::info!("Special migration to add index concurrently on job labels 2"); - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS labeled_jobs_on_jobs") - .execute(db) - .await?; - sqlx::query!( - "CREATE INDEX CONCURRENTLY labeled_jobs_on_jobs ON v2_job_completed USING GIN ((result -> 'wm_labels')) WHERE result ? 'wm_labels'" - ).execute(db).await?; - }); - - run_windmill_migration!("v2_labeled_jobs_index", &db, |tx| { - tracing::info!("Special migration to add index concurrently on job labels"); - sqlx::query!( - "CREATE INDEX CONCURRENTLY ix_v2_job_labels ON v2_job - USING GIN (labels) - WHERE labels IS NOT NULL" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("v2_jobs_rls", &db, |tx| { - sqlx::query!("ALTER TABLE v2_job ENABLE ROW LEVEL SECURITY") - .execute(db) - .await?; - }); - - run_windmill_migration!("v2_improve_v2_job_indices_ii", &db, |tx| { - sqlx::query!("create index concurrently if not exists ix_v2_job_workspace_id_created_at ON v2_job (workspace_id, created_at DESC) where kind in ('script', 'flow', 'singlestepflow') AND parent_job IS NULL") - .execute(db) - .await?; - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_job_workspace_id_created_at_new_6") - .execute(db) - .await?; - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_job_workspace_id_created_at_new_7") - .execute(db) - .await?; - }); - - run_windmill_migration!("v2_improve_v2_queued_jobs_indices", &db, |tx| { - sqlx::query!("CREATE INDEX CONCURRENTLY IF NOT EXISTS queue_sort_v2 ON v2_job_queue (priority DESC NULLS LAST, scheduled_for, tag) WHERE running = false") - .execute(db) - .await?; - - // sqlx::query!("CREATE INDEX CONCURRENTLY queue_sort_2_v2 ON v2_job_queue (tag, priority DESC NULLS LAST, scheduled_for) WHERE running = false") - // .execute(db) - // .await?; - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort") - .execute(db) - .await?; - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS queue_sort_2") - .execute(db) - .await?; - }); - - run_windmill_migration!("audit_timestamps", db, |tx| { - sqlx::query!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_audit_timestamps ON audit (timestamp DESC)" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("job_completed_completed_at", db, |tx| { - sqlx::query!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_job_completed_completed_at ON v2_job_completed (completed_at DESC)" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("alerts_by_workspace", db, |tx| { - sqlx::query!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_by_workspace ON alerts (workspace_id);" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("remove_redundant_log_file_index", db, |tx| { - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS log_file_hostname_log_ts_idx") - .execute(db) - .await?; - }); - - run_windmill_migration!("v2_job_queue_suspend", db, |tx| { - sqlx::query!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS v2_job_queue_suspend ON v2_job_queue (workspace_id, suspend) WHERE suspend > 0;" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("audit_recent_login_activities", db, |tx| { - sqlx::query!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_recent_login_activities -ON audit (timestamp, username) -WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("v2_script_lock_index", db, |tx| { - sqlx::query!( - "CREATE INDEX CONCURRENTLY IF NOT EXISTS script_not_archived ON script (workspace_id, path, created_at DESC) where archived = false;" - ) - .execute(db) - .await?; - }); - - run_windmill_migration!("v2_job_completed_completed_at_9", db, |tx| { - let migration_job_name = "v2_job_completed_completed_at"; - let mut i = 1; - tracing::info!("step {i} of {migration_job_name} migration"); - sqlx::query!("create index concurrently if not exists ix_job_workspace_id_completed_at_all ON v2_job_completed (workspace_id, completed_at DESC)") - .execute(db) - .await?; - i += 1; - - sqlx::query!("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_job_v2_job_root_by_path_2 ON v2_job (workspace_id, runnable_path) WHERE parent_job IS NULL;") - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("create index concurrently if not exists ix_job_root_job_index_by_path_2 ON v2_job (workspace_id, runnable_path, created_at desc) WHERE parent_job IS NULL") - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!( - "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new_2" - ) - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_job_created_at") - .execute(db) - .await?; - - i += 1; - - sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS ix_v2_job_root_by_path") - .execute(db) - .await?; - - i += 1; - tracing::info!("step {i} of {migration_job_name} migration"); - }); - - Ok(()) -} diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 570b4c0de7..90419aed4d 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -11,7 +11,7 @@ use windmill_mcp::common::transform::apply_key_transformation; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpAuth, McpBackend}; +use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend}; use crate::db::ApiAuthed; use crate::jobs::{ @@ -39,36 +39,7 @@ use axum::{ }; use windmill_common::error::JsonResult; -/// Implement McpAuth for ApiAuthed -impl McpAuth for ApiAuthed { - fn username(&self) -> &str { - &self.username - } - - fn email(&self) -> &str { - &self.email - } - - fn is_admin(&self) -> bool { - self.is_admin - } - - fn is_operator(&self) -> bool { - self.is_operator - } - - fn groups(&self) -> &[String] { - &self.groups - } - - fn folders(&self) -> &[(String, bool, bool)] { - &self.folders - } - - fn scopes(&self) -> Option<&[String]> { - self.scopes.as_deref() - } -} +// McpAuth impl for ApiAuthed is in windmill-api-auth (same crate as the type) /// Windmill's MCP backend implementation #[derive(Clone)] diff --git a/backend/windmill-api/src/native_triggers/mod.rs b/backend/windmill-api/src/native_triggers/mod.rs index 8c0b829478..95b4535217 100644 --- a/backend/windmill-api/src/native_triggers/mod.rs +++ b/backend/windmill-api/src/native_triggers/mod.rs @@ -1,1053 +1 @@ -//! Native Triggers Module -//! -//! This module provides integration with external services (like Nextcloud) that can -//! trigger Windmill scripts/flows via webhooks. -//! -//! ## Adding a New Native Trigger Service -//! -//! When adding a new service (e.g., "NewService"), you need to update the following locations: -//! -//! ### 1. This file (mod.rs): -//! - Add `pub mod newservice;` under the `#[cfg(feature = "native_trigger")]` block -//! - Add `NewService` variant to `ServiceName` enum -//! - Update `ServiceName::as_str()` - add match arm returning `"newservice"` -//! - Update `TryFrom for ServiceName` - add match arm for `"newservice"` -//! - Update `ServiceName::as_trigger_kind()` - add match arm (requires TriggerKind::NewService in windmill_common) -//! - Update `ServiceName::as_job_trigger_kind()` - add match arm (requires JobTriggerKind::NewService in windmill_common) -//! - Update `ServiceName::fmt()` (Display impl) - add match arm -//! -//! ### 2. sync.rs: -//! - Add `sync_service!()` macro call in `sync_all_triggers()` -//! -//! ### 3. handler.rs: -//! - Add `.nest("/newservice", service_routes(NewServiceHandler))` in `generate_native_trigger_routers()` -//! -//! ### 4. Database migration: -//! - Add `'newservice'` to the `native_trigger_service` enum type -//! -//! ### 5. windmill_common (if needed): -//! - Add `NewService` variant to `TriggerKind` enum -//! - Add `'newservice'` to `job_trigger_kind` enum type in migration -//! -//! The generic code (trait definitions, route handlers, database operations) does NOT -//! need modification when adding new services. - -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use http::StatusCode; -use itertools::Itertools; -use reqwest::{Client, Method}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::json; -use serde_json::value::RawValue; -use sqlx::{FromRow, PgConnection, Postgres}; -use std::{collections::HashMap, fmt::Debug}; -use strum::{EnumIter, IntoEnumIterator}; -use tokio::task; -use windmill_common::{ - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - variables::{build_crypt, decrypt, encrypt}, - DB, -}; -use windmill_queue::PushArgsOwned; - -#[cfg(feature = "native_trigger")] -use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; - -use crate::db::ApiAuthed; -pub mod handler; -pub mod sync; -pub mod workspace_integrations; - -// Service modules - add new services here: -#[cfg(feature = "native_trigger")] -pub mod nextcloud; -// #[cfg(feature = "native_trigger")] -// pub mod newservice; - -/// Enum of all supported native trigger services. -/// When adding a new service, add a variant here (e.g., `NewService`). -#[derive(EnumIter, sqlx::Type, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[sqlx(type_name = "native_trigger_service", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum ServiceName { - Nextcloud, - // Add new services here: - // NewService, -} - -impl TryFrom for ServiceName { - type Error = Error; - fn try_from(value: String) -> std::result::Result { - // Add new service match arms here: - let service = match value.as_str() { - "nextcloud" => ServiceName::Nextcloud, - // "newservice" => ServiceName::NewService, - _ => { - return Err(anyhow::anyhow!( - "Unknown service, currently supported services are: [{}]", - ServiceName::iter().join(",") - ) - .into()) - } - }; - - Ok(service) - } -} - -impl ServiceName { - /// Returns the lowercase string identifier for this service. - /// Add new service match arms here. - pub fn as_str(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "nextcloud", - // ServiceName::NewService => "newservice", - } - } - - /// Returns the corresponding TriggerKind for this service. - /// Requires adding the variant to TriggerKind in windmill_common. - pub fn as_trigger_kind(&self) -> TriggerKind { - match self { - ServiceName::Nextcloud => TriggerKind::Nextcloud, - // ServiceName::NewService => TriggerKind::NewService, - } - } - - /// Returns the corresponding JobTriggerKind for this service. - /// Requires adding the variant to JobTriggerKind in windmill_common. - pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind { - match self { - ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud, - // ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService, - } - } - - /// Returns the OAuth token endpoint path for this service. - /// Used for building OAuth clients dynamically. - pub fn token_endpoint(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", - // ServiceName::NewService => "/oauth/token", - } - } - - /// Returns the OAuth authorization endpoint path for this service. - /// Used for building OAuth authorization URLs. - pub fn auth_endpoint(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "/apps/oauth2/authorize", - // ServiceName::NewService => "/oauth/authorize", - } - } -} - -impl std::fmt::Display for ServiceName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct NativeTrigger { - pub external_id: String, - pub workspace_id: String, - pub service_name: ServiceName, - pub script_path: String, - pub is_flow: bool, - pub webhook_token_prefix: String, - pub service_config: Option, - pub error: Option, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NativeTriggerConfig { - pub script_path: String, - pub is_flow: bool, - pub webhook_token: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct NativeTriggerData { - pub script_path: String, - pub is_flow: bool, - pub service_config: C, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct WorkspaceIntegration { - pub workspace_id: String, - pub service_name: ServiceName, - pub oauth_data: serde_json::Value, - pub created_at: DateTime, - pub updated_at: DateTime, - pub created_by: String, -} - -#[async_trait] -pub trait External: Send + Sync + 'static { - type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - type TriggerData: Debug + Serialize + Send + Sync; - type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; - type CreateResponse: DeserializeOwned + Send + Sync; - - const SUPPORT_WEBHOOK: bool; - const SERVICE_NAME: ServiceName; - const DISPLAY_NAME: &'static str; - const TOKEN_ENDPOINT: &'static str; - const REFRESH_ENDPOINT: &'static str; - - async fn create( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn update( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()>; - - async fn get( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn delete( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()>; - - #[allow(unused)] - async fn exists( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn list_all( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - db: &DB, - tx: &mut PgConnection, - ) -> Result>; - - async fn prepare_webhook( - &self, - _db: &DB, - _w_id: &str, - _header: HashMap, - _body: String, - _script_path: &str, - _is_flow: bool, - ) -> Result { - Ok(PushArgsOwned { extra: None, args: HashMap::new() }) - } - - fn external_id_and_metadata_from_response( - &self, - resp: &Self::CreateResponse, - ) -> (String, Option); - - fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String; - - /// Extracts the service-specific config from trigger data (from external service). - /// Used for comparison during sync to detect config drift. - /// Default implementation converts the trigger data to a JSON value - /// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method. - fn extract_service_config_from_trigger_data( - &self, - data: &Self::TriggerData, - ) -> Result { - serde_json::to_value(data).map_err(|e| { - Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e)) - }) - } - - fn additional_routes(&self) -> axum::Router { - axum::Router::new() - } - - async fn http_client_request( - &self, - url: &str, - method: Method, - workspace_id: &str, - tx: &mut PgConnection, - db: &DB, - headers: Option>, - body: Option<&B>, - ) -> Result { - let oauth_config: OAuthConfig = - decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?; - - let result = make_http_request( - url, - method.clone(), - headers.clone(), - body.as_ref(), - &oauth_config.access_token, - ) - .await; - - match result { - Ok(response) => Ok(response), - Err(err) - if err.status() == Some(StatusCode::UNAUTHORIZED) - || err.status() == Some(StatusCode::FORBIDDEN) => - { - tracing::info!( - "HTTP auth error ({}), attempting token refresh", - err.status().unwrap() - ); - - let refreshed_oauth_config = - refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?; - - task::spawn({ - let db_clone = db.clone(); - let workspace_id_clone = workspace_id.to_string(); - let refreshed_json = oauth_config_to_json(&refreshed_oauth_config); - async move { - update_workspace_integration_tokens_helper( - db_clone, - workspace_id_clone, - Self::SERVICE_NAME, - refreshed_json, - ) - .await; - } - }); - - let response = make_http_request( - url, - method, - headers, - body.as_ref(), - &refreshed_oauth_config.access_token, - ) - .await - .map_err(to_anyhow)?; - Ok(response) - } - Err(e) => Err(to_anyhow(e).into()), - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct OAuthConfig { - pub base_url: String, - pub access_token: String, - pub refresh_token: Option, - pub client_id: String, - pub client_secret: String, -} - -pub async fn make_http_request( - url: &str, - method: Method, - headers: Option>, - body: Option<&B>, - access_token: &str, -) -> std::result::Result { - let client = Client::new(); - let mut request = client.request(method, url); - - request = request - .header("Accept", "application/json") - .header("Authorization", format!("Bearer {}", access_token)); - - if body.is_some() { - request = request.header("Content-Type", "application/json"); - } - - if let Some(custom_headers) = headers { - for (key, value) in custom_headers { - request = request.header(key, value); - } - } - - if let Some(body_content) = body { - request = request.json(body_content); - } - - let response = request.send().await?.error_for_status()?; - - let response_json = response.json().await?; - - Ok(response_json) -} - -pub async fn decrypt_oauth_data< - 'c, - E: sqlx::Executor<'c, Database = Postgres>, - T: DeserializeOwned, ->( - tx: E, - db: &DB, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let integration = get_workspace_integration(tx, workspace_id, service_name).await?; - - let mc = build_crypt(db, workspace_id).await?; - let mut oauth_data: serde_json::Value = integration.oauth_data; - - if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) { - let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string()) - .map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?; - oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token); - } - - if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str()) - { - let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string()) - .map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?; - oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token); - } - - serde_json::from_value(oauth_data) - .map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e))) -} - -#[allow(unused)] -pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result { - let base_url = oauth_data - .get("base_url") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))? - .to_string(); - - let access_token = oauth_data - .get("access_token") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))? - .to_string(); - - let refresh_token = oauth_data - .get("refresh_token") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let client_id = oauth_data - .get("client_id") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))? - .to_string(); - - let client_secret = oauth_data - .get("client_secret") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))? - .to_string(); - - Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret }) -} - -#[inline] -pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value { - let mut json = json!({ - "base_url": config.base_url, - "access_token": config.access_token, - "client_id": config.client_id, - "client_secret": config.client_secret, - }); - - if let Some(refresh_token) = &config.refresh_token { - json["refresh_token"] = serde_json::Value::String(refresh_token.clone()); - } - - json -} - -/// Token refresh response -#[cfg(feature = "native_trigger")] -#[derive(Debug, Deserialize)] -struct RefreshTokenResponse { - access_token: String, - refresh_token: Option, -} - -/// Refresh OAuth tokens using windmill-oauth. -#[cfg(feature = "native_trigger")] -pub async fn refresh_oauth_tokens( - oauth_config: &OAuthConfig, - refresh_endpoint: &str, -) -> Result { - let refresh_token_str = oauth_config - .refresh_token - .as_ref() - .ok_or_else(|| Error::InternalErr("No refresh token available".to_string()))?; - - // Build OAuth client for token refresh - // Auth URL is not used for refresh, but required by the client constructor - let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url)) - .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; - let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint)) - .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; - - let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url); - client.set_client_secret(oauth_config.client_secret.clone()); - - let token_response: RefreshTokenResponse = client - .exchange_refresh_token(&RefreshToken::from(refresh_token_str.as_str())) - .with_client(&*OAUTH_HTTP_CLIENT) - .execute() - .await - .map_err(|e| Error::InternalErr(format!("Failed to refresh token: {:?}", e)))?; - - Ok(OAuthConfig { - base_url: oauth_config.base_url.clone(), - access_token: token_response.access_token, - refresh_token: token_response - .refresh_token - .or_else(|| oauth_config.refresh_token.clone()), - client_id: oauth_config.client_id.clone(), - client_secret: oauth_config.client_secret.clone(), - }) -} - -/// Fallback refresh without native_triggers feature -#[cfg(not(feature = "native_trigger"))] -pub async fn refresh_oauth_tokens( - _oauth_config: &OAuthConfig, - _refresh_endpoint: &str, -) -> Result { - Err(Error::InternalErr( - "Native triggers feature is not enabled".to_string(), - )) -} - -async fn update_workspace_integration_tokens_helper( - db: DB, - workspace_id: String, - service_name: ServiceName, - oauth_data: serde_json::Value, -) { - let result = async { - let mut tx = db.begin().await?; - let mc = build_crypt(&db, &workspace_id).await?; - let mut encrypted_oauth_data = oauth_data; - - if let Some(access_token) = encrypted_oauth_data - .get("access_token") - .and_then(|v| v.as_str()) - { - let encrypted_access_token = encrypt(&mc, access_token); - encrypted_oauth_data["access_token"] = - serde_json::Value::String(encrypted_access_token); - } - - if let Some(refresh_token) = encrypted_oauth_data - .get("refresh_token") - .and_then(|v| v.as_str()) - { - let encrypted_refresh_token = encrypt(&mc, refresh_token); - encrypted_oauth_data["refresh_token"] = - serde_json::Value::String(encrypted_refresh_token); - } - - sqlx::query!( - r#" - UPDATE workspace_integrations - SET oauth_data = $1, updated_at = now() - WHERE workspace_id = $2 AND service_name = $3 - "#, - encrypted_oauth_data, - workspace_id, - service_name as ServiceName, - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok::<(), Error>(()) - } - .await; - - if let Err(e) = result { - tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}", - service_name, workspace_id, e); - } -} - -/// Look up the full token from the token table using its prefix -pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - token_prefix: &str, -) -> Result> { - let token = sqlx::query_scalar!( - r#" - SELECT token - FROM token - WHERE token LIKE concat($1::text, '%') - LIMIT 1 - "#, - token_prefix - ) - .fetch_optional(db) - .await?; - - Ok(token) -} - -/// Delete a token from the token table using its prefix -pub async fn delete_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - token_prefix: &str, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM token - WHERE token LIKE concat($1::text, '%') - "#, - token_prefix - ) - .execute(db) - .await? - .rows_affected(); - - Ok(deleted > 0) -} - -pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>, C: Serialize>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - config: &NativeTriggerConfig, - service_config: C, -) -> Result<()> { - // Store only the first 10 characters of the webhook token as a prefix - let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); - - sqlx::query!( - r#" - INSERT INTO native_trigger ( - external_id, - workspace_id, - service_name, - script_path, - is_flow, - webhook_token_prefix, - service_config - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7 - ) - ON CONFLICT (external_id, workspace_id, service_name) - DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_prefix = $6, service_config = $7, error = NULL, updated_at = NOW() - "#, - external_id, - workspace_id, - service_name as ServiceName, - config.script_path, - config.is_flow, - webhook_token_prefix, - sqlx::types::Json(service_config) as _, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - config: &NativeTriggerConfig, - service_config: Option<&RawValue>, -) -> Result<()> { - // Store only the first 10 characters of the webhook token as a prefix - let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); - - sqlx::query!( - r#" - UPDATE native_trigger - SET script_path = $1, is_flow = $2, webhook_token_prefix = $3, service_config = $4, error = NULL, updated_at = NOW() - WHERE - workspace_id = $5 - AND service_name = $6 - AND external_id = $7 - "#, - config.script_path, - config.is_flow, - webhook_token_prefix, - service_config.map(sqlx::types::Json) as _, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn delete_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND external_id = $3 - "#, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await? - .rows_affected(); - - Ok(deleted > 0) -} -pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, -) -> Result> { - let trigger = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - external_id, - workspace_id, - service_name AS "service_name!: ServiceName", - script_path, - is_flow, - webhook_token_prefix, - service_config, - error, - created_at, - updated_at - FROM - native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND external_id = $3 - "#, - workspace_id, - service_name as ServiceName, - external_id - ) - .fetch_optional(db) - .await?; - - Ok(trigger) -} - -pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - script_path: &str, - is_flow: bool, -) -> Result> { - let trigger = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - external_id, - workspace_id, - service_name AS "service_name!: ServiceName", - script_path, - is_flow, - webhook_token_prefix, - service_config, - error, - created_at, - updated_at - FROM - native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND script_path = $3 - AND is_flow = $4 - LIMIT 1 - "#, - workspace_id, - service_name as ServiceName, - script_path, - is_flow - ) - .fetch_optional(db) - .await?; - - Ok(trigger) -} - -pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - page: Option, - per_page: Option, - path: Option<&str>, - is_flow: Option, -) -> Result> { - let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; - let limit = per_page.unwrap_or(100) as i64; - - let triggers = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - nt.external_id, - nt.workspace_id, - nt.service_name AS "service_name!: ServiceName", - nt.script_path, - nt.is_flow, - nt.webhook_token_prefix, - nt.service_config, - nt.error, - nt.created_at, - nt.updated_at - FROM - native_trigger nt - WHERE - nt.workspace_id = $1 AND - nt.service_name = $2 AND - ($5::text IS NULL OR nt.script_path = $5) AND - ($6::bool IS NULL OR nt.is_flow = $6) AND - ( - (nt.is_flow = false AND EXISTS ( - SELECT 1 FROM script s - WHERE s.workspace_id = nt.workspace_id - AND s.path = nt.script_path - )) - OR - (nt.is_flow = true AND EXISTS ( - SELECT 1 FROM flow f - WHERE f.workspace_id = nt.workspace_id - AND f.path = nt.script_path - )) - ) - LIMIT $3 - OFFSET $4 - "#, - workspace_id, - service_name as ServiceName, - limit, - offset, - path, - is_flow - ) - .fetch_all(db) - .await?; - - Ok(triggers) -} - -pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - error: Option<&str>, -) -> Result<()> { - sqlx::query!( - r#" - UPDATE native_trigger - SET error = $1 - WHERE - workspace_id = $2 - AND service_name = $3 - AND external_id = $4 - "#, - error, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn update_native_trigger_service_config< - 'c, - E: sqlx::Executor<'c, Database = Postgres>, ->( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - service_config: &serde_json::Value, -) -> Result<()> { - sqlx::query!( - r#" - UPDATE native_trigger - SET service_config = $1, updated_at = NOW() - WHERE - workspace_id = $2 - AND service_name = $3 - AND external_id = $4 - "#, - service_config, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn store_workspace_integration( - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - service_name: ServiceName, - oauth_data: serde_json::Value, -) -> Result<()> { - sqlx::query!( - r#" - INSERT INTO workspace_integrations ( - workspace_id, - service_name, - oauth_data, - created_by, - created_at, - updated_at - ) VALUES ( - $1, $2, $3, $4, now(), now() - ) - ON CONFLICT (workspace_id, service_name) - DO UPDATE SET - oauth_data = $3, - updated_at = now() - "#, - workspace_id, - service_name as ServiceName, - oauth_data, - authed.username, - ) - .execute(&mut *tx) - .await?; - - Ok(()) -} - -pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let integration = sqlx::query_as!( - WorkspaceIntegration, - r#" - SELECT - workspace_id, - service_name AS "service_name!: ServiceName", - oauth_data, - created_at, - updated_at, - created_by - FROM - workspace_integrations - WHERE - workspace_id = $1 - AND service_name = $2 - "#, - workspace_id, - service_name as ServiceName, - ) - .fetch_one(db) - .await?; - - Ok(integration) -} - -pub async fn delete_workspace_integration( - tx: &mut PgConnection, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM workspace_integrations - WHERE - workspace_id = $1 - AND service_name = $2 - "#, - workspace_id, - service_name as ServiceName, - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - Ok(deleted > 0) -} - -/// Generates the webhook URL that external services will call. -/// -/// `external_id` is optional because during CREATE we don't have it yet -/// (it's returned by the external service). During UPDATE, we have it. -pub fn generate_webhook_service_url( - base_url: &str, - w_id: &str, - script_path: &str, - is_flow: bool, - external_id: Option<&str>, - service_name: ServiceName, - webhook_token: &str, -) -> String { - let runnable_prefix = if is_flow { "f" } else { "p" }; - - let mut url = format!( - "{}/api/w/{}/jobs/run/{}/{}?token={}&service_name={}", - base_url, - w_id, - runnable_prefix, - script_path, - &webhook_token, - service_name.as_str(), - ); - - if let Some(id) = external_id { - url.push_str(&format!("&trigger_external_id={}", id)); - } - - url -} +pub use windmill_native_triggers::*; diff --git a/backend/windmill-api/src/npm_proxy.rs b/backend/windmill-api/src/npm_proxy.rs index 8bd7f851f7..02f8a3dbb8 100644 --- a/backend/windmill-api/src/npm_proxy.rs +++ b/backend/windmill-api/src/npm_proxy.rs @@ -39,7 +39,9 @@ fn parse_package_and_version(path: &str) -> Result<(String, String)> { // Scoped package: @scope/name/version let parts: Vec<&str> = path.splitn(3, '/').collect(); if parts.len() < 3 { - return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string())); + return Err(Error::BadRequest( + "Invalid scoped package path, expected @scope/name/version".to_string(), + )); } let package = format!("{}/{}", parts[0], parts[1]); let version = parts[2].to_string(); @@ -48,7 +50,9 @@ fn parse_package_and_version(path: &str) -> Result<(String, String)> { // Regular package: name/version let parts: Vec<&str> = path.splitn(2, '/').collect(); if parts.len() < 2 { - return Err(Error::BadRequest("Invalid package path, expected name/version".to_string())); + return Err(Error::BadRequest( + "Invalid package path, expected name/version".to_string(), + )); } Ok((parts[0].to_string(), parts[1].to_string())) } @@ -62,7 +66,10 @@ fn parse_package_version_and_file(path: &str) -> Result<(String, String, String) // Scoped package: @scope/name/version/filepath let parts: Vec<&str> = path.splitn(4, '/').collect(); if parts.len() < 4 { - return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string())); + return Err(Error::BadRequest( + "Invalid scoped package file path, expected @scope/name/version/filepath" + .to_string(), + )); } let package = format!("{}/{}", parts[0], parts[1]); let version = parts[2].to_string(); @@ -72,9 +79,15 @@ fn parse_package_version_and_file(path: &str) -> Result<(String, String, String) // Regular package: name/version/filepath let parts: Vec<&str> = path.splitn(3, '/').collect(); if parts.len() < 3 { - return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string())); + return Err(Error::BadRequest( + "Invalid package file path, expected name/version/filepath".to_string(), + )); } - Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string())) + Ok(( + parts[0].to_string(), + parts[1].to_string(), + parts[2].to_string(), + )) } } @@ -313,10 +326,7 @@ async fn get_package_filetree( .unwrap_or("index.js") .to_string(); - Ok(Json(PackageFiletree { - default: main, - files, - })) + Ok(Json(PackageFiletree { default: main, files })) } /// Get a specific file from a package version @@ -419,7 +429,10 @@ fn format_registry_url( match (version, file) { (Some(v), Some(f)) => { - format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f) + format!( + "{}/{}/{}/-/{}-{}/{}", + registry_base, package_path, v, package, v, f + ) } (Some(v), None) => { format!("{}/{}/{}", registry_base, package_path, v) @@ -453,9 +466,7 @@ fn extract_tarball_files(tarball_bytes: &[u8]) -> Result> { // Remove the package/ prefix that npm tarballs have let path_str = path.to_string_lossy().to_string(); if let Some(stripped) = path_str.strip_prefix("package/") { - files.push(FileEntry { - name: format!("/{}", stripped), - }); + files.push(FileEntry { name: format!("/{}", stripped) }); } } @@ -490,13 +501,16 @@ fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result< if let Some(stripped) = path_str.strip_prefix("package/") { if stripped == target { let mut content = String::new(); - entry - .read_to_string(&mut content) - .map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?; + entry.read_to_string(&mut content).map_err(|e| { + Error::InternalErr(format!("Failed to read file content: {}", e)) + })?; return Ok(content); } } } - Err(Error::NotFound(format!("File {} not found in tarball", target_file))) + Err(Error::NotFound(format!( + "File {} not found in tarball", + target_file + ))) } diff --git a/backend/windmill-api/src/oauth2_oss.rs b/backend/windmill-api/src/oauth2_oss.rs index fba65a11bd..2b47c66bdf 100644 --- a/backend/windmill-api/src/oauth2_oss.rs +++ b/backend/windmill-api/src/oauth2_oss.rs @@ -20,14 +20,14 @@ use hmac::Mac; #[cfg(all(feature = "oauth2", not(feature = "private")))] use itertools::Itertools; -#[cfg(all(feature = "oauth2", not(feature = "private")))] -use windmill_oauth::{OClient, AccessToken, RefreshToken, Scope, helpers}; #[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[cfg(not(feature = "private"))] use sqlx::{Postgres, Transaction}; #[cfg(all(feature = "oauth2", not(feature = "private")))] use windmill_common::more_serde::maybe_number_opt; +#[cfg(all(feature = "oauth2", not(feature = "private")))] +use windmill_oauth::{helpers, AccessToken, RefreshToken, Scope}; #[cfg(all(feature = "oauth2", not(feature = "private")))] use crate::OAUTH_CLIENTS; @@ -54,47 +54,10 @@ pub fn workspaced_service() -> Router { } #[cfg(all(feature = "oauth2", not(feature = "private")))] -#[derive(Debug, Clone)] -pub struct ClientWithScopes { - _client: OClient, - _scopes: Vec, - _extra_params: Option>, - _extra_params_callback: Option>, - _allowed_domains: Option>, - _userinfo_url: Option, -} -#[cfg(all(feature = "oauth2", not(feature = "private")))] -pub type BasicClientsMap = HashMap; +pub use windmill_oauth::{AllClients, BasicClientsMap, ClientWithScopes}; #[cfg(not(feature = "private"))] -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct OAuthConfig { - auth_url: String, - token_url: String, - userinfo_url: Option, - scopes: Option>, - extra_params: Option>, - extra_params_callback: Option>, - req_body_auth: Option, -} - -#[cfg(not(feature = "private"))] -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct OAuthClient { - id: String, - secret: String, - allowed_domains: Option>, - connect_config: Option, - login_config: Option, -} - -#[cfg(all(feature = "oauth2", not(feature = "private")))] -#[derive(Debug)] -pub struct AllClients { - pub logins: BasicClientsMap, - pub connects: BasicClientsMap, - pub slack: Option, -} +pub use windmill_oauth::{OAuthClient, OAuthConfig}; #[cfg(all(feature = "oauth2", not(feature = "private")))] pub async fn build_oauth_clients( diff --git a/backend/windmill-api/src/openapi.rs b/backend/windmill-api/src/openapi.rs index 04d17a6127..a178c38163 100644 --- a/backend/windmill-api/src/openapi.rs +++ b/backend/windmill-api/src/openapi.rs @@ -25,8 +25,7 @@ use crate::{ db::ApiAuthed, resources::try_get_resource_from_db_as, triggers::http::{ - http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, - AuthenticationMethod, RequestType, + http_trigger_auth::ApiKeyAuthentication, AuthenticationMethod, HttpMethod, RequestType, }, }; diff --git a/backend/windmill-api/src/public_app_rate_limit.rs b/backend/windmill-api/src/public_app_rate_limit.rs index bff89d694b..5d49c17a71 100644 --- a/backend/windmill-api/src/public_app_rate_limit.rs +++ b/backend/windmill-api/src/public_app_rate_limit.rs @@ -17,8 +17,7 @@ struct RateLimitEntry { minute_bucket: i64, } -static RATE_LIMIT_COUNTER: LazyLock> = - LazyLock::new(DashMap::new); +static RATE_LIMIT_COUNTER: LazyLock> = LazyLock::new(DashMap::new); pub fn check_and_increment(workspace_id: &str, limit: i32) -> Result<()> { let current_minute = Utc::now().timestamp() / 60; diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index 8aa1650034..e331aa1176 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -5,42 +5,29 @@ * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ -use crate::{ - db::{ApiAuthed, DB}, - users::require_owner_of_path, - utils::check_scopes, - webhook_util::{WebhookMessage, WebhookShared}, -}; +use crate::{db::ApiAuthed, utils::check_scopes}; use axum::{ body::Body, extract::{Extension, Json, Path, Query}, response::Response, - routing::{delete, get, post}, + routing::get, Router, }; -use hyper::{header, StatusCode}; +use hyper::header; use serde::{Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::FromRow; -use std::str; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; use windmill_common::{ apps::ListAppQuery, db::UserDB, error::{Error, JsonResult, Result}, utils::{not_found_if_none, paginate, Pagination, StripPath}, - worker::CLOUD_HOSTED, }; pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_apps)) .route("/get_data/:version/*path", get(get_data)) - .route("/exists/*path", get(exists_app)) - .route("/update/*path", post(update_app)) - .route("/delete/*path", delete(delete_app)) - .route("/create", post(create_app)) } #[derive(FromRow, Deserialize, Serialize)] @@ -54,20 +41,6 @@ pub struct ListableApp { pub version: i32, } -#[derive(Deserialize)] -pub struct CreateApp { - pub path: String, - pub summary: String, - pub value: String, -} - -#[derive(Deserialize)] -pub struct EditApp { - pub path: Option, - pub summary: Option, - pub value: Option, -} - async fn list_apps( authed: ApiAuthed, Extension(user_db): Extension, @@ -147,230 +120,3 @@ async fn get_data( Ok(res.body(Body::from(app)).unwrap()) } - -async fn create_app( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Extension(db): Extension, - Path(w_id): Path, - Json(app): Json, -) -> Result<(StatusCode, String)> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot create raw apps for security reasons".to_string(), - )); - } - check_scopes(&authed, || format!("raw_apps:write:{}", app.path))?; - if *CLOUD_HOSTED { - let nb_apps = sqlx::query_scalar!( - "SELECT COUNT(*) FROM raw_app WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - if nb_apps.unwrap_or(0) >= 1000 { - return Err(Error::BadRequest( - "You have reached the maximum number of apps (1000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - if app.summary.len() > 300 { - return Err(Error::BadRequest( - "Summary must be less than 300 characters on cloud".to_string(), - )); - } - } - let mut tx = user_db.begin(&authed).await?; - if &app.path == "" { - return Err(Error::BadRequest("App path cannot be empty".to_string())); - } - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)", - app.path, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if exists { - return Err(Error::BadRequest(format!( - "App with path {} already exists", - &app.path - ))); - } - - sqlx::query!( - "INSERT INTO raw_app - (workspace_id, path, summary, extra_perms, data) - VALUES ($1, $2, $3, '{}', $4)", - w_id, - app.path, - app.summary, - app.value, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "apps.create", - ActionKind::Create, - &w_id, - Some(&app.path), - None, - ) - .await?; - - tx.commit().await?; - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() }, - ); - - Ok((StatusCode::CREATED, app.path)) -} - -async fn delete_app( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("raw_apps:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "apps.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - webhook.send_message( - w_id.clone().clone(), - WebhookMessage::DeleteApp { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("app {} deleted", path)) -} - -async fn update_app( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(app): Json, -) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update raw apps for security reasons".to_string(), - )); - } - use sql_builder::prelude::*; - - let path = path.to_path(); - check_scopes(&authed, || format!("raw_apps:write:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - let mut sqlb = SqlBuilder::update_table("raw_app"); - sqlb.and_where_eq("path", "?".bind(&path)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - - let npath = &app.path; - if npath.is_some() || app.summary.is_some() { - if let Some(npath) = npath { - if npath != path { - require_owner_of_path(&authed, path)?; - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)", - npath, - w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if exists { - return Err(Error::BadRequest(format!( - "App with path {} already exists", - npath - ))); - } - } - sqlb.set_str("path", npath); - } - - if let Some(nsummary) = &app.summary { - sqlb.set_str("summary", nsummary); - } - } - - if let Some(value) = &app.value { - sqlb.set_str("data", value); - sqlb.set("version", "version + 1"); - } - - sqlb.returning("path"); - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; - not_found_if_none(npath_o, "Raw App", path)?; - - let npath = app.path.clone().unwrap_or_else(|| path.to_owned()); - audit_log( - &mut *tx, - &authed, - "apps.update", - ActionKind::Update, - &w_id, - Some(&path), - None, - ) - .await?; - tx.commit().await?; - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateApp { - workspace: w_id, - old_path: path.to_owned(), - new_path: npath.clone(), - }, - ); - - Ok(format!("app {} updated (npath: {:?})", path, npath)) -} - -async fn exists_app( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - // Note: exists_app doesn't require authentication, so no scope check needed - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 575e09421c..83203233fc 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -1,82 +1,30 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ +// Re-export everything from windmill-store +pub use windmill_store::resources::*; -use std::collections::HashMap; - -use crate::{ - db::{ApiAuthed, DB}, - secret_backend_ext::rename_vault_secret, - users::{maybe_refresh_folders, require_owner_of_path, Tokened}, - utils::{check_scopes, require_super_admin, BulkDeleteRequest}, - var_resource_cache::{cache_resource, get_cached_resource}, - variables::get_value_internal, - webhook_util::{WebhookMessage, WebhookShared}, -}; +#[cfg(feature = "mcp")] use axum::{ - body::Body, - extract::{Extension, Path, Query}, - response::Response, - routing::{delete, get, post}, + extract::{Extension, Path}, + routing::get, Json, Router, }; -use futures::future::try_join_all; -use hyper::{header, StatusCode}; -use serde::{Deserialize, Serialize}; -use serde_json::{value::RawValue, Value}; -use sql_builder::{bind::Bind, quote, SqlBuilder}; -use sqlx::{Acquire, FromRow, Postgres, Transaction}; -use std::process::Stdio; -use tokio::process::Command; -use uuid::Uuid; -use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; -use windmill_audit::ActionKind; +#[cfg(feature = "mcp")] +use serde_json::value::RawValue; +#[cfg(feature = "mcp")] +use windmill_api_auth::{check_scopes, ApiAuthed}; +#[cfg(feature = "mcp")] use windmill_common::{ - db::{DbWithOptAuthed, UserDB}, - error::{self, Error, JsonResult, Result}, - get_database_url, - utils::{ - get_custom_pg_instance_password, not_found_if_none, paginate, require_admin, Pagination, - StripPath, - }, - variables, - worker::{CLOUD_HOSTED, TMP_DIR}, - PgDatabase, + db::{UserDB, DB}, + error::{Error, JsonResult, Result}, + utils::{not_found_if_none, StripPath}, }; +#[cfg(not(feature = "mcp"))] +use axum::Router; + +/// Wraps the subcrate's workspaced_service with the mcp_tools route +/// that depends on windmill-api internals. pub fn workspaced_service() -> Router { - let router = Router::new() - .route("/list", get(list_resources)) - .route("/list_search", get(list_search_resources)) - .route("/list_names/:type", get(list_names)) - .route("/get/*path", get(get_resource)) - .route("/exists/*path", get(exists_resource)) - .route("/get_value/*path", get(get_resource_value)) - .route( - "/get_value_interpolated/*path", - get(get_resource_value_interpolated), - ) - .route("/update/*path", post(update_resource)) - .route("/update_value/*path", post(update_resource_value)) - .route("/delete/*path", delete(delete_resource)) - .route("/delete_bulk", delete(delete_resources_bulk)) - .route("/create", post(create_resource)) - .route("/git_commit_hash/*path", get(get_git_commit_hash)) - .route("/type/list", get(list_resource_types)) - .route("/type/listnames", get(list_resource_types_names)) - .route("/type/get/:name", get(get_resource_type)) - .route("/type/exists/:name", get(exists_resource_type)) - .route("/type/update/:name", post(update_resource_type)) - .route("/type/delete/:name", delete(delete_resource_type)) - .route( - "/file_resource_type_to_file_ext_map", - get(file_resource_ext_to_resource_type), - ) - .route("/type/create", post(create_resource_type)); + let router = windmill_store::resources::workspaced_service(); #[cfg(feature = "mcp")] let router = router.route("/mcp_tools/*path", get(get_mcp_tools)); @@ -84,1365 +32,6 @@ pub fn workspaced_service() -> Router { router } -pub fn public_service() -> Router { - Router::new().route("/custom_component/:name", get(custom_component)) -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct ResourceType { - pub workspace_id: String, - pub name: String, - pub schema: Option, - pub description: Option, - pub created_by: Option, - pub edited_at: Option>, - pub format_extension: Option, -} - -#[derive(Deserialize)] -pub struct CreateResourceType { - pub name: String, - pub schema: Option, - pub description: Option, - pub format_extension: Option, -} - -#[derive(Deserialize)] -pub struct EditResourceType { - pub schema: Option, - pub description: Option, -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct Resource { - pub workspace_id: String, - pub path: String, - pub value: Option, - pub description: Option, - pub resource_type: String, - pub extra_perms: serde_json::Value, - pub created_by: Option, - pub edited_at: Option>, -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct ListableResource { - pub workspace_id: String, - pub path: String, - pub value: Option, - pub description: Option, - pub resource_type: String, - pub extra_perms: serde_json::Value, - pub created_by: Option, - pub edited_at: Option>, - pub is_linked: Option, - pub is_refreshed: Option, - pub is_oauth: Option, - pub is_expired: Option, - pub refresh_error: Option, - pub account: Option, -} - -#[derive(Deserialize)] -pub struct CreateResource { - pub path: String, - pub value: Option>, - pub description: Option, - pub resource_type: String, -} -#[derive(Deserialize)] -struct EditResource { - path: Option, - description: Option, - value: Option>, -} - -#[derive(Deserialize)] -pub struct ListResourceQuery { - resource_type: Option, - resource_type_exclude: Option, - path_start: Option, -} - -#[derive(Serialize, FromRow)] -pub struct NamePath { - name: String, - path: String, -} -async fn list_names( - authed: ApiAuthed, - Path((w_id, rt)): Path<(String, String)>, - Extension(user_db): Extension, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query!( - "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2", - rt, - &w_id - ) - .fetch_all(&mut *tx) - .await? - .into_iter() - .filter_map(|x| x.name.map(|name| NamePath { name, path: x.path })) - .collect::>(); - tx.commit().await?; - Ok(Json(rows)) -} - -#[derive(Serialize, FromRow)] -pub struct SearchResource { - path: String, - value: serde_json::Value, -} -async fn list_search_resources( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - #[cfg(feature = "enterprise")] - let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; - - let rows = sqlx::query_as!( - SearchResource, - "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2", - &w_id, - n - ) - .fetch_all(&mut *tx) - .await? - .into_iter() - .collect::>(); - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_resources( - authed: ApiAuthed, - Query(lq): Query, - Query(pagination): Query, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - - let mut sqlb = SqlBuilder::select_from("resource") - .fields(&[ - "resource.workspace_id", - "resource.path", - "null::JSONB as value", - "resource.description", - "resource_type", - "resource.extra_perms", - "(now() > account.expires_at) as is_expired", - "variable.path IS NOT NULL as is_linked", - "account.refresh_token != '' as is_refreshed", - "variable.is_oauth", - "variable.account", - "account.refresh_error", - "resource.created_by", - "resource.edited_at", - ]) - .left() - .join("variable") - .on("variable.path = resource.path AND variable.workspace_id = resource.workspace_id") - .left() - .join("account") - .on("variable.account = account.id AND account.workspace_id = variable.workspace_id") - .order_by("path", true) - .and_where("resource.workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page) - .clone(); - - if let Some(rt) = &lq.resource_type { - let resource_type_filters = rt.split(',').collect::>(); - if resource_type_filters.len() == 1 { - sqlb.and_where_eq("resource_type", "?".bind(rt)); - } else { - let mut list = Vec::new(); - for rt in resource_type_filters { - let quoted_value = quote(rt); - list.push(quoted_value); - } - sqlb.and_where_in("resource_type", list.as_slice()); - } - } - if let Some(rt) = &lq.resource_type_exclude { - for rt in rt.split(',') { - sqlb.and_where_ne("resource_type", "?".bind(&rt)); - } - } - - if let Some(path_start) = &lq.path_start { - sqlb.and_where_like_left("resource.path", path_start); - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as::<_, ListableResource>(&sql) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(rows)) -} - -async fn get_resource( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let resource_o = sqlx::query_as!( - ListableResource, - "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed, - account.refresh_error, - variable.path IS NOT NULL as is_linked, - variable.is_oauth as \"is_oauth?\", - variable.account - FROM resource - LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2 - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2 - WHERE resource.path = $1 AND resource.workspace_id = $2", - path.to_owned(), - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - if resource_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; - } - let resource = not_found_if_none(resource_o, "Resource", path)?; - Ok(Json(resource)) -} - -async fn exists_resource( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn get_resource_value( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - - let value_o = sqlx::query_scalar!( - "SELECT value from resource WHERE path = $1 AND workspace_id = $2", - path.to_owned(), - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - if value_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; - } - - let value = not_found_if_none(value_o, "Resource", path)?; - - Ok(Json(value)) -} - -async fn explain_resource_perm_error( - path: &str, - w_id: &str, - db: &sqlx::Pool, - authed: &ApiAuthed, -) -> windmill_common::error::Result<()> { - let extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Resource {} not found", path)))?; - if path.starts_with("f/") { - let folder = path.split("/").nth(1).ok_or_else(|| { - Error::BadRequest(format!( - "path {} should have at least 2 components separated by /", - path - )) - })?; - let folder_extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", - folder, - w_id - ) - .fetch_optional(db) - .await?; - return Err(Error::NotAuthorized(format!( - "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() - ))); - } else { - return Err(Error::NotAuthorized(format!( - "Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default() - ))); - } -} - -async fn custom_component( - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - let cc_o = sqlx::query_scalar!( - "SELECT value->>'js' FROM resource - WHERE path = $1 AND workspace_id = $2", - format!("f/app_custom/{name}"), - &w_id - ) - .fetch_optional(&db) - .await? - .flatten(); - - let cc = not_found_if_none(cc_o, "Custom Component", name)?; - let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript"); - - Ok(res.body(Body::from(cc)).unwrap()) -} - -#[derive(Deserialize)] -struct JobInfo { - job_id: Option, - allow_cache: Option, -} - -async fn get_resource_value_interpolated( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Tokened { token }: Tokened, - Path((w_id, path)): Path<(String, StripPath)>, - Query(job_info): Query, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let db_with_opt_authed = - DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - return get_resource_value_interpolated_internal( - &db_with_opt_authed, - w_id.as_str(), - path, - job_info.job_id, - Some(token.as_str()), - job_info.allow_cache.unwrap_or(false), - ) - .await - .map(|success| Json(success)); -} - -use async_recursion::async_recursion; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; - -pub async fn get_resource_value_interpolated_internal<'a>( - db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, - workspace: &str, - path: &str, - job_id: Option, - token_for_context: Option<&str>, - allow_cache: bool, -) -> Result> { - // This is a special syntax to help debugging custom instance databases - if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") { - let db = db_with_opt_authed.db(); - require_super_admin(db_with_opt_authed.db(), &db_with_opt_authed.email()).await?; - let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; - pg_creds.dbname = dbname.to_string(); - pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); - pg_creds.user = Some("custom_instance_user".to_string()); - let pg_creds = serde_json::to_value(&pg_creds) - .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?; - return Ok(Some(pg_creds)); - } - - if allow_cache { - if let Some(cached_value) = get_cached_resource(&workspace, &path) { - return Ok(Some(cached_value)); - } - } - use sqlx::Acquire; - let mut tx = db_with_opt_authed.begin().await?; - - let value_o = sqlx::query_scalar!( - "SELECT value from resource WHERE path = $1 AND workspace_id = $2", - path, - workspace - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - if value_o.is_none() { - if let Some(authed) = db_with_opt_authed.authed() { - let db = db_with_opt_authed.db(); - explain_resource_perm_error(path, workspace, db, authed).await?; - } - } - - let value = not_found_if_none(value_o, "Resource", path)?; - if let Some(value) = value { - let r = transform_json_value( - &db_with_opt_authed, - workspace, - value, - &job_id, - token_for_context, - ) - .await?; - if allow_cache { - cache_resource(&workspace, &path, r.clone()); - } - Ok(Some(r)) - } else { - Ok(None) - } -} - -#[async_recursion] -pub async fn transform_json_value( - db_with_opt_authed: &DbWithOptAuthed, - workspace: &str, - v: Value, - job_id: &Option, - token: Option<&str>, -) -> Result { - match v { - Value::String(y) if y.starts_with("$var:") => { - let path = y.strip_prefix("$var:").unwrap(); - - let v = - crate::variables::get_value_internal(&db_with_opt_authed, workspace, path, false) - .await?; - Ok(Value::String(v)) - } - Value::String(y) if y.starts_with("$res:") => { - let path = y.strip_prefix("$res:").unwrap(); - if path.split("/").count() < 2 { - return Err(Error::internal_err(format!( - "Invalid resource path: {path}" - ))); - } - let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; - let v = sqlx::query_scalar!( - "SELECT value from resource WHERE path = $1 AND workspace_id = $2", - path, - &workspace - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - let v = not_found_if_none(v, "Resource", path)?; - if let Some(v) = v { - transform_json_value(db_with_opt_authed, workspace, v, job_id, token).await - } else { - Ok(Value::Null) - } - } - Value::String(y) if y.starts_with("$") && job_id.is_some() => { - let mut tx = db_with_opt_authed.begin().await?; - let job_id = job_id.unwrap(); - let job = sqlx::query!( - "SELECT - v2_job.permissioned_as_email, - v2_job.created_by, - v2_job.parent_job, - v2_job.permissioned_as, - v2_job.runnable_path, - CASE WHEN v2_job.trigger_kind = 'schedule'::job_trigger_kind THEN v2_job.trigger END AS schedule_path, - v2_job.flow_step_id, - v2_job.flow_innermost_root_job, - v2_job.root_job, - v2_job_queue.scheduled_for AS \"scheduled_for: chrono::DateTime\" - FROM v2_job INNER JOIN v2_job_queue ON v2_job.id = v2_job_queue.id - WHERE v2_job.id = $1 AND v2_job.workspace_id = $2", - job_id, - workspace - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let job = not_found_if_none(job, "Job", job_id.to_string())?; - - let flow_path = if let Some(uuid) = job.parent_job { - let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; - let p = sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) - .fetch_optional(&mut *tx) - .await? - .flatten(); - tx.commit().await?; - p - } else { - None - }; - - let variables = variables::get_reserved_variables( - &db_with_opt_authed.db().into(), - workspace, - token.unwrap_or_else(|| "no_token_available"), - &job.permissioned_as_email, - &job.created_by, - &job_id.to_string(), - &job.permissioned_as, - job.runnable_path.clone(), - job.parent_job.map(|x| x.to_string()), - flow_path, - job.schedule_path.clone(), - job.flow_step_id.clone(), - job.flow_innermost_root_job.map(|x| x.to_string()), - job.root_job.map(|x| x.to_string()), - Some(job.scheduled_for.clone()), - None, - None, - ) - .await; - - let name = y.strip_prefix("$").unwrap(); - - let value = variables - .iter() - .find(|x| x.name == name) - .map(|x| x.value.clone()) - .unwrap_or_else(|| y); - Ok(serde_json::json!(value)) - } - Value::Object(mut m) => { - for (a, b) in m.clone().into_iter() { - let v = - transform_json_value(db_with_opt_authed, workspace, b, job_id, token).await?; - m.insert(a.clone(), v); - } - Ok(Value::Object(m)) - } - a @ _ => Ok(a), - } -} - -// async fn authed_transaction_or_default<'c>( -// db_with_opt_authed: &'c DbWithOptAuthed, -// ) -> sqlx::error::Result> { -// if let Some(user_db) = user_db { -// user_db.begin(authed).await -// } else { -// db.clone().begin().await -// } -// } - -async fn check_path_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - path: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "Resource {} already exists", - path - ))); - } - return Ok(()); -} - -#[derive(Deserialize)] -struct CreateResourceQuery { - update_if_exists: Option, -} -async fn create_resource( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Query(q): Query, - Json(resource): Json, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || format!("resources:write:{}", resource.path))?; - if *CLOUD_HOSTED { - let nb_resources = sqlx::query_scalar!( - "SELECT COUNT(*) FROM resource WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - if nb_resources.unwrap_or(0) >= 10000 { - return Err(Error::BadRequest( - "You have reached the maximum number of resources (10000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await; - - let mut tx = user_db.begin(&authed).await?; - - let update_if_exists = q.update_if_exists.unwrap_or(false); - if !update_if_exists { - check_path_conflict(&mut tx, &w_id, &resource.path).await?; - } - - let res_value = resource.value.unwrap_or_default(); - let raw_json = sqlx::types::Json(res_value.as_ref()); - - if resource.path.starts_with("f/app_themes/") { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - w_id, - authed.username, - ) - .execute(&db) - .await?; - } else if resource.path.starts_with("f/app_custom/") { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_custom', 'App Custom Components', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - w_id, - authed.username, - ) - .execute(&db) - .await?; - } else if resource.path.starts_with("f/app_groups/") { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - w_id, - authed.username, - ) - .execute(&db) - .await?; - } - sqlx::query!( - "INSERT INTO resource - (workspace_id, path, value, description, resource_type, created_by, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now()", - w_id, - resource.path, - raw_json as sqlx::types::Json<&RawValue>, - resource.description, - resource.resource_type, - authed.username - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "resources.create", - ActionKind::Create, - &w_id, - Some(&resource.path), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: resource.path.clone(), parent_path: None }, - Some(format!("Resource '{}' created", resource.path.clone())), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, - ); - - Ok(( - StatusCode::CREATED, - format!("resource {} created", resource.path), - )) -} - -async fn delete_resource( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - - check_scopes(&authed, || format!("resources:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let deleted_path = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - not_found_if_none(deleted_path, "Resource", &path)?; - sqlx::query!( - "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "resources.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, - Some(format!("Resource '{}' deleted", path)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("resource {} deleted", path)) -} - -async fn delete_resources_bulk( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(request): Json, -) -> JsonResult> { - for path in &request.paths { - check_scopes(&authed, || format!("resources:write:{}", path))?; - } - - let mut tx = user_db.begin(&authed).await?; - - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id - ) - .fetch_all(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "resources.delete_bulk", - ActionKind::Delete, - &w_id, - Some(&deleted_paths.join(", ")), - None, - ) - .await?; - - tx.commit().await?; - - try_join_all(deleted_paths.iter().map(|path| { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { - path: path.to_string(), - parent_path: Some(path.to_string()), - }, - Some(format!("Resource '{}' deleted", path)), - true, - None, - ) - })) - .await?; - - for path in &deleted_paths { - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteResource { workspace: w_id.clone(), path: path.to_owned() }, - ); - } - - Ok(Json(deleted_paths)) -} - -async fn update_resource( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(ns): Json, -) -> Result { - use sql_builder::prelude::*; - - let path = path.to_path(); - check_scopes(&authed, || format!("resources:write:{}", path))?; - - let mut sqlb = SqlBuilder::update_table("resource"); - sqlb.and_where_eq("path", "?".bind(&path)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - - if let Some(npath) = &ns.path { - sqlb.set_str("path", npath); - } - if let Some(nvalue) = ns.value { - sqlb.set_str("value", nvalue.to_string()); - } - if let Some(ndesc) = ns.description { - sqlb.set_str("description", ndesc); - } - - sqlb.set_str("edited_at", "now()"); - - sqlb.returning("path"); - let authed = maybe_refresh_folders(path, &w_id, authed, &db).await; - - let mut tx = user_db.begin(&authed).await?; - - if let Some(npath) = ns.path.clone() { - if npath != path { - check_path_conflict(&mut tx, &w_id, &npath).await?; - - require_owner_of_path(&authed, path)?; - - // Handle Vault secret rename if the linked variable is a Vault-stored secret - let linked_var = sqlx::query!( - "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - if let Some(var) = linked_var { - if var.is_secret { - // Check if this is a Vault-stored secret and rename it - if let Some(new_value) = - rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? - { - // Update the variable's value to point to the new Vault path - sqlx::query!( - "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", - new_value, - path, - w_id - ) - .execute(&mut *tx) - .await?; - } - } - } - - sqlx::query!( - "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", - npath, - path, - w_id - ) - .execute(&mut *tx) - .await?; - } - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; - - let npath = not_found_if_none(npath_o, "Resource", path)?; - - audit_log( - &mut *tx, - &authed, - "resources.update", - ActionKind::Update, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - - // Detect if this was a rename operation - let old_path_if_renamed = if npath != path { Some(path) } else { None }; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: npath.to_string(), parent_path: Some(path.to_string()) }, - Some(format!("Resource '{}' updated", npath)), - true, - old_path_if_renamed, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateResource { - workspace: w_id, - old_path: path.to_owned(), - new_path: npath.clone(), - }, - ); - - Ok(format!("resource {} updated (npath: {:?})", path, npath)) -} - -#[derive(FromRow, Serialize, Deserialize)] -struct UpdateResource { - value: Option, -} - -async fn update_resource_value( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(nv): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "UPDATE resource SET value = $1, edited_at = now() WHERE path = $2 AND workspace_id = $3", - nv.value, - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "resources.update", - ActionKind::Update, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, - None, - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateResource { - workspace: w_id, - old_path: path.to_owned(), - new_path: path.to_owned(), - }, - ); - - Ok(format!("value of resource {} updated", path)) -} - -async fn file_resource_ext_to_resource_type( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - #[derive(Serialize, sqlx::FromRow)] - struct LocalFileResourceExtension { - name: String, - format_extension: Option, - } - - let r = sqlx::query_as!(LocalFileResourceExtension, " - SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')", w_id) - .fetch_all(&db) - .await?; - - let hashmap: HashMap = r - .into_iter() - .filter_map(|entry| { - if let Some(format_extension) = entry.format_extension { - Some((entry.name, format_extension)) - } else { - None - } - }) - .collect(); - - Ok(Json(hashmap)) -} - -async fn list_resource_types( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let rows = sqlx::query_as!( - ResourceType, - "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ - BY name", - &w_id - ) - .fetch_all(&db) - .await?; - - Ok(Json(rows)) -} - -async fn list_resource_types_names( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let rows = sqlx::query_scalar!( - "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') \ - ORDER BY name", - &w_id - ) - .fetch_all(&db) - .await?; - - Ok(Json(rows)) -} - -async fn get_resource_type( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - - let resource_type_o = sqlx::query_as!( - ResourceType, - "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", - &name, - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let resource_type = not_found_if_none(resource_type_o, "ResourceType", name)?; - Ok(Json(resource_type)) -} - -async fn exists_resource_type( - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins'))", - name, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn create_resource_type( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(resource_type): Json, -) -> Result<(StatusCode, String)> { - let mut tx = user_db.begin(&authed).await?; - - check_rt_path_conflict(&mut tx, &w_id, &resource_type.name).await?; - - sqlx::query!( - "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, now())", - w_id, - resource_type.name, - resource_type.schema, - resource_type.description, - authed.username, - resource_type.format_extension, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "resource_types.create", - ActionKind::Create, - &w_id, - Some(&resource_type.name), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::ResourceType { path: resource_type.name.clone() }, - Some(format!( - "Resource Type '{}' created", - resource_type.name.clone() - )), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateResourceType { name: resource_type.name.clone() }, - ); - - Ok(( - StatusCode::CREATED, - format!("resource_type {} created", resource_type.name), - )) -} - -async fn check_rt_path_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - name: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)", - name, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "Resource type {} already exists", - name - ))); - } - return Ok(()); -} - -async fn delete_resource_type( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - let deleted_name = sqlx::query_scalar!( - "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", - name, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - not_found_if_none(deleted_name, "ResourceType", &name)?; - - audit_log( - &mut *tx, - &authed, - "resource_types.delete", - ActionKind::Delete, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::ResourceType { path: name.clone() }, - None, - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteResourceType { name: name.clone() }, - ); - - Ok(format!("resource_type {} deleted", name)) -} - -async fn update_resource_type( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(ns): Json, -) -> Result { - use sql_builder::prelude::*; - - let mut sqlb = SqlBuilder::update_table("resource_type"); - sqlb.and_where_eq("name", "?".bind(&name)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - if let Some(nschema) = ns.schema { - sqlb.set_str("schema", nschema); - } - if let Some(ndesc) = ns.description { - sqlb.set_str("description", ndesc); - } - sqlb.set_str("edited_at", "now()"); - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query(&sql).execute(&mut *tx).await?; - audit_log( - &mut *tx, - &authed, - "resource_types.update", - ActionKind::Update, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::ResourceType { path: name.clone() }, - None, - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateResourceType { name: name.clone() }, - ); - - Ok(format!("resource_type {} updated", name)) -} - -#[cfg(any( - feature = "http_trigger", - feature = "postgres_trigger", - feature = "mqtt_trigger", - all( - feature = "enterprise", - any( - feature = "sqs_trigger", - feature = "gcp_trigger", - feature = "kafka", - feature = "nats" - ) - ) -))] -pub async fn try_get_resource_from_db_as( - authed: &ApiAuthed, - user_db: Option, - db: &DB, - resource_path: &str, - w_id: &str, -) -> Result -where - T: serde::de::DeserializeOwned, -{ - let resource = get_resource_value_interpolated_internal( - &DbWithOptAuthed::from_authed(authed, db.clone(), user_db), - &w_id, - &resource_path, - None, - None, - false, - ) - .await?; - - let resource = match resource { - Some(resource) => serde_json::from_value::(resource) - .map_err(|e| Error::SerdeJson { error: e, location: "resources.rs".to_string() })?, - None => { - return { - Err(Error::NotFound(format!( - "resource at path :{} do not exist", - &resource_path - ))) - } - } - }; - - Ok(resource) -} - /// Get list of tools from an MCP resource #[cfg(feature = "mcp")] async fn get_mcp_tools( @@ -1456,7 +45,6 @@ async fn get_mcp_tools( let mut tx = user_db.clone().begin(&authed).await?; - // Fetch the MCP resource from database let resource_value_o = sqlx::query_scalar!( "SELECT value as \"value: sqlx::types::Json>\" FROM resource WHERE path = $1 AND workspace_id = $2", &path, @@ -1474,18 +62,15 @@ async fn get_mcp_tools( let resource_value = not_found_if_none(resource_value_o, "Resource", path)? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; - // Parse MCP resource let mcp_resource = serde_json::from_str::(resource_value.0.get()) .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; - // Check if token needs refresh before creating MCP client #[cfg(feature = "oauth2")] { tracing::info!("Checking if token needs refresh before creating MCP client"); if let Some(ref token_path) = mcp_resource.token { let token_var_path = token_path.trim_start_matches("$var:"); - // Query to check if token is expired let token_info = sqlx::query!( r#" SELECT @@ -1523,12 +108,10 @@ async fn get_mcp_tools( } } - // Create MCP client connection let client = windmill_mcp::McpClient::from_resource(mcp_resource, &db, &w_id) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; - // Get raw MCP tools and convert to JSON let tools: Vec = client .available_tools() .iter() @@ -1538,284 +121,9 @@ async fn get_mcp_tools( }) .collect::>>()?; - // Gracefully shutdown the client if let Err(e) = client.shutdown().await { tracing::warn!("Failed to shutdown MCP client: {}", e); } Ok(Json(tools)) } - -#[derive(Deserialize, Serialize)] -struct GitRepositoryResource { - url: String, - #[serde(skip_serializing_if = "Option::is_none")] - branch: Option, -} - -/// Validates a git URL to prevent git option injection attacks. -/// Git URLs starting with '-' could be interpreted as command-line options. -fn validate_git_url(url: &str) -> Result<()> { - let url = url.trim(); - if url.is_empty() { - return Err(Error::BadRequest("Git URL cannot be empty".to_string())); - } - if url.starts_with('-') { - return Err(Error::BadRequest( - "Git URL cannot start with '-' (potential option injection)".to_string(), - )); - } - // Block other potentially dangerous patterns - if url.contains('\0') || url.contains('\n') || url.contains('\r') { - return Err(Error::BadRequest( - "Git URL contains invalid characters".to_string(), - )); - } - Ok(()) -} - -/// Validates a git branch/ref name to prevent injection attacks. -fn validate_git_ref(ref_name: &str) -> Result<()> { - let ref_name = ref_name.trim(); - if ref_name.is_empty() { - return Err(Error::BadRequest("Git ref cannot be empty".to_string())); - } - if ref_name.starts_with('-') { - return Err(Error::BadRequest( - "Git ref cannot start with '-' (potential option injection)".to_string(), - )); - } - // Git ref names have specific rules - block dangerous characters - if ref_name.contains('\0') - || ref_name.contains('\n') - || ref_name.contains('\r') - || ref_name.contains("..") - || ref_name.contains("@{") - || ref_name.ends_with('.') - || ref_name.ends_with('/') - || ref_name.contains("//") - { - return Err(Error::BadRequest( - "Git ref contains invalid characters or patterns".to_string(), - )); - } - Ok(()) -} - -#[derive(Serialize)] -struct GitCommitHashResponse { - commit_hash: String, -} - -#[derive(Deserialize)] -struct GitCommitHashQuery { - git_ssh_identity: Option, -} - -async fn get_git_commit_hash( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> JsonResult { - let path = path.to_path(); - - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let db_with_opt_authed = - DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - let git_repo_resource_value = get_resource_value_interpolated_internal( - &db_with_opt_authed, - &w_id, - path, - None, - None, - false, - ) - .await - .map_err(|e| Error::NotFound(format!("Access to resource {} denied: ({e})", path)))?; - - let git_resource: GitRepositoryResource = match git_repo_resource_value { - Some(value) => serde_json::from_value(value).map_err(|e| { - Error::BadRequest(format!("Invalid git repository resource format: {}", e)) - })?, - None => return Err(Error::NotFound(format!("Resource {} not found", path)).into()), - }; - - let identities: Vec = query - .git_ssh_identity - .map(|s| { - s.split(",") - .filter_map(|s| { - if !s.is_empty() { - Some(s.to_string()) - } else { - None - } - }) - .collect() - }) - .unwrap_or(vec![]); - - let (git_ssh_cmd, filenames) = - get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; - - delete_paths(&filenames).await; - - Ok(Json(GitCommitHashResponse { commit_hash: commit_hash? })) -} - -async fn write_ssh_file( - authed: &ApiAuthed, - user_db: &UserDB, - db: &DB, - w_id: &str, - var_path: &str, -) -> std::result::Result { - let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); - let loc = std::path::Path::new(TMP_DIR) - .join("ssh_ids") - .join(id_file_name); - - let userdb_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); - let mut content = get_value_internal(&userdb_authed, &w_id, &var_path, false) - .await - .map_err(|e| { - ( - error::Error::NotFound(format!( - "Variable {var_path} not found for git ssh identity: {e:#}" - )), - loc.clone(), - ) - })?; - content.push_str("\n"); - - if let Some(p) = &loc.parent() { - tokio::fs::create_dir_all(p) - .await - .map_err(|e| (e.into(), loc.clone()))?; - } - tokio::fs::write(&loc, content) - .await - .map_err(|e| (e.into(), loc.clone()))?; - - #[cfg(unix)] - { - let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); - tokio::fs::set_permissions(&loc, perm) - .await - .map_err(|e| (e.into(), loc.clone()))?; - } - - return Ok(loc); -} - -async fn delete_paths(paths: &Vec) { - for path in paths { - let _ = tokio::fs::remove_file(&path).await; - } -} - -async fn get_git_ssh_cmd( - authed: &ApiAuthed, - user_db: &UserDB, - db: &DB, - w_id: &str, - git_ssh_identity: Vec, -) -> error::Result<(Option, Vec)> { - if git_ssh_identity.len() > 5 { - return Err(error::Error::BadRequest( - "Too many ssh identities, try using at most 1".to_string(), - )); - } - if git_ssh_identity.len() == 0 { - return Ok((None, vec![])); - } - - let mut ssh_id_files = vec![]; - let mut file_paths = vec![]; - for var_path in git_ssh_identity.iter() { - match write_ssh_file(authed, user_db, db, w_id, &var_path).await { - Ok(loc) => { - ssh_id_files.push(format!( - " -i '{}'", - loc.to_string_lossy().replace('\'', r"'\''") - )); - file_paths.push(loc); - } - Err((e, loc)) => { - file_paths.push(loc); - delete_paths(&file_paths).await; - return Err(e); - } - } - } - - let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); - Ok((Some(git_ssh_cmd), file_paths)) -} - -async fn get_repo_latest_commit_hash( - git_resource: &GitRepositoryResource, - git_ssh_command: Option, -) -> Result { - // Validate URL and branch to prevent option injection attacks - validate_git_url(&git_resource.url)?; - - let ref_spec = git_resource - .branch - .as_deref() - .filter(|s| !s.is_empty()) - .unwrap_or("HEAD"); - - // Validate ref_spec if it's not the default HEAD - if ref_spec != "HEAD" { - validate_git_ref(ref_spec)?; - } - - let mut git_cmd = Command::new("git"); - git_cmd.args(["ls-remote", &git_resource.url, ref_spec]); - if let Some(git_ssh_command) = git_ssh_command { - git_cmd.env("GIT_SSH_COMMAND", git_ssh_command); - } - git_cmd.stderr(Stdio::piped()); - - let output = git_cmd - .output() - .await - .map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e)))?; - - if !output.status.success() { - let stderr = String::from_utf8(output.stderr) - .unwrap_or_else(|_| "Failed to decode stderr".to_string()); - return Err(Error::BadRequest(format!( - "Error getting git repo commit hash: {}", - stderr - ))); - } - - let stdout = String::from_utf8(output.stdout) - .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; - - let lines: Vec<&str> = stdout.lines().collect(); - - if lines.is_empty() { - return Err(Error::BadRequest(format!( - "No commits found for reference '{}' in repository '{}'", - ref_spec, git_resource.url - ))); - } - - let commit_hash = lines - .first() - .and_then(|line| line.split_whitespace().next()) - .map(|s| s.to_string()) - .ok_or_else(|| { - Error::BadRequest("Unexpected output format for git ls-remote".to_string()) - })?; - - Ok(commit_hash) -} diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index ace6833bd2..0ea5f9e054 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -1001,8 +1001,8 @@ async fn set_default_error_handler( DeployedObject::Schedule { path: updated_schedule_path }, None, true, - None, - ) + None, + ) .await?; } } @@ -1060,34 +1060,7 @@ pub struct EditSchedule { pub dynamic_skip: Option, } -pub async fn clear_schedule<'c>( - tx: &mut Transaction<'c, Postgres>, - path: &str, - w_id: &str, -) -> Result<()> { - tracing::info!("Clearing schedule {}", path); - sqlx::query!( - "WITH to_delete AS ( - SELECT id FROM v2_job_queue - JOIN v2_job j USING (id) - WHERE trigger_kind = 'schedule' - AND trigger = $1 - AND j.workspace_id = $2 - AND flow_step_id IS NULL - AND running = false - FOR UPDATE - ), deleted AS ( - DELETE FROM v2_job_queue - WHERE id IN (SELECT id FROM to_delete) - RETURNING id - ) DELETE FROM v2_job WHERE id IN (SELECT id FROM deleted)", - path, - w_id - ) - .execute(&mut **tx) - .await?; - Ok(()) -} +pub use windmill_queue::schedule::clear_schedule; #[derive(Deserialize)] pub struct SetEnabled { diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 5032bfe5db..abd0b4201b 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -6,1308 +6,22 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::{ - auth::AuthCache, - auth::{list_tokens_internal, TruncatedTokenWithEmail}, - db::{ApiAuthed, DB}, - schedule::clear_schedule, - triggers::{get_triggers_count_internal, TriggersCount}, - users::{maybe_refresh_folders, require_owner_of_path}, - utils::{check_scopes, BulkDeleteRequest, WithStarredInfoQuery}, - webhook_util::{WebhookMessage, WebhookShared}, - HTTP_CLIENT, -}; -use axum::extract::Multipart; +// Re-export everything from windmill-api-scripts +pub use windmill_api_scripts::scripts::*; +use crate::triggers::{get_triggers_count_internal, TriggersCount}; use axum::{ - extract::{Extension, Path, Query}, - response::IntoResponse, - routing::{delete, get, post}, - Json, Router, + extract::{Extension, Path}, + routing::get, + Router, }; -use futures::future::try_join_all; -use http::header; -use hyper::StatusCode; -use itertools::Itertools; -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use serde_json::value::RawValue; -use sql_builder::prelude::*; -use sqlx::{FromRow, Postgres, Transaction}; -use std::{collections::HashMap, sync::Arc}; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_worker::{process_relative_imports, scoped_dependency_map::ScopedDependencyMap}; - -use windmill_common::{ - assets::{ - clear_static_asset_usage, clear_static_asset_usage_by_script_hash, - insert_static_asset_usage, AssetUsageKind, AssetWithAltAccessType, - }, - error::{self, to_anyhow}, - min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2}, - runnable_settings::{ - min_version_supports_runnable_settings_v0, RunnableSettings, RunnableSettingsTrait, - }, - s3_helpers::upload_artifact_to_store, - scripts::{hash_script, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, - utils::{paginate_without_limits, WarnAfterExt}, - worker::CLOUD_HOSTED, -}; - -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - jobs::JobPayload, - schedule::Schedule, - schema::should_validate_schema, - scripts::{ - to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash, - ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred, - }, - users::username_to_permissioned_as, - utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath}, - worker::to_raw_value, - HUB_BASE_URL, -}; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; -use windmill_parser_ts::remove_pinned_imports; -use windmill_queue::{ - schedule::push_scheduled_job, PushIsolationLevel, WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT, -}; - -const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20; - -#[derive(Serialize, sqlx::FromRow)] -pub struct ScriptWDraft { - pub hash: ScriptHash, - pub path: String, - pub summary: String, - pub description: String, - pub content: String, - pub language: ScriptLang, - pub kind: ScriptKind, - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft: Option>>, - pub schema: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub envs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dedicated_worker: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub restart_unless_cancelled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub visible_to_runner_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub no_main_func: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub has_preprocessor: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_behalf_of_email: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(json(nullable))] - pub assets: Option>, - #[serde(flatten)] - #[sqlx(flatten)] - pub runnable_settings: SR, -} - -impl ScriptWDraft { - pub async fn prefetch_cached<'a>( - self, - db: &DB, - ) -> error::Result> { - let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - self.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; - - Ok(ScriptWDraft { - runnable_settings: ScriptRunnableSettingsInline { - concurrency_settings: concurrency_settings.maybe_fallback( - self.runnable_settings.concurrency_key, - self.runnable_settings.concurrent_limit, - self.runnable_settings.concurrency_time_window_s, - ), - debouncing_settings: debouncing_settings.maybe_fallback( - self.runnable_settings.debounce_key, - self.runnable_settings.debounce_delay_s, - ), - }, - hash: self.hash, - path: self.path, - summary: self.summary, - description: self.description, - content: self.content, - language: self.language, - kind: self.kind, - tag: self.tag, - draft: self.draft, - schema: self.schema, - draft_only: self.draft_only, - envs: self.envs, - cache_ttl: self.cache_ttl, - cache_ignore_s3_path: self.cache_ignore_s3_path, - dedicated_worker: self.dedicated_worker, - ws_error_handler_muted: self.ws_error_handler_muted, - priority: self.priority, - restart_unless_cancelled: self.restart_unless_cancelled, - delete_after_use: self.delete_after_use, - timeout: self.timeout, - visible_to_runner_only: self.visible_to_runner_only, - no_main_func: self.no_main_func, - has_preprocessor: self.has_preprocessor, - on_behalf_of_email: self.on_behalf_of_email, - assets: self.assets, - }) - } -} - -pub fn global_service() -> Router { - Router::new() - .route("/hub/top", get(get_top_hub_scripts)) - .route("/hub/get/*path", get(get_hub_script_by_path)) - .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) - .route("/hub/pick/*path", get(pick_hub_script_by_path)) -} - -pub fn global_unauthed_service() -> Router { - Router::new() - .route( - "/tokened_raw/:workspace/:token/*path", - get(get_tokened_raw_script_by_path), - ) - .route("/empty_ts/*path", get(get_empty_ts_script_by_path)) -} +use windmill_common::{error::JsonResult, utils::StripPath, DB}; +/// Wraps the subcrate's workspaced_service with the trigger count route +/// that depends on windmill-api internals. pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_scripts)) - .route("/list_search", get(list_search_scripts)) - .route("/create", post(create_script)) - .route("/create_snapshot", post(create_snapshot_script)) - .route("/archive/p/*path", post(archive_script_by_path)) - .route("/get/draft/*path", get(get_script_by_path_w_draft)) - .route("/get/p/*path", get(get_script_by_path)) + windmill_api_scripts::scripts::workspaced_service() .route("/get_triggers_count/*path", get(get_triggers_count)) - .route("/list_tokens/*path", get(list_tokens)) - .route("/raw/p/*path", get(raw_script_by_path)) - .route("/raw_unpinned/p/*path", get(raw_script_by_path_unpinned)) - .route("/exists/p/*path", get(exists_script_by_path)) - .route("/archive/h/:hash", post(archive_script_by_hash)) - .route("/delete/h/:hash", post(delete_script_by_hash)) - .route("/delete/p/*path", post(delete_script_by_path)) - .route("/delete_bulk", delete(delete_scripts_bulk)) - .route("/get/h/:hash", get(get_script_by_hash)) - .route("/raw/h/:hash", get(raw_script_by_hash)) - .route("/deployment_status/h/:hash", get(get_deployment_status)) - .route("/list_paths", get(list_paths)) - .route( - "/toggle_workspace_error_handler/p/*path", - post(toggle_workspace_error_handler), - ) - .route("/history/p/*path", get(get_script_history)) - .route("/get_latest_version/*path", get(get_latest_version)) - .route( - "/list_paths_from_workspace_runnable/*path", - get(list_paths_from_workspace_runnable), - ) - .route( - "/history_update/h/:hash/p/*path", - post(update_script_history), - ) -} - -#[derive(Serialize, FromRow)] -pub struct SearchScript { - path: String, - content: String, -} -async fn list_search_scripts( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - #[cfg(feature = "enterprise")] - let n = 10000; - - #[cfg(not(feature = "enterprise"))] - let n = 10; - - let rows = sqlx::query_as!( - SearchScript, - "SELECT path, content from script WHERE workspace_id = $1 AND archived = false LIMIT $2", - &w_id, - n - ) - .fetch_all(&mut *tx) - .await? - .into_iter() - .collect::>(); - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_scripts( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, - Query(lq): Query, -) -> JsonResult> { - let (per_page, offset) = paginate_without_limits(pagination); - let mut sqlb = SqlBuilder::select_from("script as o") - .fields(&[ - "hash", - "o.path", - "summary", - "COALESCE(draft.created_at, o.created_at) as created_at", - "archived", - "extra_perms", - if !lq.without_description.unwrap_or(false) { - "description" - } else { - "NULL as description" - }, - "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors", - "language", - "favorite.path IS NOT NULL as starred", - "tag", - "draft.path IS NOT NULL as has_draft", - "draft_only", - "ws_error_handler_muted", - "no_main_func", - "codebase IS NOT NULL as use_codebase", - "kind" - ]) - .left() - .join("favorite") - .on( - "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?" - .bind(&authed.username), - ) - .left() - .join("draft") - .on( - "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'" - ) - .order_desc("favorite.path IS NOT NULL") - .order_by("created_at", lq.order_desc.unwrap_or(true)) - .and_where("o.workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page) - .clone(); - - let lowercased_kinds: Option> = lq - .kinds - .map(|x| x.split(",").map(&str::to_lowercase).collect()); - - if (!lq.include_without_main.unwrap_or(false) - && lowercased_kinds - .as_ref() - .map(|x| !x.contains(&"preprocessor".to_string())) - .unwrap_or(true)) - || authed.is_operator - { - // only include scripts that have a main function - // do not hide scripts without main if preprocessor is in the kinds - sqlb.and_where("o.no_main_func IS NOT TRUE"); - } - - if !lq.include_draft_only.unwrap_or(false) || authed.is_operator { - sqlb.and_where("draft_only IS NOT TRUE"); - } - - if lq.show_archived.unwrap_or(false) { - sqlb.and_where_eq( - "o.ctid", - "(SELECT ctid FROM script - WHERE path = o.path - AND workspace_id = ? - ORDER BY created_at DESC - LIMIT 1)" - .bind(&w_id), - ); - sqlb.and_where_eq("archived", true); - } else { - sqlb.and_where_eq("archived", false); - } - if let Some(ps) = &lq.path_start { - sqlb.and_where_like_left("o.path", ps); - } - if let Some(p) = &lq.path_exact { - sqlb.and_where_eq("o.path", "?".bind(p)); - } - if let Some(cb) = &lq.created_by { - sqlb.and_where_eq("created_by", "?".bind(cb)); - } - if let Some(ph) = &lq.first_parent_hash { - sqlb.and_where_eq("parent_hashes[1]", &ph.0); - } - if let Some(ph) = &lq.last_parent_hash { - sqlb.and_where_eq("parent_hashes[array_upper(parent_hashes, 1)]", &ph.0); - } - if let Some(ph) = &lq.parent_hash { - sqlb.and_where_eq("any(parent_hashes)", &ph.0); - } - if let Some(it) = &lq.is_template { - sqlb.and_where_eq("is_template", it); - } - if let Some(dw) = &lq.dedicated_worker { - sqlb.and_where_eq("dedicated_worker", dw); - } - if authed.is_operator { - sqlb.and_where_eq("kind", quote("script")); - } else if let Some(lowercased_kinds) = lowercased_kinds { - let safe_kinds = lowercased_kinds - .into_iter() - .map(sql_builder::quote) - .collect_vec(); - if safe_kinds.len() > 0 { - sqlb.and_where_in("kind", safe_kinds.as_slice()); - } - } - if lq.starred_only.unwrap_or(false) { - sqlb.and_where_is_not_null("favorite.path"); - } - - if lq.with_deployment_msg.unwrap_or(false) { - sqlb.join("deployment_metadata dm") - .left() - .on("dm.script_hash = o.hash") - .fields(&["dm.deployment_msg"]); - } - - if let Some(languages) = lq.languages { - sqlb.and_where_in( - "language", - &languages - .iter() - .map(|language| quote(language.as_str())) - .collect_vec(), - ); - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as::<_, ListableScript>(&sql) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -#[derive(Deserialize)] -struct TopHubScriptsQuery { - limit: Option, - app: Option, - kind: Option, -} - -async fn get_top_hub_scripts( - Query(query): Query, - Extension(db): Extension, -) -> impl IntoResponse { - let mut query_params = vec![]; - if let Some(query_limit) = query.limit { - query_params.push(("limit", query_limit.to_string().clone())); - } - if let Some(query_app) = query.app { - query_params.push(("app", query_app.to_string().clone())); - } - if let Some(query_kind) = query.kind { - query_params.push(("kind", query_kind.to_string().clone())); - } - - let (status_code, headers, response) = query_elems_from_hub( - &HTTP_CLIENT, - &format!("{}/scripts/top", *HUB_BASE_URL.read().await), - Some(query_params), - &db, - ) - .await?; - Ok::<_, Error>((status_code, headers, response)) -} - -async fn create_snapshot_script( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Extension(db): Extension, - Path(w_id): Path, - mut multipart: Multipart, -) -> Result<(StatusCode, String)> { - // TODO: Check for debouncing here as well. - let mut script_hash = None; - let mut tx = None; - let mut uploaded = false; - let mut handle_deployment_metadata = None; - while let Some(field) = multipart.next_field().await.unwrap() { - let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); - if name == "script" { - let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap(); - let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar")); - let use_esm = ns.codebase.as_ref().is_some_and(|x| x.contains(".esm")); - let (new_hash, ntx, hdm) = create_script_internal( - ns, - w_id.clone(), - authed.clone(), - db.clone(), - user_db.clone(), - webhook.clone(), - ) - .await?; - let mut nh = new_hash.to_string(); - if use_esm { - nh = format!("{nh}.esm"); - } - if is_tar { - nh = format!("{nh}.tar"); - } - script_hash = Some(nh); - tx = Some(ntx); - handle_deployment_metadata = hdm; - } - if name == "file" { - let hash = script_hash.as_ref().ok_or_else(|| { - Error::BadRequest( - "script need to be passed first in the multipart upload".to_string(), - ) - })?; - - uploaded = true; - - let path = windmill_common::s3_helpers::bundle(&w_id, &hash); - upload_artifact_to_store( - &path, - data, - &windmill_common::worker::ROOT_STANDALONE_BUNDLE_DIR, - ) - .await?; - } - // println!("Length of `{}` is {} bytes", name, data.len()); - } - if !uploaded { - return Err(Error::BadRequest("No file uploaded".to_string())); - } - if script_hash.is_none() { - return Err(Error::BadRequest( - "No script found in the uploaded file".to_string(), - )); - } - - tx.unwrap().commit().await?; - if let Some(hdm) = handle_deployment_metadata { - hdm.handle(&db).await?; - } - return Ok((StatusCode::CREATED, format!("{}", script_hash.unwrap()))); -} - -async fn list_paths_from_workspace_runnable( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let runnables = sqlx::query_scalar!( - r#"SELECT importer_path FROM dependency_map - WHERE workspace_id = $1 AND imported_path = $2"#, - w_id, - path.to_path(), - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(runnables)) -} - -async fn create_script( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Extension(db): Extension, - Path(w_id): Path, - Json(ns): Json, -) -> Result<(StatusCode, String)> { - let (hash, tx, hdm) = - create_script_internal(ns, w_id, authed, db.clone(), user_db, webhook).await?; - tx.commit().await?; - if let Some(hdm) = hdm { - hdm.handle(&db).await?; - } - Ok((StatusCode::CREATED, format!("{}", hash))) -} - -struct HandleDeploymentMetadata { - email: String, - created_by: String, - w_id: String, - obj: DeployedObject, - deployment_message: Option, - renamed_from: Option, -} - -impl HandleDeploymentMetadata { - async fn handle(self, db: &DB) -> Result<()> { - handle_deployment_metadata( - &self.email, - &self.created_by, - &db, - &self.w_id, - self.obj, - self.deployment_message, - false, - self.renamed_from.as_deref(), - ) - .await - } -} - -async fn create_script_internal<'c>( - ns: NewScript, - w_id: String, - authed: ApiAuthed, - db: sqlx::Pool, - user_db: UserDB, - webhook: WebhookShared, -) -> Result<( - ScriptHash, - Transaction<'c, Postgres>, - Option, -)> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot create scripts for security reasons".to_string(), - )); - } - check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; - - guard_script_from_debounce_data(&ns).await?; - - let codebase = ns.codebase.as_ref(); - #[cfg(not(feature = "enterprise"))] - if ns.ws_error_handler_muted.is_some_and(|val| val) { - return Err(Error::BadRequest( - "Muting the error handler for certain script is only available in enterprise version" - .to_string(), - )); - } - if *CLOUD_HOSTED { - let nb_scripts = - sqlx::query_scalar!("SELECT COUNT(*) FROM script WHERE workspace_id = $1", &w_id) - .fetch_one(&db) - .await?; - if nb_scripts.unwrap_or(0) >= 5000 { - return Err(Error::BadRequest( - "You have reached the maximum number of scripts (5000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - - if ns.summary.len() > 300 { - return Err(Error::BadRequest( - "Summary must be less than 300 characters on cloud".to_string(), - )); - } - if ns.description.len() > 3000 { - return Err(Error::BadRequest( - "Description must be less than 3000 characters on cloud".to_string(), - )); - } - } - let script_path = ns.path.clone(); - let hash = ScriptHash(hash_script(&ns)); - let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await; - let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - if sqlx::query_scalar!( - "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2", - hash.0, - &w_id - ) - .fetch_optional(&mut *tx) - .await? - .is_some() - { - return Err(Error::BadRequest( - "A script with same hash (hence same path, description, summary, content) already \ - exists!" - .to_owned(), - )); - }; - let clashing_script = sqlx::query_as::<_, Script>( - "SELECT * FROM script WHERE path = $1 AND archived = false AND workspace_id = $2", - ) - .bind(&ns.path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - struct ParentInfo { - p_hashes: Vec, - perms: serde_json::Value, - p_path: String, - } - let parent_hashes_and_perms: Option = match (&ns.parent_hash, clashing_script) { - (None, None) => Ok(None), - (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!( - "Path conflict for {} with non-archived hash {}", - &ns.path, &s.hash - ))), - (None, Some(s)) => { - sqlx::query!( - "DELETE FROM script WHERE hash = $1 AND workspace_id = $2", - s.hash.0, - &w_id - ) - .execute(&mut *tx) - .await?; - Ok(None) - } - (Some(p_hash), o) => { - // Lock the parent row to prevent concurrent updates with the same parent_hash - // This ensures linear lineage - only one script can have a given parent at a time - if sqlx::query_scalar!( - "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2 FOR UPDATE", - p_hash.0, - &w_id - ) - .fetch_optional(&mut *tx) - .await? - .is_none() - { - return Err(Error::BadRequest( - "The parent hash does not seem to exist".to_owned(), - )); - }; - - let clashing_hash_o = sqlx::query_scalar!( - "SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2", - p_hash.0, - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - if let Some(clashing_hash) = clashing_hash_o { - return Err(Error::BadRequest(format!( - "A script with hash {} with same parent_hash has been found. However, the \ - lineage must be linear: no 2 scripts can have the same parent", - ScriptHash(clashing_hash) - ))); - }; - - let ScriptWithStarred { script: ps, .. } = - get_script_by_hash_internal(&mut tx, &w_id, p_hash, None).await?; - - if ps.path != ns.path { - require_owner_of_path(&authed, &ps.path)?; - } - - let ph = { - let v = ps.parent_hashes.map(|x| x.0).unwrap_or_default(); - let mut v: Vec = v - .into_iter() - .take(MAX_HASH_HISTORY_LENGTH_STORED - 1) - .collect(); - v.insert(0, p_hash.0); - v - }; - let r: Result> = match o { - Some(clashing_script) - if clashing_script.path == ns.path && clashing_script.hash.0 != p_hash.0 => - { - Err(Error::BadRequest(format!( - "Path conflict for {} with non-archived hash {}", - &ns.path, &clashing_script.hash - ))) - } - Some(_) | None => Ok(Some(ParentInfo { - p_hashes: ph, - perms: ps.extra_perms, - p_path: ps.path, - })), - }; - sqlx::query!( - "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2", - p_hash.0, - &w_id - ) - .execute(&mut *tx) - .await?; - - clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - - r - } - }?; - let p_hashes = parent_hashes_and_perms.as_ref().map(|v| &v.p_hashes[..]); - let extra_perms = parent_hashes_and_perms - .as_ref() - .map(|v| v.perms.clone()) - .unwrap_or(json!({})); - let lock = if ns.codebase.is_some() { - Some(String::new()) - } else if !( - ns.language == ScriptLang::Python3 - || ns.language == ScriptLang::Go - || ns.language == ScriptLang::Bun - || ns.language == ScriptLang::Bunnative - || ns.language == ScriptLang::Deno - || ns.language == ScriptLang::Rust - || ns.language == ScriptLang::Ansible - || ns.language == ScriptLang::CSharp - || ns.language == ScriptLang::Nu - || ns.language == ScriptLang::Php - || ns.language == ScriptLang::Java - || ns.language == ScriptLang::Ruby - // for related places search: ADD_NEW_LANG - ) { - Some(String::new()) - } else { - ns.lock.as_ref().and_then(|e| { - if e.is_empty() { - None - } else { - Some(e.to_string()) - } - }) - }; - - let needs_lock_gen = lock.is_none() && codebase.is_none(); - let envs = ns.envs.as_ref().map(|x| x.as_slice()); - let envs = if ns.envs.is_none() || ns.envs.as_ref().unwrap().is_empty() { - None - } else { - envs - }; - - let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative { - let anns = windmill_common::worker::TypeScriptAnnotations::parse(&ns.content); - if anns.native { - ScriptLang::Bunnative - } else { - ScriptLang::Bun - } - } else { - ns.language.clone() - }; - - let validate_schema = should_validate_schema(&ns.content, &ns.language); - - let (no_main_func, has_preprocessor) = if matches!(ns.kind, Some(ScriptKind::Preprocessor)) { - (ns.no_main_func, ns.has_preprocessor) - } else { - match lang { - ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { - let args = windmill_parser_ts::parse_deno_signature(&ns.content, true, true, None); - match args { - Ok(args) => (args.no_main_func, args.has_preprocessor), - Err(e) => { - tracing::warn!( - "Error parsing deno signature when deploying script {}: {:?}", - ns.path, - e - ); - (None, None) - } - } - } - ScriptLang::Python3 => { - let args = windmill_parser_py::parse_python_signature(&ns.content, None, true); - match args { - Ok(args) => (args.no_main_func, args.has_preprocessor), - Err(e) => { - tracing::warn!( - "Error parsing python signature when deploying script {}: {:?}", - ns.path, - e - ); - (None, None) - } - } - } - _ => (ns.no_main_func, ns.has_preprocessor), - } - }; - - let runnable_settings_handle = RunnableSettings { - debouncing_settings: ns.debouncing_settings.insert_cached(&db).await?, - concurrency_settings: ns.concurrency_settings.insert_cached(&db).await?, - } - .insert_cached(&db) - .await?; - - let ( - guarded_concurrent_limit, - guarded_concurrency_time_window_s, - guarded_concurrency_key, - guarded_debounce_key, - guarded_debounce_delay_s, - ) = if min_version_supports_runnable_settings_v0().await { - Default::default() - } else { - ( - ns.concurrency_settings.concurrent_limit.clone(), - ns.concurrency_settings.concurrency_time_window_s.clone(), - ns.concurrency_settings.concurrency_key.clone(), - ns.debouncing_settings.debounce_key.clone(), - ns.debouncing_settings.debounce_delay_s.clone(), - ) - }; - - sqlx::query!( - "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ - content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ - draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ - dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38)", - &w_id, - &hash.0, - ns.path, - p_hashes, - ns.summary, - ns.description, - &ns.content, - &authed.username, - ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()), - ns.is_template.unwrap_or(false), - extra_perms, - lock, - lang as ScriptLang, - ns.kind.unwrap_or(ScriptKind::Script) as ScriptKind, - ns.tag, - ns.draft_only, - envs, - guarded_concurrent_limit, - guarded_concurrency_time_window_s, - ns.cache_ttl, - ns.dedicated_worker, - ns.ws_error_handler_muted.unwrap_or(false), - ns.priority, - ns.restart_unless_cancelled, - ns.delete_after_use, - ns.timeout, - guarded_concurrency_key, - ns.visible_to_runner_only, - no_main_func.filter(|x| *x), // should be Some(true) or None - codebase, - has_preprocessor.filter(|x| *x), // should be Some(true) or None - if ns.on_behalf_of_email.is_some() { - Some(&authed.email) - } else { - None - }, - validate_schema, - ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()), - guarded_debounce_key, - guarded_debounce_delay_s, - ns.cache_ignore_s3_path, - runnable_settings_handle - ) - .execute(&mut *tx) - .await?; - - let p_path_opt = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()); - if let Some(ref p_path) = p_path_opt { - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", - p_path, - &w_id - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", - ns.path, - p_path, - w_id - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS FALSE", - ns.path, - p_path, - w_id - ) - .execute(&mut *tx) - .await?; - - let mut schedulables = sqlx::query_as::<_, Schedule>( - "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS false RETURNING *") - .bind(&ns.path) - .bind(&p_path) - .bind(&w_id) - .fetch_all(&mut *tx) - .await?; - - let schedule = sqlx::query_as::<_, Schedule>( - "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *") - .bind(&ns.path) - .bind(&p_path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - if let Some(schedule) = schedule { - schedulables.push(schedule); - } - - // Update dynamic_skip references when script is renamed - sqlx::query!( - "UPDATE schedule SET dynamic_skip = $1 WHERE dynamic_skip = $2 AND workspace_id = $3", - &ns.path, - &p_path, - &w_id - ) - .execute(&mut *tx) - .await?; - - for schedule in schedulables { - clear_schedule(&mut tx, &schedule.path, &w_id).await?; - - if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; - } - } - } else { - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", - ns.path, - &w_id - ) - .execute(&mut *tx) - .await?; - } - if p_hashes.is_some() && !p_hashes.unwrap().is_empty() { - audit_log( - &mut *tx, - &authed, - "scripts.update", - ActionKind::Update, - &w_id, - Some(&ns.path), - Some([("hash", hash.to_string().as_str())].into()), - ) - .await?; - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateScript { - workspace: w_id.clone(), - path: ns.path.clone(), - hash: hash.to_string(), - }, - ); - } else { - audit_log( - &mut *tx, - &authed, - "scripts.create", - ActionKind::Create, - &w_id, - Some(&ns.path), - Some( - [ - ("workspace", w_id.as_str()), - ("hash", hash.to_string().as_str()), - ] - .into(), - ), - ) - .await?; - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateScript { - workspace: w_id.clone(), - path: ns.path.clone(), - hash: hash.to_string(), - }, - ); - } - - clear_static_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; - for asset in ns.assets.as_ref().into_iter().flatten() { - insert_static_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script) - .await?; - } - - let permissioned_as = username_to_permissioned_as(&authed.username); - if let Some(parent_hash) = ns.parent_hash { - tracing::info!( - "creating script {hash:?} at path {script_path} with parent {parent_hash} on workspace {w_id}", - ); - } else { - tracing::info!("creating script {hash:?} at path {script_path} on workspace {w_id}",); - } - if needs_lock_gen { - let tag = if ns.dedicated_worker.is_some_and(|x| x) { - Some(format!("{}:{}", &w_id, &ns.path,)) - } else if ns.tag.as_ref().is_some_and(|x| x.contains("$args[")) { - None - } else { - ns.tag - }; - - let mut args: HashMap> = HashMap::new(); - if let Some(dm) = ns.deployment_message { - args.insert("deployment_message".to_string(), to_raw_value(&dm)); - } - if let Some(ref p_path) = p_path_opt { - args.insert("parent_path".to_string(), to_raw_value(&p_path)); - } - - let tx = PushIsolationLevel::Transaction(tx); - let (job_id, mut new_tx) = windmill_queue::push( - &db, - tx, - &w_id, - JobPayload::Dependencies { - hash, - language: ns.language, - path: ns.path.clone(), - dedicated_worker: ns.dedicated_worker, - debouncing_settings: Default::default(), - }, - windmill_queue::PushArgs::from(&args), - &authed.username, - &authed.email, - permissioned_as, - authed.token_prefix.as_deref(), - None, - None, - None, - None, - None, - None, - false, - false, - None, - true, - tag, - None, - None, - None, - Some(&authed.clone().into()), - false, - None, - None, - None, - ) - .await?; - - // Store the job_id in deployment_metadata for this script deployment - sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, script_hash, job_id) - VALUES ($1, $2, $3, $4) - ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL - DO UPDATE SET job_id = EXCLUDED.job_id", - w_id, - ns.path, - hash.0, - job_id - ) - .execute(&mut *new_tx) - .await?; - - Ok((hash, new_tx, None)) - } else { - if codebase.is_none() { - let db2 = db.clone(); - let w_id2 = w_id.clone(); - let authed2 = authed.clone(); - let permissioned_as2 = permissioned_as.clone(); - let script_path2 = script_path.clone(); - let parent_path = p_path_opt.clone(); - let deployment_message = ns.deployment_message.clone(); - let content = ns.content.clone(); - let language = ns.language.clone(); - tokio::spawn(async move { - // wait for 10 seconds to make sure the script is deployed and that the CLI sync that pushed it (f one) is complete - tokio::time::sleep(std::time::Duration::from_secs(10)).await; - if let Err(e) = process_relative_imports( - &db2, - None, - None, - &w_id2, - &script_path2, - parent_path, - deployment_message, - &content, - &Some(language), - &authed2.email, - &authed2.username, - &permissioned_as2, - ) - .await - { - tracing::error!(%e, "error processing relative imports"); - } - }); - } - - // handle_deployment_metadata( - // &authed.email, - // &authed.username, - // &db, - // &w_id, - // DeployedObject::Script { - // hash: hash.clone(), - // path: script_path.clone(), - // parent_path: p_path_opt, - // }, - // ns.deployment_message, - // false, - // ) - // .await?; - - Ok(( - hash, - tx, - Some(HandleDeploymentMetadata { - email: authed.email, - created_by: authed.username, - w_id, - obj: DeployedObject::Script { - hash: hash.clone(), - path: script_path.clone(), - parent_path: p_path_opt.clone(), - }, - deployment_message: ns.deployment_message, - renamed_from: p_path_opt, - }), - )) - } -} - -pub async fn get_hub_script_by_path( - Path(path): Path, - Extension(db): Extension, -) -> Result { - windmill_common::scripts::get_hub_script_by_path(path, &HTTP_CLIENT, &db).await -} - -pub async fn get_full_hub_script_by_path( - Path(path): Path, - Extension(db): Extension, -) -> JsonResult { - Ok(Json( - windmill_common::scripts::get_full_hub_script_by_path(path, &HTTP_CLIENT, Some(&db)) - .await?, - )) -} - -pub async fn pick_hub_script_by_path( - Path(path): Path, - Extension(db): Extension, -) -> impl IntoResponse { - let path_str = path.to_path(); - - // Extract version_id from path (format: {hub}/{version_id}/{summary}) - let version_id = path_str.split('/').nth(1).unwrap_or(""); - - let hub_base_url = HUB_BASE_URL.read().await.clone(); - - // Determine which hub to use based on version_id - // If version_id < PRIVATE_HUB_MIN_VERSION, use default hub - let target_hub_url = if version_id - .parse::() - .is_ok_and(|v| v < windmill_common::PRIVATE_HUB_MIN_VERSION) - { - windmill_common::DEFAULT_HUB_BASE_URL - } else { - &hub_base_url - }; - - // Call the hub's pick endpoint: /scripts/{version_id}/pick - let (status_code, headers, response) = query_elems_from_hub( - &HTTP_CLIENT, - &format!("{}/scripts/{}/pick", target_hub_url, version_id), - None, - &db, - ) - .await?; - - Ok::<_, Error>((status_code, headers, response)) -} - -#[axum::debug_handler] -async fn get_script_by_path( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("scripts:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let script_o = if query.with_starred_info.unwrap_or(false) { - sqlx::query_as::<_, ScriptWithStarred>( - "SELECT s.*, favorite.path IS NOT NULL as starred - FROM script s - LEFT JOIN favorite - ON favorite.favorite_kind = 'script' - AND favorite.workspace_id = s.workspace_id - AND favorite.path = s.path - AND favorite.usr = $3 - WHERE s.path = $1 - AND s.workspace_id = $2 - ORDER BY s.created_at DESC LIMIT 1", - ) - .bind(path) - .bind(w_id) - .bind(&authed.username) - .fetch_optional(&mut *tx) - .await? - } else { - sqlx::query_as::<_, ScriptWithStarred>( - "SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", - ) - .bind(path) - .bind(w_id) - .fetch_optional(&mut *tx) - .await? - }; - tx.commit().await?; - - let script = not_found_if_none(script_o, "Script", path)? - .prefetch_cached(&db) - .await?; - - Ok(Json(script)) -} - -async fn list_tokens( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - list_tokens_internal(&db, &w_id, &path, false).await } async fn get_triggers_count( @@ -1317,1012 +31,3 @@ async fn get_triggers_count( let path = path.to_path(); get_triggers_count_internal(&db, &w_id, &path, false).await } - -async fn get_script_by_path_w_draft( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("scripts:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON - script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' - WHERE script.path = $1 AND script.workspace_id = $2 - ORDER BY script.created_at DESC LIMIT 1", - ) - .bind(path) - .bind(w_id) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let script = not_found_if_none(script_o, "Script", path)?; - Ok(Json(script.prefetch_cached(&db).await?)) -} - -async fn get_script_history( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("scripts:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let query_result = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg - FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash - WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by s.created_at DESC", - w_id, - path, - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - let result: Vec = query_result - .into_iter() - .map(|row| ScriptHistory { - script_hash: ScriptHash(row.hash), - deployment_msg: row.deployment_msg, - }) - .collect(); - return Ok(Json(result)); -} - -async fn get_latest_version( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("scripts:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let row_o = sqlx::query!( - "SELECT s.hash as hash, dm.deployment_msg as deployment_msg - FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash - WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by s.created_at DESC LIMIT 1", - w_id, - path, - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - if let Some(row) = row_o { - let result = ScriptHistory { - script_hash: ScriptHash(row.hash), - deployment_msg: row.deployment_msg, // - }; - return Ok(Json(Some(result))); - } else { - return Ok(Json(None)); - } -} - -async fn update_script_history( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, script_hash, script_path)): Path<(String, ScriptHash, StripPath)>, - Json(script_history_update): Json, -) -> Result<()> { - let script_path = script_path.to_path(); - check_scopes(&authed, || format!("scripts:write:{}", script_path))?; - - let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL - DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", - w_id, - script_path, - script_hash.0, - script_history_update.deployment_msg, - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - return Ok(()); -} - -async fn list_paths( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - - let scripts = sqlx::query_scalar!( - "SELECT distinct(path) FROM script WHERE workspace_id = $1", - w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(scripts)) -} - -#[derive(Deserialize)] -pub struct ToggleWorkspaceErrorHandler { - #[cfg(feature = "enterprise")] - pub muted: Option, -} - -#[cfg(not(feature = "enterprise"))] -async fn toggle_workspace_error_handler( - _authed: ApiAuthed, - Extension(_user_db): Extension, - Path((_w_id, _path)): Path<(String, StripPath)>, - Json(_req): Json, -) -> Result { - return Err(Error::BadRequest( - "Muting the error handler for certain script is only available in enterprise version" - .to_string(), - )); -} - -#[cfg(feature = "enterprise")] -async fn toggle_workspace_error_handler( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(req): Json, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - let error_handler_maybe: Option = sqlx::query_scalar!( - "SELECT error_handler->>'path' FROM workspace_settings WHERE workspace_id = $1", - w_id - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(None); - - match error_handler_maybe { - Some(_) => { - sqlx::query_scalar!( - "UPDATE script - SET ws_error_handler_muted = $3 - WHERE ctid = ( - SELECT ctid FROM script - WHERE path = $1 AND workspace_id = $2 - ORDER BY created_at DESC - LIMIT 1 - ) -", - path.to_path(), - w_id, - req.muted, - ) - .execute(&mut *tx) - .await?; - tx.commit().await?; - Ok("".to_string()) - } - None => { - tx.commit().await?; - Err(Error::ExecutionErr( - "Workspace error handler needs to be defined".to_string(), - )) - } - } -} - -async fn get_tokened_raw_script_by_path( - Extension(user_db): Extension, - Extension(db): Extension, - Extension(cache): Extension>, - Path((w_id, token, path)): Path<(String, String, StripPath)>, - Query(query): Query, -) -> Result { - let authed = cache - .get_authed(Some(w_id.clone()), &token) - .await - .ok_or_else(|| Error::NotAuthorized("Invalid token".to_string()))?; - return raw_script_by_path( - authed, - Extension(user_db), - Extension(db), - Path((w_id, path)), - Query(query), - ) - .await; -} - -async fn get_empty_ts_script_by_path() -> String { - return String::new(); -} - -#[derive(Deserialize)] -struct RawScriptByPathQuery { - // used to make cache immutable with respect to importer - cache_key: Option, - // used specifically for python to cache folders on import success to avoid extra db calls on package fetch - cache_folders: Option, -} - -struct StringWithLength(String); - -impl IntoResponse for StringWithLength { - fn into_response(self) -> axum::response::Response { - let len = self.0.len(); - ([(header::CONTENT_LENGTH, len.to_string())], self.0).into_response() - } -} - -async fn raw_script_by_path( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> Result { - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {}", path.to_path()); - } - let r = raw_script_by_path_internal(path, user_db, db, authed, w_id, false, query).await?; - Ok(StringWithLength(r)) -} - -async fn raw_script_by_path_unpinned( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> Result { - let r = raw_script_by_path_internal(path, user_db, db, authed, w_id, true, query).await?; - Ok(StringWithLength(r)) -} - -lazy_static::lazy_static! { - static ref DEBUG_RAW_SCRIPT_ENDPOINTS: bool = - std::env::var("DEBUG_RAW_SCRIPT_ENDPOINTS").is_ok(); -} - -lazy_static::lazy_static! { - pub static ref RAW_SCRIPT_CACHE: Cache = Cache::new(1000); - pub static ref CACHE_FOLDERS_PATH: Cache = Cache::new(1000); - -} - -async fn raw_script_by_path_internal( - path: StripPath, - user_db: UserDB, - db: DB, - authed: ApiAuthed, - w_id: String, - unpin: bool, - query: RawScriptByPathQuery, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("scripts:read:{}", path))?; - let cache_path = query - .cache_key - .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); - if let Some(cache_path) = cache_path.clone() { - let cached_content = RAW_SCRIPT_CACHE.get(&cache_path); - if let Some(cached_content) = cached_content { - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (cached)", path); - } - return Ok(cached_content); - } - } - - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (not cached)", path); - } - - if !path.ends_with(".py") - && !path.ends_with(".ts") - && !path.ends_with(".go") - && !path.ends_with(".sh") - { - return Err(Error::BadRequest(format!( - "Path must ends with a .py, .ts, .go. or .sh extension: {}", - path - ))); - } - let path = path - .trim_end_matches(".py") - .trim_end_matches(".bun.ts") - .trim_end_matches(".deno.ts") - .trim_end_matches(".ts") - .trim_end_matches(".go") - .trim_end_matches(".sh"); - - // folder cache is only useful for python given it needs to recuse over all intermediate folders to find the package. - // When a script exists in a folder, we can cache the fact that the folder exists to avoid extra db calls. - let mut split_path = path.split("/").collect::>(); - let folder_path = if query.cache_folders.is_some() && split_path.len() > 2 { - Some(format!("{w_id}:{path}/")) - } else { - None - }; - - let has_folder_cache = folder_path.is_some(); - if let Some(cache_folders) = folder_path { - let cached_content = CACHE_FOLDERS_PATH.get(&cache_folders); - if let Some(cached_ts) = cached_content { - if cached_ts >= chrono::Utc::now().timestamp() - 300 { - // 5 minutes - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (cached folders)", path); - } - return Ok("WINDMILL_IS_FOLDER".to_string()); - } else { - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!( - "Raw script by path request: {} (cached folders expired)", - path - ); - } - } - } - } - - let mut tx = user_db.begin(&authed).await?; - - let content_o = sqlx::query_scalar!( - "SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", - path, - w_id - ) - .fetch_optional(&mut *tx) - .warn_after_seconds(5) - .await?; - tx.commit().await?; - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!( - "Raw script by path request: {} (content: {:?})", - path, - content_o - ); - } - - if content_o.is_none() { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1)", - path, - w_id - ) - .fetch_one(&db) - .warn_after_seconds(5) - .await? - .unwrap_or(false); - - if exists { - return Err(Error::NotFound(format!( - "Script {path} exists but {} does not have permissions to access it", - authed.username - ))); - } else { - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - let other_script_o = sqlx::query_scalar!( - "SELECT path FROM script WHERE workspace_id = $1 AND archived = false", - w_id - ) - .fetch_all(&db) - .await?; - let other_script_archived = sqlx::query_scalar!( - "SELECT distinct(path) FROM script WHERE workspace_id = $1 AND archived = true", - w_id - ) - .fetch_all(&db) - .await?; - tracing::warn!( - "Script {path} does not exist in workspace {w_id} but these paths do, non-archived: {:?} | archived: {:?}", - other_script_o.join(", "), - other_script_archived.join(", ") - ) - } - } - } - - let content = not_found_if_none(content_o, "Script", path)?; - - let content = if unpin { - remove_pinned_imports(&content)? - } else { - content - }; - - if has_folder_cache { - while split_path.len() >= 2 { - split_path.pop(); - let npath = split_path.join("/"); - CACHE_FOLDERS_PATH.insert(format!("{w_id}:{npath}/"), chrono::Utc::now().timestamp()); - } - } - - if let Some(cache_path) = cache_path { - RAW_SCRIPT_CACHE.insert(cache_path, content.clone()); - } - if *DEBUG_RAW_SCRIPT_ENDPOINTS { - tracing::warn!("Raw script by path request: {} (content response)", path); - } - Ok(content) -} - -async fn exists_script_by_path( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn get_script_by_hash_internal<'c>( - db: &mut Transaction<'c, Postgres>, - workspace_id: &str, - hash: &ScriptHash, - with_starred_info_for_username: Option<&str>, -) -> Result> { - let script_o = if let Some(username) = with_starred_info_for_username { - sqlx::query_as::<_, ScriptWithStarred>( - "SELECT s.*, favorite.path IS NOT NULL as starred - FROM script s - LEFT JOIN favorite - ON favorite.favorite_kind = 'script' - AND favorite.workspace_id = s.workspace_id - AND favorite.path = s.path - AND favorite.usr = $1 - WHERE s.hash = $2 AND s.workspace_id = $3", - ) - .bind(&username) - .bind(hash) - .bind(workspace_id) - .fetch_optional(&mut **db) - .await? - } else { - sqlx::query_as::<_, ScriptWithStarred>( - "SELECT *, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2", - ) - .bind(hash) - .bind(workspace_id) - .fetch_optional(&mut **db) - .await? - }; - - let script = not_found_if_none(script_o, "Script", hash.to_string())?; - Ok(script) -} - -#[derive(Deserialize)] -struct GetScriptByHashQuery { - authed: Option, -} -async fn get_script_by_hash( - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, hash)): Path<(String, ScriptHash)>, - Query(query): Query, - Query(query_auth): Query, - Extension(authed): Extension, -) -> JsonResult> { - let mut tx = if query_auth.authed.is_some_and(|x| x) { - user_db.begin(&authed).await? - } else { - db.begin().await? - }; - let r = get_script_by_hash_internal( - &mut tx, - &w_id, - &hash, - query.with_starred_info.and_then(|x| { - if x { - Some(authed.username.as_str()) - } else { - None - } - }), - ) - .await?; - - check_scopes(&authed, || format!("scripts:read:{}", &r.script.path))?; - - tx.commit().await?; - - Ok(Json(r.prefetch_cached(&db).await?)) -} - -async fn raw_script_by_hash( - Extension(db): Extension, - Path((w_id, hash_str)): Path<(String, String)>, -) -> Result { - let mut tx = db.begin().await?; - let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| { - Error::BadRequest("Raw script path must end with .ts".to_string()) - })?)?); - let r = get_script_by_hash_internal(&mut tx, &w_id, &hash, None).await?; - tx.commit().await?; - - Ok(r.script.content) -} - -#[derive(Serialize)] -struct DeploymentStatus { - lock: Option, - lock_error_logs: Option, - job_id: Option, -} -async fn get_deployment_status( - Extension(db): Extension, - Path((w_id, hash)): Path<(String, ScriptHash)>, -) -> JsonResult { - let mut tx = db.begin().await?; - let status_o = sqlx::query!( - "SELECT s.lock, s.lock_error_logs, dm.job_id - FROM script s - LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash AND s.workspace_id = dm.workspace_id - WHERE s.hash = $1 AND s.workspace_id = $2", - hash.0, - w_id, - ) - .fetch_optional(&mut *tx) - .await?; - - let status = not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?; - - let deployment_status = DeploymentStatus { - lock: status.lock, - lock_error_logs: status.lock_error_logs, - job_id: status.job_id, - }; - - tx.commit().await?; - Ok(Json(deployment_status)) -} - -pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { - return crate::users::require_is_writer( - authed, - path, - w_id, - db, - "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", - "script", - ) - .await; -} - -async fn archive_script_by_path( - authed: ApiAuthed, - Extension(webhook): Extension, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result<()> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot archive scripts for security reasons".to_string(), - )); - } - let path = path.to_path(); - check_scopes(&authed, || format!("scripts:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - require_owner_of_path(&authed, path)?; - - let hash: i64 = sqlx::query_scalar!( - "UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash", - path, - &w_id - ) - .fetch_one(&db) - .await - .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; - - clear_static_asset_usage(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; - - audit_log( - &mut *tx, - &authed, - "scripts.archive", - ActionKind::Delete, - &w_id, - Some(&ScriptHash(hash).to_string()), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - - ScopedDependencyMap::clear_map_for_item(path, &w_id, "script", tx, &None) - .await - .commit() - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Script { - hash: ScriptHash(0), // dummy hash as it will not get inserted in db - path: path.to_string(), - parent_path: Some(path.to_string()), - }, - Some(format!("Script '{}' archived", path)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, - ); - - Ok(()) -} - -async fn archive_script_by_hash( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Extension(webhook): Extension, - Path((w_id, hash)): Path<(String, ScriptHash)>, -) -> JsonResult> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot archive scripts for security reasons".to_string(), - )); - } - let mut tx = user_db.begin(&authed).await?; - - let script = sqlx::query_as::<_, Script>( - "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2 RETURNING *", - ) - .bind(&hash.0) - .bind(&w_id) - .fetch_one(&mut *tx) - .await - .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; - - check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; - clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - - audit_log( - &mut *tx, - &authed, - "scripts.archive", - ActionKind::Delete, - &w_id, - Some(&hash.to_string()), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - - ScopedDependencyMap::clear_map_for_item(&script.path, &w_id, "script", tx, &None) - .await - .commit() - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, - ); - - Ok(Json(script.prefetch_cached(&db).await?)) -} - -async fn delete_script_by_hash( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Extension(db): Extension, - Path((w_id, hash)): Path<(String, ScriptHash)>, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - - require_admin(authed.is_admin, &authed.username)?; - let script = sqlx::query_as::<_, Script>( - "UPDATE script SET content = '', archived = true, deleted = true, lock = '', schema = null WHERE hash = $1 AND \ - workspace_id = $2 RETURNING *", - ) - .bind(&hash.0) - .bind(&w_id) - .fetch_one(&db) - .await - .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; - - check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; - - clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; - - audit_log( - &mut *tx, - &authed, - "scripts.delete", - ActionKind::Delete, - &w_id, - Some(&hash.to_string()), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - tx.commit().await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteScript { workspace: w_id, hash: hash.to_string() }, - ); - - Ok(Json(script.prefetch_cached(&db).await?)) -} - -#[derive(Deserialize)] -struct DeleteScriptQuery { - keep_captures: Option, -} - -async fn delete_script_by_path( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(webhook): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> JsonResult { - let path = path.to_path(); - - check_scopes(&authed, || format!("scripts:write:{}", path))?; - - if path == "u/admin/hub_sync" && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - - let mut tx = user_db.begin(&authed).await?; - - let draft_only = sqlx::query_scalar!( - "SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - let script = if !draft_only { - require_admin(authed.is_admin, &authed.username)?; - sqlx::query_scalar!( - "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id - ) - .fetch_one(&db) - .await - .map_err(|e| Error::internal_err(format!("deleting script by path {w_id}: {e:#}")))? - } else { - sqlx::query_scalar!( - "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id - ) - .fetch_one(&mut *tx) - .await - .map_err(|e| Error::internal_err(format!("deleting script by path {w_id}: {e:#}")))? - }; - - sqlx::query!( - "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'", - path, - w_id - ) - .execute(&db) - .await?; - - if !query.keep_captures.unwrap_or(false) { - sqlx::query!( - "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", - path, - w_id - ) - .execute(&db) - .await?; - - sqlx::query!( - "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", - path, - w_id - ) - .execute(&db) - .await?; - } - - audit_log( - &mut *tx, - &authed, - "scripts.delete", - ActionKind::Delete, - &w_id, - Some(&path), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Script { - hash: ScriptHash(0), // Temporary value as it will get removed right after - path: path.to_string(), - parent_path: Some(path.to_string()), - }, - Some(format!("Script '{}' deleted", path)), - true, - None, - ) - .await?; - - sqlx::query!( - "DELETE FROM deployment_metadata WHERE path = $1 AND workspace_id = $2 AND script_hash IS NOT NULL", - path, - w_id - ) - .execute(&db) - .await - .map_err(|e| { - Error::internal_err(format!( - "error deleting deployment metadata for script with path {path} in workspace {w_id}: {e:#}" - )) - })?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteScriptPath { workspace: w_id, path: path.to_owned() }, - ); - - Ok(Json(script)) -} - -async fn delete_scripts_bulk( - authed: ApiAuthed, - Extension(webhook): Extension, - Extension(db): Extension, - Path(w_id): Path, - Json(request): Json, -) -> JsonResult> { - for path in &request.paths { - check_scopes(&authed, || format!("scripts:write:{}", path))?; - } - - require_admin(authed.is_admin, &authed.username)?; - - if request.paths.contains(&"u/admin/hub_sync".to_string()) && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - - let mut tx = db.begin().await?; - - let mut deleted_paths = sqlx::query_scalar!( - "DELETE FROM script WHERE workspace_id = $1 AND path = ANY($2) RETURNING path", - w_id, - &request.paths - ) - .fetch_all(&mut *tx) - .await - .map_err(|e| Error::internal_err(format!("deleting scripts in bulk {w_id}: {e:#}")))?; - - // remove duplicates from deleted_paths - deleted_paths.sort(); - deleted_paths.dedup(); - - sqlx::query!( - "DELETE FROM draft WHERE workspace_id = $1 AND path = ANY($2) AND typ = 'script'", - w_id, - &deleted_paths - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM capture_config WHERE workspace_id = $1 AND path = ANY($2) AND is_flow IS FALSE", - w_id, - &deleted_paths - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM capture WHERE workspace_id = $1 AND path = ANY($2) AND is_flow IS FALSE", - w_id, - &deleted_paths - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "scripts.delete_bulk", - ActionKind::Delete, - &w_id, - Some(&deleted_paths.join(", ")), - Some([("workspace", w_id.as_str())].into()), - ) - .await?; - - tx.commit().await?; - - try_join_all(deleted_paths.iter().map(|path| { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Script { hash: ScriptHash(0), path: path.clone(), parent_path: None }, - Some(format!("Script '{}' deleted", path)), - true, - None, - ) - })) - .await?; - - sqlx::query!( - "DELETE FROM deployment_metadata WHERE workspace_id = $1 AND path = ANY($2) AND script_hash IS NOT NULL", - w_id, - &deleted_paths - ) - .execute(&db) - .await - .map_err(|e| { - Error::internal_err(format!( - "error deleting deployment metadata for scripts with paths {} in workspace {w_id}: {e:#}", deleted_paths.join(", ") - )) - })?; - - for path in &deleted_paths { - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteScriptPath { workspace: w_id.clone(), path: path.to_owned() }, - ); - } - - Ok(Json(deleted_paths)) -} - -/// Validates that script debouncing configuration is supported by all workers -/// Returns an error if debouncing is configured but workers are behind required version -async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { - if !MIN_VERSION_SUPPORTS_DEBOUNCING.met().await && !ns.debouncing_settings.is_default() { - tracing::warn!( - "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature" - ); - Err(Error::WorkersAreBehind { feature: "Debouncing".into(), min_version: "1.566.0".into() }) - } else if !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await - && !ns.debouncing_settings.is_legacy_compatible() - && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT - { - tracing::warn!( - "Script debouncing configuration rejected: workers are behind minimum required version for debouncing feature" - ); - Err(Error::WorkersAreBehind { - feature: "V2 Debouncing".into(), - min_version: "1.597.0".into(), - }) - } else { - Ok(()) - } -} diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs index 41d2267a94..2631e4fa37 100644 --- a/backend/windmill-api/src/secret_backend_ext.rs +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -14,14 +14,16 @@ //! Note: HashiCorp Vault integration requires Enterprise Edition. //! The OSS version only supports the database backend. +#[cfg(all(feature = "private", feature = "enterprise"))] use std::sync::Arc; -use windmill_common::{ - db::DB, - error::{Error, Result}, - secret_backend::{database::DatabaseBackend, SecretBackend}, - variables::{build_crypt, decrypt, encrypt}, -}; +use windmill_common::{db::DB, error::Result}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::error::Error; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::secret_backend::{database::DatabaseBackend, SecretBackend}; #[cfg(all(feature = "private", feature = "enterprise"))] use windmill_common::{ @@ -45,17 +47,9 @@ lazy_static::lazy_static! { static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); } -/// Get the current secret backend based on global settings -/// -/// OSS: Always returns DatabaseBackend -/// EE: Returns configured backend (Database or Vault) -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn get_secret_backend(db: &DB) -> Result> { - Ok(Arc::new(DatabaseBackend::new(db.clone()))) -} - +/// Get the current secret backend based on global settings (EE only) #[cfg(all(feature = "private", feature = "enterprise"))] -pub async fn get_secret_backend(db: &DB) -> Result> { +async fn get_secret_backend(db: &DB) -> Result> { let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { Some(value) => serde_json::from_value::(value).unwrap_or_default(), None => SecretBackendConfig::default(), @@ -109,25 +103,14 @@ async fn get_or_create_vault_backend( }; // Cache it - *cache = Some(CachedVaultBackend { - backend: backend.clone(), - settings, - }); + *cache = Some(CachedVaultBackend { backend: backend.clone(), settings }); Ok(backend) } -/// Check if a Vault backend is currently configured -/// -/// OSS: Always returns false -/// EE: Checks global settings -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn is_vault_backend_configured(_db: &DB) -> Result { - Ok(false) -} - +/// Check if a Vault backend is currently configured (EE only) #[cfg(all(feature = "private", feature = "enterprise"))] -pub async fn is_vault_backend_configured(db: &DB) -> Result { +async fn is_vault_backend_configured(db: &DB) -> Result { let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { Some(value) => serde_json::from_value::(value).unwrap_or_default(), None => SecretBackendConfig::default(), @@ -136,186 +119,10 @@ pub async fn is_vault_backend_configured(db: &DB) -> Result { Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) } -/// Get a secret value using the configured backend -/// -/// For database backend: decrypts using workspace key -/// For vault backend (EE only): fetches from Vault directly -pub async fn get_secret_value( - db: &DB, - workspace_id: &str, - path: &str, - encrypted_value: &str, -) -> Result { - let backend = get_secret_backend(db).await?; - - match backend.backend_name() { - "database" => { - // Use existing database decryption - let mc = build_crypt(db, workspace_id).await?; - decrypt(&mc, encrypted_value.to_string()).map_err(|e| { - Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) - }) - } - "hashicorp_vault" => { - // Fetch from Vault directly - backend.get_secret(workspace_id, path).await - } - _ => Err(Error::internal_err(format!( - "Unknown backend: {}", - backend.backend_name() - ))), - } -} - -/// Store a secret value using the configured backend -/// -/// For database backend: encrypts using workspace key and returns encrypted value -/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage -pub async fn store_secret_value( - db: &DB, - workspace_id: &str, - path: &str, - plain_value: &str, -) -> Result { - let backend = get_secret_backend(db).await?; - - match backend.backend_name() { - "database" => { - // Use existing database encryption - let mc = build_crypt(db, workspace_id).await?; - Ok(encrypt(&mc, plain_value)) - } - "hashicorp_vault" => { - // Store in Vault and return a marker for DB - backend.set_secret(workspace_id, path, plain_value).await?; - // Return a marker indicating the value is stored in Vault - // The actual value in the DB will be this marker - Ok(format!("$vault:{}", path)) - } - _ => Err(Error::internal_err(format!( - "Unknown backend: {}", - backend.backend_name() - ))), - } -} - -/// Delete a secret from the configured backend (if using Vault) -/// -/// For database backend: no-op (DB delete is handled separately) -/// For vault backend (EE only): deletes from Vault -pub async fn delete_secret_from_backend( - db: &DB, - workspace_id: &str, - path: &str, -) -> Result<()> { - if is_vault_backend_configured(db).await? { - let backend = get_secret_backend(db).await?; - // Ignore NotFound errors during deletion (secret might not exist in Vault) - match backend.delete_secret(workspace_id, path).await { - Ok(()) => Ok(()), - Err(Error::NotFound(_)) => Ok(()), - Err(e) => Err(e), - } - } else { - Ok(()) - } -} - /// Check if a value is stored in Vault (indicated by the $vault: prefix) -pub fn is_vault_stored_value(value: &str) -> bool { - value.starts_with("$vault:") -} - -/// Rename a secret in Vault when a variable path changes (EE only) -/// -/// This function: -/// 1. Reads the secret value from the old path -/// 2. Writes it to the new path -/// 3. Deletes from the old path -/// 4. Returns the new marker value ($vault:new_path) -/// -/// If the value is not a Vault-stored value, returns None (no action needed). -/// If Vault is not configured, returns None. -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn rename_vault_secret( - _db: &DB, - _workspace_id: &str, - _old_path: &str, - new_path: &str, - current_value: &str, -) -> Result> { - // OSS: If value has $vault: prefix, just update the reference - // (This handles edge case where EE was used before downgrading to OSS) - if is_vault_stored_value(current_value) { - tracing::warn!( - "Variable has $vault: prefix but Vault requires Enterprise Edition. \ - Updating DB reference to {}", - new_path - ); - return Ok(Some(format!("$vault:{}", new_path))); - } - Ok(None) -} - #[cfg(all(feature = "private", feature = "enterprise"))] -pub async fn rename_vault_secret( - db: &DB, - workspace_id: &str, - old_path: &str, - new_path: &str, - current_value: &str, -) -> Result> { - // Only handle Vault-stored values - if !is_vault_stored_value(current_value) { - return Ok(None); - } - - // Check if Vault backend is configured - if !is_vault_backend_configured(db).await? { - // Vault not configured but value has $vault: prefix - this is an inconsistent state - // Log warning and return new marker to at least update the DB reference - tracing::warn!( - "Variable value has $vault: prefix but Vault is not configured. \ - Updating DB reference from {} to {}", - old_path, - new_path - ); - return Ok(Some(format!("$vault:{}", new_path))); - } - - let backend = get_secret_backend(db).await?; - - // Read from old path - let secret_value = match backend.get_secret(workspace_id, old_path).await { - Ok(value) => value, - Err(Error::NotFound(_)) => { - // Secret doesn't exist in Vault - just update the DB reference - tracing::warn!( - "Secret not found in Vault at path {} during rename to {}", - old_path, - new_path - ); - return Ok(Some(format!("$vault:{}", new_path))); - } - Err(e) => return Err(e), - }; - - // Write to new path - backend - .set_secret(workspace_id, new_path, &secret_value) - .await?; - - // Delete from old path (ignore errors - new path is already written) - if let Err(e) = backend.delete_secret(workspace_id, old_path).await { - tracing::warn!( - "Failed to delete old secret at {} after rename to {}: {}", - old_path, - new_path, - e - ); - } - - Ok(Some(format!("$vault:{}", new_path))) +fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") } /// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) @@ -383,7 +190,10 @@ pub async fn rename_vault_secrets_with_prefix( }; // Write to new path - if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await { + if let Err(e) = backend + .set_secret(workspace_id, &new_path, &secret_value) + .await + { tracing::error!( "Failed to write secret to {} during bulk rename: {}", new_path, diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 5c43e33d30..d68797f7ee 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -58,6 +58,7 @@ pub fn global_service() -> Router { .route("/test_smtp", post(test_email)) .route("/test_license_key", post(test_license_key)) .route("/send_stats", post(send_stats)) + .route("/get_stats", get(get_stats)) .route( "/latest_key_renewal_attempt", get(get_latest_key_renewal_attempt), @@ -434,6 +435,25 @@ pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Resu Ok("Sent stats".to_string()) } +#[cfg(feature = "enterprise")] +pub async fn get_stats(Extension(db): Extension, authed: ApiAuthed) -> Result { + require_super_admin(&db, &authed.email).await?; + let stats = windmill_common::stats_oss::get_stats_payload( + &db, + &windmill_common::stats_oss::SendStatsReason::Manual, + ) + .await?; + let encrypted = windmill_common::stats_oss::encrypt_stats(&stats)?; + Ok(encrypted) +} + +#[cfg(not(feature = "enterprise"))] +pub async fn get_stats() -> Result { + Err(error::Error::BadRequest( + "Downloading telemetry is only available on enterprise edition".to_string(), + )) +} + #[derive(serde::Serialize)] pub struct KeyRenewalAttempt { result: String, diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 82c1b68d86..ed6368dada 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -11,13 +11,16 @@ use std::collections::HashMap; use windmill_common::error::Error; use windmill_common::variables::get_secret_value_as_admin; -use crate::{approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryDefaultArgsJson, QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, - ResumeFormField, ResumeSchema, -}, auth::OptTokened}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; +use crate::{ + approvals::{ + extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, + MessageFormat, QueryDefaultArgsJson, QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, + ResumeFormField, ResumeSchema, + }, + auth::OptTokened, +}; #[derive(Deserialize, Debug)] pub struct SlackFormData { @@ -125,8 +128,12 @@ pub async fn slack_app_callback_handler( tracing::debug!("Payload: {:#?}", payload); match payload.r#type { - PayloadType::ViewSubmission => handle_submission(authed, opt_tokened, db, &payload, "resume").await?, - PayloadType::ViewClosed => handle_submission(authed, opt_tokened, db, &payload, "cancel").await?, + PayloadType::ViewSubmission => { + handle_submission(authed, opt_tokened, db, &payload, "resume").await? + } + PayloadType::ViewClosed => { + handle_submission(authed, opt_tokened, db, &payload, "cancel").await? + } _ => { if let Some(actions) = &payload.actions { if let Some(action) = actions.first() { @@ -296,7 +303,15 @@ async fn handle_submission( } // Use the common handler to process the resume/cancel action - handle_resume_action(authed, opt_tokened, db.clone(), &resume_url, state_json, action).await?; + handle_resume_action( + authed, + opt_tokened, + db.clone(), + &resume_url, + state_json, + action, + ) + .await?; let w_id = extract_w_id_from_resume_url(&resume_url)?; let slack_token = get_slack_token(&db, &resource_path, w_id).await?; diff --git a/backend/windmill-api/src/teams_oss.rs b/backend/windmill-api/src/teams_oss.rs index 037e556c99..0d4fa7f17a 100644 --- a/backend/windmill-api/src/teams_oss.rs +++ b/backend/windmill-api/src/teams_oss.rs @@ -44,7 +44,6 @@ pub async fn run_teams_message_test_job() -> Result { )); } - #[cfg(all(feature = "enterprise", not(feature = "private")))] pub fn teams_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/triggers/email/mod.rs b/backend/windmill-api/src/triggers/email/mod.rs index 836af77fa0..1210a56460 100644 --- a/backend/windmill-api/src/triggers/email/mod.rs +++ b/backend/windmill-api/src/triggers/email/mod.rs @@ -1,11 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct EmailTrigger; +pub use windmill_trigger_email::*; diff --git a/backend/windmill-api/src/triggers/gcp/mod.rs b/backend/windmill-api/src/triggers/gcp/mod.rs index dd2f2a65df..c12f93724d 100644 --- a/backend/windmill-api/src/triggers/gcp/mod.rs +++ b/backend/windmill-api/src/triggers/gcp/mod.rs @@ -1,16 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Clone, Copy)] -pub struct GcpTrigger; +pub use windmill_trigger_gcp::*; diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs index 4ca7e2e15e..660ecf6051 100644 --- a/backend/windmill-api/src/triggers/handler.rs +++ b/backend/windmill-api/src/triggers/handler.rs @@ -1,685 +1,11 @@ -use crate::{ - db::ApiAuthed, - triggers::{StandardTriggerQuery, TriggerData, TriggerMode}, -}; -use async_trait::async_trait; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::{FromRow, PgConnection}; -use std::fmt::Debug; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::{paginate, Pagination, StripPath}, - worker::CLOUD_HOSTED, - DB, -}; -use windmill_git_sync::DeployedObject; - -use axum::{ - extract::{Path, Query}, - http::StatusCode, - routing::{delete, get, post}, - Extension, Json, Router, -}; -use std::sync::Arc; -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_git_sync::handle_deployment_metadata; - -use crate::utils::check_scopes; - -#[async_trait] -pub trait TriggerCrud: Send + Sync + 'static { - type Trigger: Serialize - + DeserializeOwned - + for<'r> FromRow<'r, sqlx::postgres::PgRow> - + Send - + Sync - + Unpin; - - type TriggerConfig: Debug - + DeserializeOwned - + for<'r> FromRow<'r, sqlx::postgres::PgRow> - + Serialize - + Send - + Sync - + Unpin; - - type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; - type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - - const TABLE_NAME: &'static str; - const TRIGGER_TYPE: &'static str; - const SUPPORTS_SERVER_STATE: bool; - const SUPPORTS_TEST_CONNECTION: bool; - const ROUTE_PREFIX: &'static str; - const DEPLOYMENT_NAME: &'static str; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; - const IS_ALLOWED_ON_CLOUD: bool; - - fn get_deployed_object(path: String) -> DeployedObject; - - async fn validate_new( - &self, - db: &DB, - workspace_id: &str, - new: &Self::TriggerConfigRequest, - ) -> Result<()> { - self.validate_config(db, new, workspace_id).await - } - - async fn validate_edit( - &self, - db: &DB, - workspace_id: &str, - edit: &Self::TriggerConfigRequest, - _path: &str, - ) -> Result<()> { - self.validate_config(db, edit, workspace_id).await - } - - async fn validate_config( - &self, - _db: &DB, - _config: &Self::TriggerConfigRequest, - _workspace_id: &str, - ) -> Result<()> { - Ok(()) - } - - fn scope_domain_name() -> &'static str { - &Self::ROUTE_PREFIX[1..] - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()>; - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()>; - - async fn test_connection( - &self, - _db: &DB, - _authed: &ApiAuthed, - _user_db: &UserDB, - _workspace_id: &str, - _config: Self::TestConnectionConfig, - ) -> Result<()> { - Err( - anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) - .into(), - ) - } - - fn additional_routes(&self) -> axum::Router { - axum::Router::new() - } - - async fn get_trigger_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - ]; - - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); - } - - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let sql = format!( - r#"SELECT - {} - FROM - {} - WHERE - workspace_id = $1 AND - path = $2 - "#, - fields.join(", "), - Self::TABLE_NAME - ); - - sqlx::query_as(&sql) - .bind(workspace_id) - .bind(path) - .fetch_optional(&mut *tx) - .await? - .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) - } - - async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { - let exists = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .fetch_one(db) - .await?; - - Ok(exists) - } - - async fn delete_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let deleted = sqlx::query(&format!( - "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); - - Ok(deleted > 0) - } - - async fn set_trigger_mode_extra_action(&self, _: &mut PgConnection) -> Result<()> { - Ok(()) - } - - async fn set_trigger_mode( - &self, - authed: &ApiAuthed, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - mode: &TriggerMode, - ) -> Result { - let updated = if Self::SUPPORTS_SERVER_STATE { - sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = $1, - email = $2, - edited_by = $3, - edited_at = now(), - server_id = NULL, - error = NULL - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(mode) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected() - } else { - sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = $1, - email = $2, - edited_by = $3, - edited_at = now() - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(mode) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected() - }; - - self.set_trigger_mode_extra_action(&mut *tx).await?; - - Ok(updated > 0) - } - - #[allow(unused)] - async fn trigger_count( - &self, - tx: &mut PgConnection, - workspace_id: &str, - is_flow: bool, - script_path: &str, - ) -> i64 { - let count = sqlx::query_scalar(&format!( - r#" - SELECT - COUNT(*) - FROM - {} - WHERE - workspace_id = $1 AND - is_flow = $2 AND - script_path = $3 - "#, - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(is_flow) - .bind(script_path) - .fetch_one(&mut *tx) - .await - .unwrap_or(0); - - count - } - - async fn list_triggers( - &self, - tx: &mut PgConnection, - workspace_id: &str, - query: Option<&StandardTriggerQuery>, - ) -> Result> { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - ]; - - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); - } - - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); - - sqlb.fields(&fields) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&workspace_id)); - - if let Some(query) = query { - let (per_page, offset) = - paginate(Pagination { per_page: query.per_page, page: query.page }); - if let Some(path) = &query.path { - sqlb.and_where_eq("script_path", "?".bind(path)); - } - - if let Some(is_flow) = query.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - - if let Some(path_start) = &query.path_start { - sqlb.and_where_like_left("path", path_start); - } - - sqlb.offset(offset).limit(per_page); - } - - let sql = sqlb - .sql() - .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; - - let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; - - Ok(triggers) - } -} - -pub fn trigger_routes() -> Router { - let mut router = Router::new() - .route("/create", post(create_trigger::)) - .route("/list", get(list_triggers::)) - .route("/get/*path", get(get_trigger::)) - .route("/update/*path", post(update_trigger::)) - .route("/delete/*path", delete(delete_trigger::)) - .route("/exists/*path", get(exists_trigger::)) - .route("/setmode/*path", post(set_trigger_mode::)); - - if T::SUPPORTS_TEST_CONNECTION { - router = router.route("/test", post(test_connection::)); - } - - router -} - -async fn create_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(new_trigger): Json>, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || { - format!( - "{}:write:{}", - T::scope_domain_name(), - &new_trigger.base.path - ) - })?; - - if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { - return Err(Error::BadRequest(format!( - "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", - T::TRIGGER_TYPE - ))); - } - - handler - .validate_new(&db, &workspace_id, &new_trigger.config) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let new_path = new_trigger.base.path.clone(); - - handler - .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.create", T::TRIGGER_TYPE), - ActionKind::Create, - &workspace_id, - Some(&new_path), - None, - ) - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(new_path.clone()), - Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), - true, - None, - ) - .await?; - - tx.commit().await?; - - Ok((StatusCode::CREATED, new_path)) -} - -async fn list_triggers( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path(workspace_id): Path, - Query(query): Query, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let triggers = handler - .list_triggers(&mut *tx, &workspace_id, Some(&query)) - .await?; - tx.commit().await?; - - Ok(Json(triggers)) -} - -async fn get_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:read:{}", T::scope_domain_name(), &path) - })?; - - let mut tx = user_db.begin(&authed).await?; - let trigger = handler - .get_trigger_by_path(&mut *tx, &workspace_id, path) - .await?; - - tx.commit().await?; - - Ok(Json(trigger)) -} - -async fn update_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, - Json(edit_trigger): Json>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || { - format!( - "{}:write:{}", - T::scope_domain_name(), - &edit_trigger.base.path - ) - })?; - - handler - .validate_edit(&db, &workspace_id, &edit_trigger.config, path) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let new_path = edit_trigger.base.path.to_string(); - - handler - .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.update", T::TRIGGER_TYPE), - ActionKind::Update, - &workspace_id, - Some(&new_path), - None, - ) - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(new_path.clone()), - Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), - true, - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Trigger '{}' updated", path)) -} - -async fn delete_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:write:{}", T::scope_domain_name(), &path) - })?; - - let mut tx = user_db.begin(&authed).await?; - let deleted = handler - .delete_by_path(&mut *tx, &workspace_id, path) - .await?; - - if !deleted { - return Err(Error::NotFound(format!( - "Trigger not found at path: {}", - path - ))); - } - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.delete", T::TRIGGER_TYPE), - ActionKind::Delete, - &workspace_id, - Some(&path), - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Trigger '{}' deleted", path)) -} - -async fn exists_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:read:{}", T::scope_domain_name(), path) - })?; - let exists = handler.exists(&db, &workspace_id, path).await?; - - Ok(Json(exists)) -} - -#[derive(serde::Deserialize)] -struct SetTriggerModePayload { - mode: TriggerMode, -} - -async fn set_trigger_mode( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, - Json(payload): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; - - let mut tx = user_db.begin(&authed).await?; - let updated = handler - .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) - .await?; - - if !updated { - return Err(Error::NotFound(format!( - "Trigger not found at path: {}", - path - ))); - } - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(path.to_owned()), - Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), - true, - None, - ) - .await?; - - Ok(format!( - "Trigger '{}' {}", - path, - if payload.mode == TriggerMode::Enabled { - "enabled" - } else if payload.mode == TriggerMode::Disabled { - "disabled" - } else { - "suspended" - } - )) -} - -async fn test_connection( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(config): Json, -) -> Result<()> { - let connect_f = async move { - handler - .test_connection(&db, &authed, &user_db, &workspace_id, config) - .await - }; - - tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) - .await - .map_err(|_| { - Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) - })??; - Ok(()) -} - -#[allow(unused)] -pub fn complete_trigger_routes(handler: T) -> Router { - let standard_routes = trigger_routes::(); - - let additional_routes = handler.additional_routes(); - - standard_routes - .merge(additional_routes) - .layer(Extension(Arc::new(handler))) -} +use axum::{routing::post, Router}; +use serde::{Deserialize, Serialize}; +use windmill_common::{error::JsonResult, DB}; + +#[allow(unused_imports)] +use windmill_trigger::handler::complete_trigger_routes; +#[allow(unused_imports)] +use windmill_trigger::TriggerCrud; pub fn generate_trigger_routers() -> Router { #[allow(unused_mut)] @@ -687,7 +13,7 @@ pub fn generate_trigger_routers() -> Router { #[cfg(feature = "http_trigger")] { - use crate::triggers::http::handler::HttpTrigger; + use crate::triggers::http::HttpTrigger; router = router.nest( HttpTrigger::ROUTE_PREFIX, @@ -776,7 +102,7 @@ pub fn generate_trigger_routers() -> Router { } { - use crate::triggers::global_handler::{ + use windmill_trigger::global_handler::{ cancel_suspended_trigger_jobs, resume_suspended_trigger_jobs, }; @@ -794,14 +120,9 @@ pub fn generate_trigger_routers() -> Router { router } -#[derive(Serialize, Deserialize, Debug)] -pub struct TriggerPrimarySchedule { - schedule: String, -} - #[derive(Serialize, Deserialize, Debug)] pub struct TriggersCount { - primary_schedule: Option, + primary_schedule: Option, schedule_count: i64, http_routes_count: i64, webhook_count: i64, @@ -847,7 +168,7 @@ pub async fn get_triggers_count_internal( #[cfg(feature = "http_trigger")] let http_routes_count = { - use crate::triggers::http::handler::HttpTrigger; + use crate::triggers::http::HttpTrigger; let count = HttpTrigger .trigger_count(&mut tx, w_id, is_flow, path) .await; @@ -984,8 +305,9 @@ pub async fn get_triggers_count_internal( .await? .unwrap_or(0); - Ok(Json(TriggersCount { - primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), + Ok(axum::Json(TriggersCount { + primary_schedule: primary_schedule + .map(|s| windmill_trigger::handler::TriggerPrimarySchedule { schedule: s }), schedule_count, http_routes_count, webhook_count, diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index a38532c46d..211861f17e 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -1,51 +1,36 @@ use super::{ - http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, RequestType, - TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, + http_trigger_args::RawHttpTriggerArgs, refresh_routers, AuthenticationMethod, HttpMethod, + RequestType, TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, }; use crate::{ auth::{AuthCache, OptTokened}, db::{ApiAuthed, DB}, jobs::start_job_update_sse_stream, resources::try_get_resource_from_db_as, - triggers::{ - http::{ - refresh_routers, validate_authentication_method, HttpConfig, HttpConfigRequest, - RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, - }, - trigger_helpers::{ - get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, - trigger_runnable_inner, RunnableId, - }, - Trigger, TriggerCrud, TriggerData, TriggerMode, + triggers::trigger_helpers::{ + get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, + trigger_runnable_inner, RunnableId, }, users::fetch_api_authed, utils::{check_scopes, ExpiringCacheEntry}, }; use axum::{ - async_trait, extract::Path, response::{IntoResponse, Response}, - routing::{get, post}, - Extension, Json, Router, + routing::get, + Extension, Router, }; use futures::StreamExt; use http::{HeaderMap, StatusCode}; -use sqlx::PgConnection; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - sync::Arc, -}; -use windmill_audit::{audit_oss::audit_log, ActionKind}; +use std::{collections::HashMap, sync::Arc}; use windmill_common::{ db::UserDB, error::{Error, Result}, jobs::JobTriggerKind, triggers::{TriggerKind, TriggerMetadata}, - utils::{not_found_if_none, require_admin, StripPath}, - worker::CLOUD_HOSTED, + utils::{not_found_if_none, StripPath}, }; -use windmill_git_sync::handle_deployment_metadata; +use windmill_trigger::TriggerMode; #[cfg(feature = "parquet")] use { @@ -53,572 +38,6 @@ use { windmill_common::s3_helpers::build_object_store_client, }; -use windmill_git_sync::DeployedObject; - -pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { - sqlx::query!("SELECT nextval('http_trigger_version_seq')") - .fetch_one(tx) - .await?; - Ok(()) -} - -pub fn generate_route_path_key(route_path: &str) -> String { - ROUTE_PATH_KEY_RE - .replace_all(route_path, "${1}${2}key") - .to_string() -} - -pub async fn route_path_key_exists( - route_path_key: &str, - http_method: &HttpMethod, - w_id: &str, - trigger_path: Option<&str>, - workspaced_route: Option, - db: &DB, -) -> Result { - let exists = if *CLOUD_HOSTED { - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - route_path_key = $1 - AND workspace_id = $2 - AND http_method = $3 - AND ($4::TEXT IS NULL OR path != $4) - ) - "#, - &route_path_key, - w_id, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - } else { - let route_path_key = match workspaced_route { - Some(true) => Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))), - _ => Cow::Borrowed(route_path_key), - }; - - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) - OR (workspaced_route IS FALSE AND route_path_key = $1)) - AND http_method = $2 - AND ($3::TEXT IS NULL OR path != $3) - ) - "#, - &route_path_key, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; - - Ok(exists) -} - -pub async fn exists_route( - Extension(db): Extension, - Path(w_id): Path, - Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< - RouteExists, - >, -) -> Result> { - let route_path_key = generate_route_path_key(&route_path); - - let exists = route_path_key_exists( - &route_path_key, - &http_method, - &w_id, - trigger_path.as_deref(), - workspaced_route, - &db, - ) - .await?; - - Ok(Json(exists)) -} - -fn check_no_duplicates<'trigger>( - new_http_triggers: &[TriggerData], - route_path_key: &[String], -) -> Result<()> { - let mut seen = HashSet::with_capacity(new_http_triggers.len()); - - for (i, trigger) in new_http_triggers.iter().enumerate() { - if !seen.insert(( - &route_path_key[i], - trigger.config.http_method, - trigger.config.workspaced_route, - )) { - return Err(Error::BadRequest(format!( - "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", - &trigger.config.route_path - ))); - } - } - - Ok(()) -} - -pub async fn insert_new_trigger_into_db( - authed: &ApiAuthed, - tx: &mut PgConnection, - w_id: &str, - trigger: &TriggerData, - route_path_key: &str, -) -> Result<()> { - require_admin(authed.is_admin, &authed.username)?; - - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - INSERT INTO http_trigger ( - workspace_id, - path, - route_path, - route_path_key, - workspaced_route, - authentication_resource_path, - wrap_body, - raw_string, - script_path, - summary, - description, - is_flow, - mode, - request_type, - authentication_method, - http_method, - static_asset_config, - edited_by, - email, - edited_at, - is_static_website, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 - ) - "#, - w_id, - trigger.base.path, - trigger.config.route_path, - route_path_key, - trigger.config.workspaced_route.unwrap_or(false), - trigger.config.authentication_resource_path, - trigger.config.wrap_body.unwrap_or(false), - trigger.config.raw_string.unwrap_or(false), - trigger.base.script_path, - trigger.config.summary, - trigger.config.description, - trigger.base.is_flow, - trigger.base.mode() as _, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(&mut *tx) - .await?; - Ok(()) -} - -pub async fn create_many_http_triggers( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(new_http_triggers): Json>>, -) -> Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - let handler = HttpTrigger; - - let error_wrapper = |route_path: &str, error: Error| -> Error { - anyhow::anyhow!( - "Error occurred for HTTP route at route path: {}, error: {}", - route_path, - error - ) - .into() - }; - - let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); - - for new_http_trigger in new_http_triggers.iter() { - handler - .validate_new(&db, &w_id, &new_http_trigger.config) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; - - let route_path_key = - check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; - - route_path_keys.push(route_path_key.clone()); - } - - check_no_duplicates(&new_http_triggers, &route_path_keys)?; - - let mut tx = user_db.begin(&authed).await?; - - for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { - insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; - - audit_log( - &mut *tx, - &authed, - "http_trigger.create", - ActionKind::Create, - &w_id, - Some(&new_http_trigger.base.path), - None, - ) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; - - increase_trigger_version(&mut tx) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; - } - - tx.commit().await?; - - for http_trigger in new_http_triggers.into_iter() { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, - Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), - true, - None, - ) - .await - .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; - } - Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) -} - -async fn check_if_route_exist( - db: &DB, - config: &HttpConfigRequest, - workspace_id: &str, - trigger_path: Option<&str>, -) -> Result { - let route_path_key = generate_route_path_key(&config.route_path); - - let exists = route_path_key_exists( - &route_path_key, - &config.http_method, - workspace_id, - trigger_path, - config.workspaced_route, - db, - ) - .await?; - - if exists { - return Err(Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - Ok(route_path_key) -} - -pub struct HttpTrigger; - -#[async_trait] -impl TriggerCrud for HttpTrigger { - type TriggerConfig = HttpConfig; - type Trigger = Trigger; - type TriggerConfigRequest = HttpConfigRequest; - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = "http_trigger"; - const TRIGGER_TYPE: &'static str = "http"; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/http_triggers"; - const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; - const IS_ALLOWED_ON_CLOUD: bool = true; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ - "route_path", - "route_path_key", - "request_type", - "authentication_method", - "http_method", - "summary", - "description", - "static_asset_config", - "is_static_website", - "authentication_resource_path", - "workspaced_route", - "wrap_body", - "raw_string", - ]; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::HttpTrigger { path } - } - - fn additional_routes(&self) -> Router { - Router::new() - .route("/create_many", post(create_many_http_triggers)) - .route("/route_exists", post(exists_route)) - } - - async fn validate_new( - &self, - _db: &DB, - _workspace_id: &str, - new: &Self::TriggerConfigRequest, - ) -> Result<()> { - if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { - return Err(Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { - return Err(Error::BadRequest("Invalid route path".to_string())); - } - - validate_authentication_method(new.authentication_method, new.raw_string)?; - - Ok(()) - } - - async fn validate_edit( - &self, - _db: &DB, - _workspace_id: &str, - edit: &Self::TriggerConfigRequest, - _path: &str, - ) -> Result<()> { - if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { - return Err(Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - validate_authentication_method(edit.authentication_method, edit.raw_string)?; - - Ok(()) - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()> { - let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; - - insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; - - increase_trigger_version(tx).await?; - - Ok(()) - } - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()> { - if authed.is_admin { - if trigger.config.route_path.is_empty() { - return Err(Error::BadRequest("route_path is required".to_string())); - }; - - let route_path = &trigger.config.route_path; - if !VALID_ROUTE_PATH_RE.is_match(route_path) { - return Err(Error::BadRequest("Invalid route path".to_string())); - } - - let route_path_key = - check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; - - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - UPDATE - http_trigger - SET - route_path = $1, - route_path_key = $2, - workspaced_route = $3, - wrap_body = $4, - raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - mode = $10, - http_method = $11, - static_asset_config = $12, - edited_by = $13, - email = $14, - request_type = $15, - authentication_method = $16, - summary = $17, - description = $18, - edited_at = now(), - is_static_website = $19, - error_handler_path = $20, - error_handler_args = $21, - retry = $22 - WHERE - workspace_id = $23 AND - path = $24 - "#, - route_path, - &route_path_key, - trigger.config.workspaced_route, - trigger.config.wrap_body, - trigger.config.raw_string, - trigger.config.authentication_resource_path, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - trigger.base.mode() as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.summary, - trigger.config.description, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _, - workspace_id, - path, - ) - .execute(&mut *tx) - .await?; - } else { - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - UPDATE - http_trigger - SET - wrap_body = $1, - raw_string = $2, - authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - mode = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - email = $11, - request_type = $12, - authentication_method = $13, - summary = $14, - description = $15, - edited_at = now(), - is_static_website = $16, - error_handler_path = $17, - error_handler_args = $18, - retry = $19 - WHERE - workspace_id = $20 AND - path = $21 - "#, - trigger.config.wrap_body, - trigger.config.raw_string, - trigger.config.authentication_resource_path, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - trigger.base.mode() as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.summary, - trigger.config.description, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _, - workspace_id, - path, - ) - .execute(&mut *tx) - .await?; - } - - increase_trigger_version(tx).await?; - - Ok(()) - } - - async fn set_trigger_mode_extra_action(&self, tx: &mut PgConnection) -> Result<()> { - increase_trigger_version(tx).await - } - - async fn delete_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let deleted = sqlx::query(&format!( - "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); - - increase_trigger_version(tx).await?; - - Ok(deleted > 0) - } -} - async fn conditional_cors_middleware( req: axum::extract::Request, next: axum::middleware::Next, @@ -757,10 +176,10 @@ async fn get_http_route_trigger( r#" SELECT EXISTS( SELECT 1 - FROM - http_trigger - WHERE - workspace_id = $1 AND + FROM + http_trigger + WHERE + workspace_id = $1 AND path = $2 ) "#, diff --git a/backend/windmill-api/src/triggers/http/http_trigger_args.rs b/backend/windmill-api/src/triggers/http/http_trigger_args.rs index af829cec53..4866f9be34 100644 --- a/backend/windmill-api/src/triggers/http/http_trigger_args.rs +++ b/backend/windmill-api/src/triggers/http/http_trigger_args.rs @@ -4,7 +4,7 @@ use axum::{ extract::{FromRequest, Request}, response::Response, }; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use serde_json::value::RawValue; use windmill_common::{ error::Error, @@ -14,6 +14,7 @@ use windmill_common::{ }; use windmill_queue::PushArgsOwned; +use super::HttpMethod; use crate::{ args::{ build_headers, build_query, try_from_request_body, Body, RawWebhookArgs, WebhookArgs, @@ -24,31 +25,6 @@ use crate::{ pub struct RawHttpTriggerArgs(pub RawWebhookArgs); -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - #[axum::async_trait] impl FromRequest for RawHttpTriggerArgs where diff --git a/backend/windmill-api/src/triggers/http/http_trigger_auth.rs b/backend/windmill-api/src/triggers/http/http_trigger_auth.rs deleted file mode 100644 index 85288d87b8..0000000000 --- a/backend/windmill-api/src/triggers/http/http_trigger_auth.rs +++ /dev/null @@ -1,751 +0,0 @@ -use axum::response::{IntoResponse, Response}; -use base64::{ - prelude::{BASE64_STANDARD, BASE64_URL_SAFE}, - Engine, -}; -use hmac::{Hmac, Mac}; -use http::{header, HeaderMap, HeaderValue, StatusCode}; -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use sha1::Sha1; -use sha2::{Sha256, Sha512}; -use std::{borrow::Cow, collections::HashMap}; - -pub type HmacSha256 = Hmac; -pub type HmacSha512 = Hmac; -pub type HmacSha1 = Hmac; - -mod github { - use super::*; - pub struct Github; - - impl WebhookHandler for Github { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let github_secret_header = headers.try_get_webhook_header("X-Hub-Signature-256")?; - - let authentication_data = SignatureAuthenticationData::new( - Cow::Borrowed(raw_payload), - github_secret_header, - Some("sha256="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - ); - - Ok(authentication_data) - } - } -} - -mod slack { - use super::*; - pub struct Slack; - - impl WebhookHandler for Slack { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let slack_secret_signature = headers.try_get_webhook_header("X-Slack-Signature")?; - let slack_timestamp_header = - headers.try_get_webhook_header("X-Slack-Request-Timestamp")?; - let signed_payload = format!("v0:{}:{}", slack_timestamp_header, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(signed_payload), - slack_secret_signature, - Some("v0="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -mod stripe { - use super::*; - - pub struct Stripe; - - impl WebhookHandler for Stripe { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let stripe_signature_header = headers.try_get_webhook_header("STRIPE-SIGNATURE")?; - - let stripe_signature = parse_signature(stripe_signature_header, (",", "=")); - - let timestamp = *stripe_signature - .get("t") - .ok_or(AuthenticationError::InvalidTimestamp)?; - let v1 = *stripe_signature - .get("v1") - .ok_or(AuthenticationError::InvalidSignature)?; - - let signed_payload = format!("{}.{}", timestamp, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(signed_payload), - v1, - None, - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -mod tiktok { - use super::*; - - pub struct TikTok; - - impl WebhookHandler for TikTok { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let tiktok_secret_signature = headers.try_get_webhook_header("TikTok-Signature")?; - - let stripe_signature = parse_signature(tiktok_secret_signature, (",", "=")); - - let timestamp = *stripe_signature - .get("t") - .ok_or(AuthenticationError::InvalidTimestamp)?; - let s = *stripe_signature - .get("s") - .ok_or(AuthenticationError::InvalidSignature)?; - - let signed_payload = format!("{}.{}", timestamp, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(signed_payload), - s, - None, - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -mod twitch { - use super::*; - use http::header; - use serde_json::value::RawValue; - #[derive(Debug, Deserialize)] - struct TwitchCrcBody { - challenge: String, - #[allow(unused)] - subscription: Box, - } - - pub struct Twitch; - - impl WebhookHandler for Twitch { - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let twitch_secret_signature = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Signature")?; - let twitch_message_id_header = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Id")?; - let twitch_timestamp_header = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Timestamp")?; - - let message = format!( - "{}{}{}", - twitch_message_id_header, twitch_timestamp_header, raw_payload - ); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(message), - twitch_secret_signature, - Some("sha256="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - - fn handle_challenge_request<'header>( - &self, - headers: &'header HeaderMap, - signature_config_data: &SignatureConfigData, - raw_payload: &str, - ) -> Result, AuthenticationError> { - let authentication_data = self.get_hmac_authentication_data(headers, raw_payload)?; - verify_hmac_signature(authentication_data, &signature_config_data.secret_key)?; - - let twitch_eventsub_message_type = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Type")?; - - if twitch_eventsub_message_type != "webhook_callback_verification" { - return Ok(None); - } - let twitch_crc_body = - serde_json::from_str::(raw_payload).map_err(|e| { - AuthenticationError::InvalidChallengeResponse(format!( - "Twitch :{}", - e.to_string() - )) - })?; - - let response = ( - StatusCode::OK, - [(header::CONTENT_TYPE, "text/plain")], - twitch_crc_body.challenge.to_string(), - ); - - Ok(Some(response.into_response())) - } - } -} - -mod zoom { - use axum::Json; - - use super::*; - - #[derive(Debug, Deserialize)] - struct ZoomPayload { - #[serde(rename = "plainToken")] - plain_token: String, - } - - #[derive(Debug, Deserialize)] - #[allow(unused)] - struct ZoomChallengeResponse { - payload: ZoomPayload, - event_ts: u64, - event: String, - } - - pub struct Zoom; - - impl WebhookHandler for Zoom { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - signature_config_data: &SignatureConfigData, - raw_payload: &str, - ) -> Result, AuthenticationError> { - let Ok(zoom_request_body) = serde_json::from_str::(raw_payload) - else { - return Ok(None); - }; - - if zoom_request_body.event != "endpoint.url_validation" { - return Ok(None); - } - - let hmac_signature = calculate_hmac_signature( - HmacAlgorithm::Sha256, - &signature_config_data.secret_key, - &zoom_request_body.payload.plain_token, - ); - - let encoded_hmac_signature = encode_hmac_signature(Encoding::Hex, &hmac_signature); - - let response = ( - StatusCode::OK, - Json(json!({ - "plainToken": zoom_request_body.payload.plain_token, - "encryptedToken": encoded_hmac_signature - })), - ); - - Ok(Some(response.into_response())) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let zoom_signature_header = headers.try_get_webhook_header("x-zm-signature")?; - let zoom_timestamp_header = headers.try_get_webhook_header("x-zm-request-timestamp")?; - - let message = format!("v0:{}:{}", zoom_timestamp_header, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(message), - zoom_signature_header, - Some("v0="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -use constant_time_eq::constant_time_eq; -use github::Github; -use slack::Slack; -use stripe::Stripe; -use tiktok::TikTok; -use twitch::Twitch; -use zoom::Zoom; - -#[derive(Debug)] -pub struct SignatureAuthenticationDetails { - pub algorithm_to_use: HmacAlgorithm, - pub header_key_encoding: Encoding, -} - -impl SignatureAuthenticationDetails { - #[inline] - fn new(algorithm_to_use: HmacAlgorithm, header_key_encoding: Encoding) -> Self { - Self { algorithm_to_use, header_key_encoding } - } -} - -fn parse_signature<'header>( - signature: &'header str, - splitters: (&str, &str), -) -> HashMap<&'header str, &'header str> { - let headers: HashMap<&str, &str> = signature - .split(splitters.0) - .map(|header| { - let mut key_and_value = header.split(splitters.1); - let key = key_and_value.next(); - let value = key_and_value.next(); - (key, value) - }) - .filter_map(|(key, value)| match (key, value) { - (Some(key), Some(value)) => Some((key, value)), - _ => None, - }) - .collect(); - headers -} - -#[derive(Debug)] -pub struct SignatureAuthenticationData<'payload, 'header, 'prefix> { - pub signed_payload: Cow<'payload, str>, - pub header_key_value: &'header str, - pub signature_prefix: Option<&'prefix str>, - pub config: SignatureAuthenticationDetails, -} - -impl<'payload, 'header, 'prefix> SignatureAuthenticationData<'payload, 'header, 'prefix> { - pub fn new( - signed_payload: Cow<'payload, str>, - header_key_value: &'header str, - signature_prefix: Option<&'prefix str>, - config: SignatureAuthenticationDetails, - ) -> Self { - Self { signed_payload, header_key_value, signature_prefix, config } - } -} - -pub trait WebhookHandler { - fn handle_challenge_request<'header>( - &self, - headers: &'header HeaderMap, - signature_config_data: &SignatureConfigData, - raw_payload: &str, - ) -> Result, AuthenticationError>; - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError>; -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum HmacAlgorithm { - Sha1, - Sha256, - Sha512, -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Encoding { - Base64, - Base64Uri, - Hex, -} -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct SignatureAuthenticationMethod { - algorithm: HmacAlgorithm, - encoding: Encoding, - signature_header_name: String, - signature_prefix: Option, -} - -pub struct SignatureConfigData<'config> { - secret_key: &'config str, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct SignatureAuthentication { - signature_provider: WebhookType, - secret_key: String, - authentication_config: Option, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct BasicAuthAuthentication { - pub username: String, - pub password: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ApiKeyAuthentication { - pub api_key_header: String, - pub api_key_secret: String, -} - -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] -#[non_exhaustive] -pub enum WebhookType { - Github, - Slack, - Stripe, - TikTok, - Twitch, - Zoom, - Custom, -} - -impl WebhookType { - pub fn get_webhook_handler(&self) -> Option<&'static dyn WebhookHandler> { - let handler: &'static dyn WebhookHandler = match *self { - WebhookType::Github => &Github, - WebhookType::Slack => &Slack, - WebhookType::Stripe => &Stripe, - WebhookType::TikTok => &TikTok, - WebhookType::Twitch => &Twitch, - WebhookType::Zoom => &Zoom, - WebhookType::Custom => return None, - }; - Some(handler) - } -} - -trait TryGetWebhookHeader { - fn try_get_webhook_header<'header>( - &'header self, - header_name: &str, - ) -> Result<&'header str, AuthenticationError>; -} - -impl TryGetWebhookHeader for HeaderMap { - fn try_get_webhook_header<'header>( - &'header self, - header_name: &str, - ) -> Result<&'header str, AuthenticationError> { - let Some(signature_header) = self.get(header_name) else { - return Err(AuthenticationError::MissingHeader(header_name.to_string())); - }; - let Some(signature_header) = signature_header.to_str().ok() else { - return Err(AuthenticationError::InvalidHeader(header_name.to_string())); - }; - - Ok(signature_header) - } -} - -pub fn calculate_hmac_signature(algorithm: HmacAlgorithm, secret: &str, payload: &str) -> Vec { - match algorithm { - HmacAlgorithm::Sha1 => { - let mut mac = - HmacSha1::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - mac.finalize().into_bytes().to_vec() - } - HmacAlgorithm::Sha256 => { - let mut mac = HmacSha256::new_from_slice(secret.as_bytes()) - .expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - mac.finalize().into_bytes().to_vec() - } - HmacAlgorithm::Sha512 => { - let mut mac = HmacSha512::new_from_slice(secret.as_bytes()) - .expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - mac.finalize().into_bytes().to_vec() - } - } -} - -pub fn encode_hmac_signature(encoding: Encoding, hmac_signature: &[u8]) -> String { - match encoding { - Encoding::Hex => hex::encode(hmac_signature), - Encoding::Base64 => BASE64_STANDARD.encode(hmac_signature), - Encoding::Base64Uri => BASE64_URL_SAFE.encode(hmac_signature), - } -} - -pub fn verify_hmac_signature( - authentication_data: SignatureAuthenticationData, - webhook_signing_secret: &str, -) -> Result<(), AuthenticationError> { - let hmac_signature = calculate_hmac_signature( - authentication_data.config.algorithm_to_use, - &webhook_signing_secret, - &authentication_data.signed_payload, - ); - - let encoded_signature = encode_hmac_signature( - authentication_data.config.header_key_encoding, - &hmac_signature, - ); - - let final_expected_signature = - if let Some(signature_prefix) = authentication_data.signature_prefix { - format!("{}{}", signature_prefix, encoded_signature) - } else { - encoded_signature - }; - - if !constant_time_eq( - final_expected_signature.as_bytes(), - authentication_data.header_key_value.as_bytes(), - ) { - return Err(AuthenticationError::InvalidSignature); - } - - Ok(()) -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(untagged)] -pub enum AuthenticationMethod { - Signature(SignatureAuthentication), - BasicAuth(BasicAuthAuthentication), - ApiKey(ApiKeyAuthentication), -} - -impl AuthenticationMethod { - pub fn authenticate_http_request( - &self, - headers: &HeaderMap, - raw_payload: Option<&String>, - ) -> Result, AuthenticationError> { - match self { - AuthenticationMethod::Signature(SignatureAuthentication { - secret_key, - authentication_config, - signature_provider, - }) => { - let raw_payload = raw_payload.ok_or(AuthenticationError::InvalidPayload)?; - let config_data = SignatureConfigData { secret_key: &secret_key }; - let handler = signature_provider.get_webhook_handler(); - let challenge_response = handler - .map(|handler| { - handler.handle_challenge_request(headers, &config_data, raw_payload) - }) - .transpose()? - .flatten(); - - if let Some(challenge_response) = challenge_response { - return Ok(Some(challenge_response)); - } - - let authentication_data = match handler { - Some(handler) => handler.get_hmac_authentication_data(headers, raw_payload)?, - None => { - let authentication_config = authentication_config - .as_ref() - .ok_or(AuthenticationError::InvalidCustomConfig)?; - let signature_header_value = headers - .try_get_webhook_header(&authentication_config.signature_header_name)?; - SignatureAuthenticationData::new( - Cow::Borrowed(raw_payload), - signature_header_value, - authentication_config.signature_prefix.as_deref(), - SignatureAuthenticationDetails::new( - authentication_config.algorithm, - authentication_config.encoding, - ), - ) - } - }; - - verify_hmac_signature(authentication_data, &secret_key)?; - } - AuthenticationMethod::ApiKey(ApiKeyAuthentication { - api_key_header, - api_key_secret, - }) => { - let api_key_to_cmp = headers - .try_get_webhook_header(&api_key_header) - .map_err(|_| AuthenticationError::InvalidApiKey)?; - if api_key_to_cmp != api_key_secret { - return Err(AuthenticationError::InvalidApiKey); - } - } - AuthenticationMethod::BasicAuth(BasicAuthAuthentication { username, password }) => { - let mut credentials_store = headers - .try_get_webhook_header("Authorization") - .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)? - .split(' '); - - let _ = credentials_store - .next() - .filter(|r#type| *r#type == "Basic") - .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials_as_base64 = credentials_store - .next() - .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials_from_base64_as_bytes = BASE64_STANDARD - .decode(credentials_as_base64.as_bytes()) - .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials_separated_with_colon = - String::from_utf8(credentials_from_base64_as_bytes) - .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials = credentials_separated_with_colon.split(':').collect_vec(); - - if credentials.len() != 2 { - return Err(AuthenticationError::UnauthorizedBasicHttpAuth); - } - - if credentials.get(0).unwrap() != username - || credentials.get(1).unwrap() != password - { - return Err(AuthenticationError::UnauthorizedBasicHttpAuth); - } - } - } - - Ok(None) - } -} - -#[derive(thiserror::Error, Debug)] -#[allow(unused)] -pub enum AuthenticationError { - #[error("failed to parse timestamp")] - InvalidTimestamp, - - #[error("invalid secret")] - InvalidSecret(#[from] base64::DecodeError), - - #[error("invalid header `{0}`")] - InvalidHeader(String), - - #[error("signature timestamp too old")] - TimestampTooOldError, - - #[error("signature timestamp too far in future")] - FutureTimestampError, - - #[error("missing header {0}")] - MissingHeader(String), - - #[error("signature invalid")] - InvalidSignature, - - #[error("payload invalid")] - InvalidPayload, - - #[error("invalid custom config")] - InvalidCustomConfig, - - #[error("invalid auth header: {0}")] - InvalidAuthHeader(String), - - #[error("invalid api key")] - InvalidApiKey, - - #[error("invalid challenge response: {0}")] - InvalidChallengeResponse(String), - - #[error("")] - UnauthorizedBasicHttpAuth, -} - -impl IntoResponse for AuthenticationError { - fn into_response(self) -> Response { - let (status, error_message) = match &self { - AuthenticationError::InvalidTimestamp - | AuthenticationError::InvalidPayload - | AuthenticationError::InvalidHeader(_) - | AuthenticationError::MissingHeader(_) - | AuthenticationError::TimestampTooOldError - | AuthenticationError::FutureTimestampError - | AuthenticationError::InvalidCustomConfig - | AuthenticationError::InvalidChallengeResponse(_) => { - (StatusCode::BAD_REQUEST, self.to_string()) - } - - AuthenticationError::InvalidSecret(_) - | AuthenticationError::InvalidSignature - | AuthenticationError::InvalidAuthHeader(_) => { - (StatusCode::UNAUTHORIZED, self.to_string()) - } - AuthenticationError::UnauthorizedBasicHttpAuth => { - return ( - StatusCode::UNAUTHORIZED, - [(header::WWW_AUTHENTICATE, r#"Basic realm="Restricted Area""#)], - "Unauthorized", - ) - .into_response() - } - AuthenticationError::InvalidApiKey => { - return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response() - } - }; - - let body = json!({ "error": error_message }); - - let mut headers = HeaderMap::new(); - headers.insert("Content-Type", HeaderValue::from_static("application/json")); - - (status, headers, body.to_string()).into_response() - } -} diff --git a/backend/windmill-api/src/triggers/http/mod.rs b/backend/windmill-api/src/triggers/http/mod.rs index 291d9eadfe..f65a26c03e 100644 --- a/backend/windmill-api/src/triggers/http/mod.rs +++ b/backend/windmill-api/src/triggers/http/mod.rs @@ -1,410 +1,4 @@ -use std::collections::HashMap; - -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::{types::Json as SqlxJson, FromRow}; -use tokio::sync::{RwLock, RwLockReadGuard}; -use windmill_common::{ - error::{Error, Result}, - flows::Retry, - s3_helpers::S3Object, - worker::CLOUD_HOSTED, - DB, -}; - -use crate::{db::ApiAuthed, triggers::TriggerMode, utils::ExpiringCacheEntry}; +pub use windmill_trigger_http::*; pub mod handler; pub mod http_trigger_args; -pub mod http_trigger_auth; - -lazy_static::lazy_static! { - static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); - static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); - - static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { - routers: HashMap::new(), - version: 0, - }); -} - -#[derive(Debug, Deserialize, Clone)] -pub struct TriggerRoute { - path: String, - script_path: String, - is_flow: bool, - route_path: String, - workspace_id: String, - request_type: RequestType, - authentication_method: AuthenticationMethod, - edited_by: String, - email: String, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: bool, - wrap_body: bool, - raw_string: bool, - error_handler_path: Option, - error_handler_args: Option>>, - retry: Option>, - mode: TriggerMode, -} - -pub struct RoutersCache { - routers: HashMap>, - version: i64, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] -#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum RequestType { - Sync, - Async, - SyncSse, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] -#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] -#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] -pub enum AuthenticationMethod { - None, - Windmill, - ApiKey, - BasicHttp, - CustomScript, - Signature, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct HttpConfig { - pub route_path: String, - pub route_path_key: String, - pub request_type: RequestType, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: bool, - pub wrap_body: bool, - pub raw_string: bool, -} - -#[derive(Debug, Clone, Serialize)] -pub struct HttpConfigRequest { - #[serde(default)] - pub route_path: String, - pub request_type: RequestType, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: Option, - pub wrap_body: Option, - pub raw_string: Option, -} - -#[derive(Deserialize)] -struct HttpConfigRequestHelper { - #[serde(default)] - route_path: String, - request_type: Option, - is_async: Option, - authentication_method: AuthenticationMethod, - http_method: HttpMethod, - summary: Option, - description: Option, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: Option, - wrap_body: Option, - raw_string: Option, -} - -impl<'de> Deserialize<'de> for HttpConfigRequest { - fn deserialize(deserializer: D) -> std::result::Result - where - D: serde::Deserializer<'de>, - { - let helper = HttpConfigRequestHelper::deserialize(deserializer)?; - - // Determine request_type with backward compatibility - let request_type = if let Some(mode) = helper.request_type { - mode - } else if let Some(is_async) = helper.is_async { - if is_async { - RequestType::Async - } else { - RequestType::Sync - } - } else { - RequestType::Sync - }; - - Ok(HttpConfigRequest { - route_path: helper.route_path, - request_type, - authentication_method: helper.authentication_method, - http_method: helper.http_method, - summary: helper.summary, - description: helper.description, - static_asset_config: helper.static_asset_config, - is_static_website: helper.is_static_website, - authentication_resource_path: helper.authentication_resource_path, - workspaced_route: helper.workspaced_route, - wrap_body: helper.wrap_body, - raw_string: helper.raw_string, - }) - } -} - -// Regex patterns for route validation -lazy_static::lazy_static! { - // Matches named params like :id or wildcards like :* or * - static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); - static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); -} - -#[derive(Deserialize)] -pub struct RouteExists { - pub route_path: String, - pub http_method: HttpMethod, - pub trigger_path: Option, - pub workspaced_route: Option, -} - -pub fn validate_authentication_method( - authentication_method: AuthenticationMethod, - raw_string: Option, -) -> Result<()> { - match (authentication_method, raw_string) { - (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { - Err(Error::BadRequest( - "To use custom script authentication, please enable the raw body option." - .to_string(), - )) - } - _ => Ok(()), - } -} - -pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { - let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) - .fetch_one(db) - .await?; - let routers_cache = HTTP_ROUTERS_CACHE.read().await; - if routers_cache.version == 0 || version > routers_cache.version { - drop(routers_cache); - let mut routers = HashMap::new(); - - for http_method in [ - HttpMethod::Get, - HttpMethod::Post, - HttpMethod::Put, - HttpMethod::Patch, - HttpMethod::Delete, - ] { - let triggers = sqlx::query_as!( - TriggerRoute, - r#" - SELECT - path, - script_path, - is_flow, - route_path, - authentication_resource_path, - workspace_id, - request_type AS "request_type: _", - authentication_method AS "authentication_method: _", - edited_by, - email, - static_asset_config AS "static_asset_config: _", - wrap_body, - raw_string, - workspaced_route, - is_static_website, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _", - mode as "mode: _" - FROM - http_trigger - WHERE - http_method = $1 AND - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - "#, - &http_method as &HttpMethod - ) - .fetch_all(db) - .await?; - - let mut router = matchit::Router::new(); - - for trigger in triggers { - let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { - format!("/{}/{}", trigger.workspace_id, trigger.route_path) - } else { - format!("/{}", trigger.route_path) - }; - - if trigger.is_static_website { - router - .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!( - "Failed to consider HTTP route {}/*wm_subpath: {:?}", - full_path, - e, - ); - }); - } - router - .insert(full_path.clone(), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); - }); - } - - routers.insert(http_method, router); - } - - let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; - *routers_cache = RoutersCache { routers, version }; - - Ok((true, routers_cache.downgrade())) - } else { - tracing::debug!("No HTTP routers refresh needed"); - Ok((false, routers_cache)) - } -} - -pub async fn refresh_routers_loop( - db: &DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - match refresh_routers(db).await { - Ok(_) => { - tracing::info!("Loaded HTTP routers"); - } - Err(err) => { - tracing::error!("Error loading HTTP routers: {err:#}"); - } - }; - let db = db.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = killpill_rx.recv() => { - break; - } - _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - match refresh_routers(&db).await { - Ok((true, _)) => { - tracing::info!("Refreshed HTTP routers"); - } - Err(err) => { - tracing::error!("Error refreshing HTTP routers: {err:#}"); - } - _ => {} - } - } - } - } - }); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_request_type_backward_compatibility() { - // Test with new request_type field - let json_new = r#"{ - "route_path": "/test", - "request_type": "sync_sse", - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); - assert_eq!(config.request_type, RequestType::SyncSse); - - // Test with legacy is_async = true - let json_legacy_async = r#"{ - "route_path": "/test", - "is_async": true, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); - assert_eq!(config.request_type, RequestType::Async); - - // Test with legacy is_async = false - let json_legacy_sync = r#"{ - "route_path": "/test", - "is_async": false, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); - assert_eq!(config.request_type, RequestType::Sync); - - // Test with neither field (default to sync) - let json_default = r#"{ - "route_path": "/test", - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); - assert_eq!(config.request_type, RequestType::Sync); - - // Test that request_type takes precedence over is_async - let json_both = r#"{ - "route_path": "/test", - "request_type": "sync_sse", - "is_async": true, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); - assert_eq!(config.request_type, RequestType::SyncSse); - } -} diff --git a/backend/windmill-api/src/triggers/kafka/mod.rs b/backend/windmill-api/src/triggers/kafka/mod.rs index f09b2283ff..21e0c06b44 100644 --- a/backend/windmill-api/src/triggers/kafka/mod.rs +++ b/backend/windmill-api/src/triggers/kafka/mod.rs @@ -1,15 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct KafkaTrigger; +pub use windmill_trigger_kafka::*; diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs index ce4f651ba9..3e67f36109 100644 --- a/backend/windmill-api/src/triggers/listener.rs +++ b/backend/windmill-api/src/triggers/listener.rs @@ -1,846 +1,13 @@ -use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use windmill_common::DB; +#[allow(unused_imports)] +use windmill_trigger::listener::{listen_to_unlistened_events, listening}; +#[allow(unused_imports)] +use windmill_trigger::Listener; -use crate::{ - capture::insert_capture_payload, - db::ApiAuthed, - triggers::{ - handler::TriggerCrud, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - Trigger, TriggerErrorHandling, TriggerMode, - }, - users::fetch_api_authed, -}; -use async_trait::async_trait; -use itertools::Itertools; -use rand::seq::SliceRandom; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sql_builder::SqlBuilder; -use sqlx::{FromRow, Row}; -use tokio::sync::RwLock; -use windmill_common::{ - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerKind, TriggerMetadata}, - utils::report_critical_error, - DB, INSTANCE_NAME, -}; - -#[allow(unused)] -#[async_trait] -pub trait Listener: TriggerCrud + TriggerJobArgs { - type Consumer: Send; - type Extra: Send + Sync; - type ExtraState: Send + Sync; - - const JOB_TRIGGER_KIND: JobTriggerKind; - const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; - const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; - - async fn get_consumer( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result>; - async fn consume( - &self, - db: &DB, - consumer: Self::Consumer, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - extra: Option<&Self::ExtraState>, - ); - async fn fetch_enabled_unlistened_triggers( - &self, - db: &DB, - ) -> Result>> { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - "error_handler_path", - "error_handler_args", - "retry", - ]; - - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); - - sqlb.fields(&fields) - .and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)") - .and_where( - "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - ); - - for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { - sqlb.and_where(where_clause); - } - - let sql = sqlb - .sql() - .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; - - let triggers: Vec> = - sqlx::query_as(&sql).fetch_all(db).await?; - - let triggers = triggers - .into_iter() - .map(|trigger| ListeningTrigger { - path: trigger.base.path, - workspace_id: trigger.base.workspace_id, - is_flow: trigger.base.is_flow, - username: trigger.base.edited_by, - email: trigger.base.email, - script_path: trigger.base.script_path, - trigger_config: trigger.config, - error_handling: Some(trigger.error_handling), - trigger_mode: true, - suspended_mode: trigger.base.mode == TriggerMode::Suspended, - }) - .collect_vec(); - - Ok(triggers) - } - - async fn fetch_unlistened_captures( - &self, - db: &DB, - ) -> Result>> { - let fields = vec![ - "path", - "is_flow", - "workspace_id", - "owner AS username", - "email", - "trigger_config", - ]; - - let mut sqlb = SqlBuilder::select_from("capture_config"); - sqlb.fields(&fields) - .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) - .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") - .and_where("trigger_config IS NOT NULL") - .and_where( - "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", - ); - - for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { - sqlb.and_where(where_clause); - } - - let sql = sqlb.sql().expect("failed to build SQL"); - - let captures: Vec> = - sqlx::query_as(&sql).fetch_all(db).await?; - - let captures = captures - .into_iter() - .map(|capture| ListeningTrigger { - username: capture.username, - path: capture.path, - workspace_id: capture.workspace_id, - script_path: "".to_string(), - email: capture.email, - trigger_config: capture.trigger_config, - trigger_mode: false, - is_flow: capture.is_flow, - error_handling: None, - suspended_mode: false, - }) - .collect_vec(); - - Ok(captures) - } - - async fn get_extra_state(&self) -> Option { - None - } - - async fn cleanup( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _extra: Option<&Self::ExtraState>, - ) -> Result<()> { - Ok(()) - } - - async fn loop_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - status: Arc>>, - error_message: Option, - ) { - update_rw_lock(status.clone(), error_message).await; - loop { - if let None = self - .update_ping(db, listening_trigger, status.read().await.as_deref()) - .await - { - return; - } - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } - } - - async fn update_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - if listening_trigger.trigger_mode { - self.update_trigger_ping(db, listening_trigger, error).await - } else { - self.update_capture_ping(db, listening_trigger, error).await - } - } - - async fn update_ping_and_loop_ping_status( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - loop_ping_status: Arc>>, - error: Option, - ) -> Option<()> { - // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status - update_rw_lock(loop_ping_status.clone(), error.clone()).await; - if let None = self - .update_ping(db, listening_trigger, error.as_deref()) - .await - { - return None; - } - Some(()) - } - - async fn update_trigger_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - let updated = sqlx::query_scalar::<_, i32>(&format!( - r#" - UPDATE - {} - SET - last_server_ping = now(), error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - server_id = $4 AND - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - RETURNING 1 - "#, - Self::TABLE_NAME - )) - .bind(error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .bind(&*INSTANCE_NAME) - .fetch_optional(db) - .await; - - self.handle_ping_result(updated, db, listening_trigger, "trigger") - .await - } - - async fn update_capture_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 AND - server_id = $6 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - &listening_trigger.workspace_id, - &listening_trigger.path, - &listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind, - &*INSTANCE_NAME - ) - .fetch_optional(db) - .await - .map(|result| result.flatten()); - - self.handle_ping_result(updated, db, listening_trigger, "capture") - .await - } - - async fn handle_ping_result( - &self, - result: sqlx::Result>, - db: &DB, - listening_trigger: &ListeningTrigger, - entity_type: &str, - ) -> Option<()> { - match result { - Ok(updated) => { - if updated.is_none() { - self.reset_ping_for_restart(db, listening_trigger).await; - tracing::info!( - "{} {} {} changed, disabled, or deleted, stopping...", - Self::TRIGGER_KIND, - entity_type, - listening_trigger.path - ); - return None; - } - } - Err(error) => { - tracing::warn!( - "Error updating ping of {} {} {}: {:?}", - Self::TRIGGER_KIND, - entity_type, - &listening_trigger.path, - error - ); - } - } - - Some(()) - } - - async fn reset_ping_for_restart( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - ) { - if listening_trigger.trigger_mode { - let _ = sqlx::query(&format!( - r#" - UPDATE - {} - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - server_id IS NULL - "#, - Self::TABLE_NAME - )) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) - .await; - } else { - let _ = sqlx::query!( - r#" - UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = $4 AND - server_id IS NULL - "#, - &listening_trigger.workspace_id, - &listening_trigger.path, - &listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind - ) - .execute(db) - .await; - } - } - - async fn disable_with_error( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: String, - ) { - if listening_trigger.trigger_mode { - let report_status = sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = 'disabled'::TRIGGER_MODE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - Self::TABLE_NAME - )) - .bind(&error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) - .await; - - match report_status { - Ok(_) => { - report_critical_error( - format!( - "Disabling {} trigger {} because of error: {}", - Self::TRIGGER_KIND, - listening_trigger.path, - error - ), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ).await; - } - } - return; - } - - let report_status = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 - "#, - error, - listening_trigger.workspace_id, - listening_trigger.path, - listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind - ) - .execute(db) - .await; - - if let Err(disable_err) = report_status { - tracing::error!( - "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", - Self::TRIGGER_KIND, - listening_trigger.path, - listening_trigger.workspace_id, - disable_err, - error - ) - } - } - - async fn handle_trigger( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - _extra: Option, - ) -> Result<()> { - let args = Self::build_job_args( - &listening_trigger.script_path, - listening_trigger.is_flow, - &listening_trigger.workspace_id, - db, - payload, - trigger_info, - ) - .await?; - - let authed = listening_trigger - .authed(db, &Self::TRIGGER_KIND.to_string()) - .await?; - - let (retry, error_handler_path, error_handler_args) = - match listening_trigger.error_handling.as_ref() { - Some(error_handling) => ( - error_handling.retry.as_ref(), - error_handling.error_handler_path.as_deref(), - error_handling.error_handler_args.as_ref(), - ), - None => (None, None, None), - }; - - tracing::debug!( - "Triggering job from {} event {} with args {:?}", - Self::TRIGGER_KIND, - listening_trigger.path, - args - ); - - trigger_runnable( - db, - None, - authed, - &listening_trigger.workspace_id, - &listening_trigger.script_path, - listening_trigger.is_flow, - args, - retry, - error_handler_path.as_deref(), - error_handler_args, - format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), - None, - listening_trigger.suspended_mode, - TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND), - ) - .await?; - - Ok(()) - } - - async fn handle_event( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - extra: Option, - ) -> Result<()> { - if listening_trigger.trigger_mode { - if let Err(err) = self - .handle_trigger(db, listening_trigger, payload, trigger_info, extra) - .await - { - report_critical_error( - format!( - "Failed to trigger job from {} event {}: {:?}", - Self::TRIGGER_KIND, - listening_trigger.path, - err - ), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ) - .await; - return Err(err); - }; - return Ok(()); - } - - let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); - if let Err(err) = insert_capture_payload( - db, - &listening_trigger.workspace_id, - &listening_trigger.path, - listening_trigger.is_flow, - &Self::TRIGGER_KIND, - main_args, - preprocessor_args, - &listening_trigger.username, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - return Err(err); - } - Ok(()) - } -} - -#[allow(unused)] -async fn listening( - db: DB, - listener: T, - listening_trigger: ListeningTrigger, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - let killpill_rx_consumer = killpill_rx.resubscribe(); - let killpill_rx_get_consumer = killpill_rx.resubscribe(); - - let loop_ping_status = Arc::new(RwLock::new(None)); - let extra_state = listener.get_extra_state().await; - let path = listening_trigger.path.clone(); - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - } - _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - } - consumer = { - tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path); - listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) - } => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path); - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { - tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path); - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - _ = async { - match consumer { - Ok(Some(consumer)) => { - listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; - tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path); - listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; - tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path); - } - Err(error) => { - tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error); - listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; - } - Ok(None) => { - tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path); - } - } - } => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - } - } - } -} - -#[allow(unused)] -async fn listen_to_unlistened_events( - listener: T, - db: DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; - - match unlistend_enabled_triggers { - Ok(mut unlistend_enabled_triggers) => { - unlistend_enabled_triggers.shuffle(&mut rand::rng()); - for trigger in unlistend_enabled_triggers { - let has_lock = sqlx::query_scalar(&format!( - r#" - UPDATE - {} - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - T::TABLE_NAME, - )) - .bind(&*INSTANCE_NAME) - .bind(&trigger.workspace_id) - .bind(&trigger.path) - .fetch_optional(&db) - .await; - match has_lock { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!( - "Spawning new task to listen for {} event", - T::TABLE_NAME - ); - tokio::spawn({ - let db = db.clone(); - let killpill_rx = killpill_rx.resubscribe(); - async move { listening(db, listener, trigger, killpill_rx).await } - }); - } else { - tracing::info!( - "{} trigger {} already being listened to", - T::TRIGGER_KIND, - trigger.path - ); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for {} trigger {}: {:?}", - T::TRIGGER_KIND, - trigger.path, - err - ); - } - }; - } - } - Err(err) => { - tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); - } - } - - let unlisted_captures = listener.fetch_unlistened_captures(&db).await; - - match unlisted_captures { - Ok(unlistened_captures) => { - for capture in unlistened_captures { - let has_lock = sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - &capture.workspace_id, - &capture.path, - &capture.is_flow, - T::TRIGGER_KIND as TriggerKind - ) - .fetch_optional(&db) - .await; - match has_lock { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn({ - let db = db.clone(); - let killpill_rx = killpill_rx.resubscribe(); - async move { listening(db, listener, capture, killpill_rx).await } - }); - } else { - tracing::info!( - "{} capture {} already being listened to", - T::TRIGGER_KIND.to_string(), - capture.path - ); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture {} {}: {:?}", - T::TRIGGER_KIND, - capture.path, - err - ); - } - }; - } - } - Err(err) => { - tracing::error!( - "Error fetching captures {} triggers: {:?}", - T::TRIGGER_KIND, - err - ); - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -struct Capture -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - path: String, - is_flow: bool, - workspace_id: String, - username: String, - email: String, - #[serde(flatten)] - trigger_config: T, -} - -impl FromRow<'_, sqlx::postgres::PgRow> for Capture -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, -{ - fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { - let trigger_config_value = row.try_get("trigger_config")?; - let trigger_config: T = serde_json::from_value(trigger_config_value) - .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; - Ok(Capture { - path: row.try_get("path")?, - is_flow: row.try_get("is_flow")?, - workspace_id: row.try_get("workspace_id")?, - username: row.try_get("username")?, - email: row.try_get("email")?, - trigger_config, - }) - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ListeningTrigger { - pub path: String, - pub is_flow: bool, - pub workspace_id: String, - pub username: String, - pub email: String, - pub trigger_config: T, - pub script_path: String, - pub trigger_mode: bool, - pub error_handling: Option, - pub suspended_mode: bool, -} - -impl ListeningTrigger { - pub async fn authed(&self, db: &DB, username: &str) -> Result { - fetch_api_authed( - self.username.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("{}-{}", username, self.path)), - ) - .await - } -} - -#[allow(unused)] -pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { - let mut w = lock.write().await; - *w = value; -} +// Re-export for backward compat with concrete trigger modules that use +// `crate::triggers::listener::ListeningTrigger` +#[allow(unused_imports)] +pub use windmill_trigger::listener::ListeningTrigger; #[allow(unused)] fn listen_to( diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index ce8538de2d..94c5039775 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -1,16 +1,4 @@ -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use sqlx::{types::Json as SqlxJson, FromRow}; -use std::{collections::HashMap, fmt::Debug}; -use windmill_common::jobs::JobTriggerKind; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum HandlerAction { - Trigger { path: String, trigger_kind: JobTriggerKind }, - // Future variants can be added here (e.g., Script, Flow, etc.) -} - +// Concrete trigger submodules (feature-gated) #[cfg(all(feature = "smtp", feature = "private"))] pub mod email; #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] @@ -30,153 +18,33 @@ pub mod sqs; #[cfg(feature = "websocket")] pub mod websocket; -pub mod filter; -pub mod global_handler; +// Assembly modules that stay local (reference concrete trigger types) mod handler; mod listener; -pub mod trigger_helpers; +// Re-export everything from windmill-trigger base crate +pub use windmill_trigger::capture; +pub use windmill_trigger::filter; +pub use windmill_trigger::global_handler; +pub use windmill_trigger::trigger_helpers; +pub use windmill_trigger::types::*; + +// Re-export traits #[allow(unused)] -pub(crate) use handler::TriggerCrud; +pub(crate) use windmill_trigger::Listener; +#[allow(unused)] +pub(crate) use windmill_trigger::TriggerCrud; + +// Re-export windmill-trigger's handler module items needed by concrete triggers +pub use windmill_trigger::handler::{ + complete_trigger_routes, trigger_routes, TriggerPrimarySchedule, +}; + +// Re-export windmill-trigger's listener module items needed by concrete triggers +pub use windmill_trigger::listener::{ + listen_to_unlistened_events, listening, update_rw_lock, Capture, ListeningTrigger, +}; + +// Assembly functions (stay local because they reference concrete trigger types) pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; pub use listener::start_all_listeners; -#[allow(unused)] -pub(crate) use listener::Listener; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StandardTriggerQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, - pub path_start: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct BaseTrigger { - pub workspace_id: String, - pub path: String, - pub script_path: String, - pub mode: TriggerMode, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: DateTime, - pub extra_perms: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct ServerState { - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct TriggerErrorHandling { - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct Trigger -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - #[serde(flatten)] - pub base: BaseTrigger, - - #[serde(flatten)] - pub config: T, - - #[serde(flatten)] - pub server_state: Option, - - #[serde(flatten)] - pub error_handling: TriggerErrorHandling, -} - -impl FromRow<'_, sqlx::postgres::PgRow> for Trigger -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { - let base = BaseTrigger::from_row(row)?; - - Ok(Trigger { - base, - config: T::from_row(row)?, - server_state: ServerState::from_row(row).ok(), - error_handling: TriggerErrorHandling::from_row(row)?, - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BaseTriggerData { - pub path: String, - pub script_path: String, - pub is_flow: bool, - #[deprecated(note = "Use mode instead")] - enabled: Option, // Kept for backwards compatibility, use mode instead - mode: Option, -} - -impl BaseTriggerData { - pub fn mode(&self) -> &TriggerMode { - self.mode.as_ref().unwrap_or( - #[allow(deprecated)] - if self.enabled.unwrap_or(true) { - &TriggerMode::Enabled - } else { - &TriggerMode::Disabled - }, - ) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TriggerData { - #[serde(flatten)] - pub base: BaseTriggerData, - - #[serde(flatten)] - pub config: T, - - #[serde(flatten)] - pub error_handling: TriggerErrorHandling, -} - -impl StandardTriggerQuery { - pub fn offset(&self) -> i64 { - let page = self.page.unwrap_or(0); - let per_page = self.per_page.unwrap_or(100); - (page * per_page) as i64 - } - - pub fn limit(&self) -> i64 { - self.per_page.unwrap_or(100) as i64 - } -} - -impl Default for StandardTriggerQuery { - fn default() -> Self { - Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] -#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum TriggerMode { - Enabled, - Disabled, - Suspended, -} diff --git a/backend/windmill-api/src/triggers/mqtt/mod.rs b/backend/windmill-api/src/triggers/mqtt/mod.rs index cfc8e36900..0effa0e75a 100644 --- a/backend/windmill-api/src/triggers/mqtt/mod.rs +++ b/backend/windmill-api/src/triggers/mqtt/mod.rs @@ -1,346 +1 @@ -use base64::{engine, prelude::*}; -use itertools::Itertools; -use rumqttc::{ - v5::{ - mqttbytes::{ - v5::{ConnectProperties, Filter}, - QoS as V5QoS, - }, - AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, - }, - AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, - QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, FromRow, Type}; -use std::{collections::HashMap, time::Duration}; -use windmill_common::{ - error::{to_anyhow, Error}, - triggers::TriggerKind, - worker::to_raw_value, -}; - -use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs}; - -pub mod handler; -pub mod listener; - -#[derive(Clone, Copy)] -pub struct MqttTrigger; - -impl TriggerJobArgs for MqttTrigger { - type Payload = Vec; - const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; - - fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { - let base64_payload = engine::general_purpose::STANDARD.encode(payload); - HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[serde(rename_all = "lowercase")] -pub enum QualityOfService { - Qos0, - Qos1, - Qos2, -} - -impl From for V3QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V3QoS::AtMostOnce, - QualityOfService::Qos1 => V3QoS::AtLeastOnce, - QualityOfService::Qos2 => V3QoS::ExactlyOnce, - } - } -} - -impl From for V5QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V5QoS::AtMostOnce, - QualityOfService::Qos1 => V5QoS::AtLeastOnce, - QualityOfService::Qos2 => V5QoS::ExactlyOnce, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MqttV3Config { - clean_session: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MqttV5Config { - clean_start: Option, - session_expiry_interval: Option, - topic_alias_maximum: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[sqlx(type_name = "MQTT_CLIENT_VERSION")] -#[sqlx(rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum MqttClientVersion { - V3, - V5, -} - -#[derive(Debug, Deserialize)] -pub struct Tls { - enabled: bool, - ca_certificate: String, - pkcs12_client_certificate: Option, - pkcs12_certificate_password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct Credentials { - username: Option, - password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct MqttResource { - broker: String, - port: u16, - credentials: Option, - tls: Option, -} - -#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] -pub struct SubscribeTopic { - qos: QualityOfService, - topic: String, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct MqttConfig { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec>, - pub v3_config: Option>, - pub v5_config: Option>, - pub client_id: Option, - pub client_version: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MqttConfigRequest { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec, - pub v3_config: Option, - pub v5_config: Option, - pub client_id: Option, - pub client_version: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestMqttConfig { - pub mqtt_resource_path: String, - pub client_version: Option, - pub v3_config: Option, - pub v5_config: Option, -} - -// Constants -pub const KEEP_ALIVE: u64 = 60; -pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; -pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; -pub const TIMEOUT_DURATION: u64 = 10; -pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); - -pub struct V3MqttHandler; -pub struct V5MqttHandler; - -pub enum MqttClientResult { - V3((V3MqttHandler, V3EventLoop)), - V5((V5MqttHandler, V5EventLoop)), -} - -#[derive(Debug, thiserror::Error)] -pub enum MqttError { - #[error("{0}")] - Common(#[from] Error), - #[error("{0}")] - V5RumqttClient(#[from] rumqttc::v5::ClientError), - #[error("{0}")] - V5ConnectionError(#[from] rumqttc::v5::ConnectionError), - #[error("{0}")] - V3RumqttClient(#[from] rumqttc::ClientError), - #[error("{0}")] - V3ConnectionError(#[from] rumqttc::ConnectionError), - #[error("{0}")] - Base64Decode(#[from] base64::DecodeError), -} - -pub struct MqttClientBuilder<'client> { - mqtt_resource: MqttResource, - client_id: &'client str, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, -} - -impl<'client> MqttClientBuilder<'client> { - pub fn new( - mqtt_resource: MqttResource, - client_id: Option<&'client str>, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, - ) -> Self { - Self { - mqtt_resource, - client_id: client_id.unwrap_or(""), - subscribe_topics, - v3_config, - v5_config, - mqtt_client_version, - } - } - - pub async fn build_client(&self) -> Result { - match self.mqtt_client_version { - Some(MqttClientVersion::V5) | None => self.build_v5_client().await, - Some(MqttClientVersion::V3) => self.build_v3_client().await, - } - } - - fn get_tls_configuration(&self) -> Result, Error> { - let transport = match self.mqtt_resource.tls { - Some(ref tls) if tls.enabled => { - let transport = match tls.ca_certificate.trim().is_empty() { - true => rumqttc::Transport::Tls(TlsConfiguration::Native), - false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { - ca: tls.ca_certificate.as_bytes().to_vec(), - client_auth: { - match tls.pkcs12_client_certificate.as_ref() { - Some(client_certificate) - if !client_certificate.trim().is_empty() => - { - let client_certificate = BASE64_STANDARD - .decode(client_certificate) - .map_err(to_anyhow)?; - let password = tls - .pkcs12_certificate_password - .clone() - .unwrap_or("".to_string()); - Some((client_certificate, password)) - } - _ => None, - } - }, - }), - }; - - Some(transport) - } - _ => None, - }; - - Ok(transport) - } - - async fn build_v5_client(&self) -> Result { - let mut mqtt_options = V5MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - - mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); - - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - - if let Some(v5_config) = self.v5_config { - mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); - mqtt_options.set_connect_properties(ConnectProperties { - session_expiry_interval: v5_config.session_expiry_interval, - receive_maximum: None, - max_packet_size: None, - topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), - request_response_info: None, - request_problem_info: None, - user_properties: vec![], - authentication_method: None, - authentication_data: None, - }); - } - - let (async_client, mut event_loop) = - V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) - } - - async fn build_v3_client(&self) -> Result { - let mut mqtt_options = V3MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - if let Some(v3_config) = self.v3_config { - mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); - } - - let (async_client, mut event_loop) = - V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) - } -} +pub use windmill_trigger_mqtt::*; diff --git a/backend/windmill-api/src/triggers/nats/mod.rs b/backend/windmill-api/src/triggers/nats/mod.rs index 668df073c9..a07227cf53 100644 --- a/backend/windmill-api/src/triggers/nats/mod.rs +++ b/backend/windmill-api/src/triggers/nats/mod.rs @@ -1,15 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct NatsTrigger; +pub use windmill_trigger_nats::*; diff --git a/backend/windmill-api/src/triggers/postgres/bool.rs b/backend/windmill-api/src/triggers/postgres/bool.rs deleted file mode 100644 index 13b0a8fa68..0000000000 --- a/backend/windmill-api/src/triggers/postgres/bool.rs +++ /dev/null @@ -1,24 +0,0 @@ -use thiserror::Error; - -/** -* This implementation is inspired by Postgres replication functionality -* from https://github.com/supabase/pg_replicate -* -* Original implementation: -* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs -* -*/ - -#[derive(Debug, Error)] -pub enum ParseBoolError { - #[error("invalid input value: {0}")] - InvalidInput(String), -} - -pub fn parse_bool(s: &str) -> Result { - match s { - "t" => Ok(true), - "f" => Ok(false), - _ => Err(ParseBoolError::InvalidInput(s.to_string())), - } -} diff --git a/backend/windmill-api/src/triggers/postgres/hex.rs b/backend/windmill-api/src/triggers/postgres/hex.rs deleted file mode 100644 index 2ada70dd35..0000000000 --- a/backend/windmill-api/src/triggers/postgres/hex.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::num::ParseIntError; - - -/** -* This implementation is inspired by Postgres replication functionality -* from https://github.com/supabase/pg_replicate -* -* Original implementation: -* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs -* -*/ - -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum ByteaHexParseError { - #[error("missing prefix '\\x'")] - InvalidPrefix, - - #[error("invalid byte")] - OddNumerOfDigits, - - #[error("parse int result: {0}")] - ParseInt(#[from] ParseIntError), -} - -pub fn from_bytea_hex(s: &str) -> Result, ByteaHexParseError> { - if s.len() < 2 || &s[..2] != "\\x" { - return Err(ByteaHexParseError::InvalidPrefix); - } - - let mut result = Vec::with_capacity((s.len() - 2) / 2); - let s = &s[2..]; - - if s.len() % 2 != 0 { - return Err(ByteaHexParseError::OddNumerOfDigits); - } - - for i in (0..s.len()).step_by(2) { - let val = u8::from_str_radix(&s[i..i + 2], 16)?; - result.push(val); - } - - Ok(result) -} diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs index ea267721a2..a13d931090 100644 --- a/backend/windmill-api/src/triggers/postgres/mod.rs +++ b/backend/windmill-api/src/triggers/postgres/mod.rs @@ -1,553 +1 @@ -use std::collections::HashMap; - -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - triggers::trigger_helpers::TriggerJobArgs, -}; -use chrono::Utc; -use itertools::Itertools; -use native_tls::{Certificate, TlsConnector}; -use pg_escape::quote_identifier; -use rand::Rng; -use rust_postgres::{config::SslMode, Client, Config, NoTls}; -use rust_postgres_native_tls::MakeTlsConnector; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::value::RawValue; -use sqlx::FromRow; -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - utils::empty_as_none, -}; - -mod bool; -mod converter; -pub mod handler; -mod hex; -pub mod listener; -mod mapper; -mod relation; -mod replication_message; - -#[derive(Clone, Copy)] -pub struct PostgresTrigger; - -impl TriggerJobArgs for PostgresTrigger { - type Payload = HashMap>; - const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; - fn v1_payload_fn(payload: &HashMap>) -> HashMap> { - payload.to_owned() - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct PostgresConfig { - pub postgres_resource_path: String, - pub replication_slot_name: String, - pub publication_name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub basic_mode: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PostgresConfigRequest { - postgres_resource_path: String, - #[serde(default)] - replication_slot_name: String, - #[serde(default)] - publication_name: String, - publication: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestPostgresConfig { - pub postgres_resource_path: String, -} - -fn check_if_valid_relation<'de, D>( - relations: D, -) -> std::result::Result>, D::Error> -where - D: Deserializer<'de>, -{ - let relations: Option> = Option::deserialize(relations)?; - let mut track_all_table_in_schema = false; - let mut track_specific_columns_in_table = false; - match relations { - Some(relations) => { - for relation in relations.iter() { - if relation.schema_name.is_empty() { - return Err(serde::de::Error::custom( - "Schema Name must not be empty".to_string(), - )); - } - - if !track_all_table_in_schema && relation.table_to_track.is_empty() { - track_all_table_in_schema = true; - continue; - } - - for table_to_track in relation.table_to_track.iter() { - if table_to_track.table_name.trim().is_empty() { - return Err(serde::de::Error::custom( - "Table name must not be empty".to_string(), - )); - } - - if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { - track_specific_columns_in_table = true; - } - } - - if track_all_table_in_schema && track_specific_columns_in_table { - return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); - } - } - - if !relations - .iter() - .map(|relation| relation.schema_name.as_str()) - .all_unique() - { - return Err(serde::de::Error::custom( - "You cannot choose a schema more than one time".to_string(), - )); - } - - Ok(Some(relations)) - } - None => Ok(None), - } -} - -fn check_if_valid_transaction_type<'de, D>( - transaction_type: D, -) -> std::result::Result, D::Error> -where - D: Deserializer<'de>, -{ - let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; - if transaction_type.len() > 3 { - return Err(serde::de::Error::custom( - "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" - .to_string(), - )); - } - transaction_type.sort_unstable(); - transaction_type.dedup(); - - for transaction in transaction_type.iter() { - match transaction.to_lowercase().as_ref() { - "insert" => {}, - "update" => {}, - "delete" => {}, - _ => { - return Err(serde::de::Error::custom( - "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" - .to_string(), - )) - } - } - } - - Ok(transaction_type) -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct PublicationData { - #[serde(default, deserialize_with = "check_if_valid_relation")] - pub table_to_track: Option>, - #[serde(deserialize_with = "check_if_valid_transaction_type")] - pub transaction_to_track: Vec, -} - -impl PublicationData { - pub fn new( - table_to_track: Option>, - transaction_to_track: Vec, - ) -> PublicationData { - PublicationData { table_to_track, transaction_to_track } - } -} - -// Slot list struct -#[derive(FromRow, Debug, Serialize)] -pub struct SlotList { - pub slot_name: Option, - pub active: Option, -} - -// Slot struct -#[derive(Debug, Serialize, Deserialize)] -pub struct Slot { - pub name: String, -} - -// Template script struct -#[derive(Debug, Deserialize)] -pub struct TemplateScript { - pub postgres_resource_path: String, - #[serde(deserialize_with = "check_if_valid_relation")] - pub relations: Option>, - pub language: Language, -} - -// Language enum -#[derive(Deserialize, Debug)] -pub enum Language { - #[serde(rename = "typescript", alias = "Typescript")] - Typescript, -} - -// Test postgres struct -#[derive(Serialize, Deserialize)] -pub struct TestPostgres { - pub postgres_resource_path: String, -} - -// PostgreSQL publication replication struct -#[derive(Serialize, Deserialize)] -pub struct PostgresPublicationReplication { - pub publication_name: String, - pub replication_slot_name: String, -} - -impl PostgresPublicationReplication { - pub fn new( - publication_name: String, - replication_slot_name: String, - ) -> PostgresPublicationReplication { - PostgresPublicationReplication { publication_name, replication_slot_name } - } -} - -pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -#[derive(FromRow, Serialize, Deserialize, Debug)] -pub struct Postgres { - pub user: String, - pub password: String, - pub host: String, - pub port: Option, - pub dbname: String, - #[serde(default)] - pub sslmode: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub root_certificate_pem: Option, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct TableToTrack { - pub table_name: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub where_clause: Option, - #[serde(default, deserialize_with = "empty_as_none")] - pub columns_name: Option>, -} - -impl TableToTrack { - pub fn new( - table_name: String, - where_clause: Option, - columns_name: Option>, - ) -> TableToTrack { - TableToTrack { table_name, where_clause, columns_name } - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct Relations { - pub schema_name: String, - pub table_to_track: Vec, -} - -impl Relations { - pub fn new(schema_name: String, table_to_track: Vec) -> Relations { - Relations { schema_name, table_to_track } - } - - pub fn add_new_table(&mut self, table_to_track: TableToTrack) { - self.table_to_track.push(table_to_track); - } -} - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = - Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new( - connector.build().map_err(to_anyhow)?, - ))) -} - -pub async fn get_raw_postgres_connection( - database: &Postgres, - logical_mode: bool, -) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - )) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode); - - if logical_mode { - config.replication_mode(rust_postgres::config::ReplicationMode::Logical); - } - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; - tokio::spawn(async move { - tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); - if let Err(e) = connection.await { - tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); - }; - tracing::info!("PostgreSQL trigger connection closed"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; - - Ok(client) -} - -pub async fn get_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, - logical_mode: bool, -) -> Result { - let database = - try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) - .await?; - - Ok(get_raw_postgres_connection(&database, logical_mode).await?) -} - -pub async fn get_default_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, -) -> Result { - get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await -} - -pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { - tx.execute( - &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - Ok(()) -} - -pub async fn check_if_valid_publication_for_postgres_version( - pg_connection: &Client, - table_to_track: Option<&[Relations]>, -) -> Result { - use crate::triggers::postgres::handler::get_postgres_version_internal; - - let postgres_version = get_postgres_version_internal(pg_connection).await?; - - let pg_14 = postgres_version.starts_with("14"); - if pg_14 { - let unsupported_publication = table_to_track - .and_then(|relations| { - relations.iter().find(|relation| { - let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { - table_to_track.where_clause.is_some() - || table_to_track.columns_name.is_some() - }); - - relation.table_to_track.is_empty() || invalid_relation.is_some() - }) - }) - .is_some(); - - if unsupported_publication { - return Err(Error::BadRequest( - "Your PostgreSQL database is running version 14, which does not support the following publication features: \ - - WHERE clause filtering, \ - - selective column tracking, and \ - - tracking all tables within a schema.\n\ - These features are only available in PostgreSQL 15 and above.".to_string(), - )); - } - } - Ok(pg_14) -} - -pub async fn create_pg_publication( - pg_connection: &Client, - publication_name: &str, - table_to_track: Option<&[Relations]>, - transaction_to_track: &[String], -) -> Result<()> { - let pg_14 = - check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; - let mut query = String::from("CREATE PUBLICATION "); - - query.push_str("e_identifier(publication_name)); - - match table_to_track { - Some(database_component) if !database_component.is_empty() => { - query.push_str(" FOR"); - let mut first = true; - for (i, schema) in database_component.iter().enumerate() { - if schema.table_to_track.is_empty() { - query.push_str(" TABLES IN SCHEMA "); - query.push_str("e_identifier(&schema.schema_name)); - } else { - if pg_14 && first { - query.push_str(" TABLE ONLY "); - first = false - } else if !pg_14 { - query.push_str(" TABLE ONLY "); - } - for (j, table) in schema.table_to_track.iter().enumerate() { - let table_name = quote_identifier(&table.table_name); - let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", &schema_name, &table_name); - query.push_str(&full_name); - if let Some(columns) = table.columns_name.as_ref() { - query.push_str(" ("); - let columns = columns - .iter() - .map(|column| quote_identifier(column)) - .join(", "); - query.push_str(&columns); - query.push_str(")"); - } - - if let Some(where_clause) = &table.where_clause { - query.push_str(" WHERE ("); - query.push_str(where_clause); - query.push(')'); - } - - if j + 1 != schema.table_to_track.len() { - query.push_str(", "); - } - } - } - if i < database_component.len() - 1 { - query.push_str(", "); - } - } - } - _ => { - query.push_str(" FOR ALL TABLES "); - } - }; - - if !transaction_to_track.is_empty() { - let transactions = || transaction_to_track.iter().join(", "); - query.push_str(" WITH (publish = '"); - query.push_str(&transactions()); - query.push_str("');"); - } - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - Ok(()) -} - -pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { - let mut query = String::from("DROP PUBLICATION IF EXISTS "); - let quoted_publication_name = quote_identifier(publication_name); - query.push_str("ed_publication_name); - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} - -pub fn generate_random_string() -> String { - let timestamp = Utc::now().timestamp_millis().to_string(); - let mut rng = rand::rng(); - let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; - - let random_part = (0..10) - .map(|_| { - charset - .chars() - .nth(rng.random_range(0..charset.len())) - .unwrap() - }) - .collect::(); - - format!("{}_{}", timestamp, random_part) -} +pub use windmill_trigger_postgres::*; diff --git a/backend/windmill-api/src/triggers/postgres/relation.rs b/backend/windmill-api/src/triggers/postgres/relation.rs deleted file mode 100644 index f313efb893..0000000000 --- a/backend/windmill-api/src/triggers/postgres/relation.rs +++ /dev/null @@ -1,74 +0,0 @@ -use core::str; - -use serde_json::{Map, Value}; -use std::{collections::HashMap, str::Utf8Error}; - -use super::{ - converter::{Converter, ConverterError}, - replication_message::{Columns, RelationBody, TupleData}, -}; -use rust_postgres::types::Oid; -#[derive(Debug, thiserror::Error)] -pub enum RelationConversionError { - #[error("Could not find matching table")] - FailToFindMatchingTable, - - #[error("Binary data not supported")] - BinaryFormatNotSupported, - - #[error("decode error: {0}")] - FromBytes(#[from] ConverterError), - - #[error("invalid string value")] - InvalidStr(#[from] Utf8Error), -} - -pub struct RelationConverter(HashMap); - -impl RelationConverter { - pub fn new() -> Self { - Self(HashMap::new()) - } - - pub fn add_relation(&mut self, relation: RelationBody) { - self.0.insert(relation.o_id, relation); - } - - pub fn get_columns(&self, o_id: Oid) -> Result<&Columns, RelationConversionError> { - self.0 - .get(&o_id) - .map(|relation_body| &relation_body.columns) - .ok_or(RelationConversionError::FailToFindMatchingTable) - } - - pub fn get_relation(&self, o_id: Oid) -> Result<&RelationBody, RelationConversionError> { - self.0 - .get(&o_id) - .ok_or(RelationConversionError::FailToFindMatchingTable) - } - - pub fn row_to_json( - &self, - to_decode: (Oid, Vec), - ) -> Result, RelationConversionError> { - let (o_id, tuple_data) = to_decode; - let mut object: Map = Map::new(); - let columns = self.get_columns(o_id)?; - - for (i, column) in columns.iter().enumerate() { - let value = match &tuple_data[i] { - TupleData::Null | TupleData::UnchangedToast => Value::Null, - TupleData::Binary(_) => { - return Err(RelationConversionError::BinaryFormatNotSupported) - } - TupleData::Text(bytes) => { - let str = str::from_utf8(&bytes[..])?; - Converter::try_from_str(column.type_o_id.clone(), str)? - } - }; - - object.insert(column.name.clone(), value); - } - Ok(object) - } -} diff --git a/backend/windmill-api/src/triggers/sqs/mod.rs b/backend/windmill-api/src/triggers/sqs/mod.rs index 8c7217725c..1c85eebbc3 100644 --- a/backend/windmill-api/src/triggers/sqs/mod.rs +++ b/backend/windmill-api/src/triggers/sqs/mod.rs @@ -1,15 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct SqsTrigger; +pub use windmill_trigger_sqs::*; diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs index f20b6334f7..7833ee704b 100644 --- a/backend/windmill-api/src/triggers/websocket/mod.rs +++ b/backend/windmill-api/src/triggers/websocket/mod.rs @@ -1,128 +1 @@ -use std::collections::HashMap; - -use crate::{ - db::ApiAuthed, - triggers::trigger_helpers::{ - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, - }, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, FromRow}; -use windmill_common::{ - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerMetadata, TriggerKind}, - worker::to_raw_value, - DB, -}; -use windmill_queue::PushArgsOwned; - -mod handler; -mod listener; - -#[derive(Copy, Clone)] -pub struct WebsocketTrigger; - -impl TriggerJobArgs for WebsocketTrigger { - type Payload = String; - const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; - fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { - HashMap::from([("msg".to_string(), to_raw_value(&payload))]) - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct WebsocketConfig { - pub url: String, - #[serde(default)] - pub filters: Vec>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub initial_messages: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_runnable_args: Option>>, - #[serde(default)] - pub can_return_message: bool, - #[serde(default)] - pub can_return_error_result: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebsocketConfigRequest { - url: String, - filters: Vec, - initial_messages: Option>, - url_runnable_args: Option, - can_return_message: bool, - can_return_error_result: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestWebsocketConfig { - url: String, - url_runnable_args: Option, -} - -pub fn value_to_args_hashmap( - args: Option<&Box>, -) -> Result>> { - let args = if let Some(args) = args { - let args_map: Option> = serde_json::from_str(args.get()) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; - - args_map - .unwrap_or_else(HashMap::new) - .into_iter() - .map(|(k, v)| { - let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { - Error::BadRequest(format!("failed to convert to raw value: {}", e)) - })?; - Ok((k, raw_value)) - }) - .collect::>>>() - } else { - Ok(HashMap::new()) - }?; - Ok(args) -} - -pub async fn get_url_from_runnable_value( - path: &str, - is_flow: bool, - db: &DB, - authed: ApiAuthed, - args: Option<&Box>, - workspace_id: &str, -) -> Result { - tracing::info!( - "Running {} {} to get WebSocket URL", - if is_flow { "flow" } else { "script" }, - path - ); - - let args = value_to_args_hashmap(args)?; - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed, - workspace_id, - path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket), - ) - .await?; - - serde_json::from_str::(result.get()).map_err(|_| { - Error::BadConfig(format!( - "{} {} did not return a string", - if is_flow { "Flow" } else { "Script" }, - path, - )) - }) -} +pub use windmill_trigger_websocket::*; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 4986525d21..aa1f73405b 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -6,1747 +6,51 @@ * LICENSE-AGPL for a copy of the license. */ -#![allow(non_snake_case)] +// Re-export everything from windmill-api-users +pub use windmill_api_users::users::*; -use quick_cache::sync::Cache; -use sqlx::{PgConnection, Postgres, Transaction}; - -use std::sync::atomic::AtomicBool; use std::sync::Arc; -use std::time::Duration; use crate::db::ApiAuthed; - -pub use crate::auth::Tokened; - use crate::secret_backend_ext::rename_vault_secrets_with_prefix; -use crate::utils::{ - generate_instance_wide_unique_username, get_instance_username_or_create_pending, -}; -use crate::{ - auth::ExpiringAuthCache, db::DB, utils::require_super_admin, webhook_util::WebhookShared, - COOKIE_DOMAIN, IS_SECURE, -}; -use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use argon2::Argon2; use axum::{ - async_trait, - extract::{Extension, FromRequestParts, Path, Query}, - http::request::Parts, - response::{IntoResponse, Response}, - routing::{delete, get, post}, + extract::{Extension, Path}, + routing::post, Json, Router, }; -use hyper::{header::LOCATION, StatusCode}; -use lazy_static::lazy_static; -use regex::Regex; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use time::OffsetDateTime; -use tower_cookies::{Cookie, Cookies}; -use tracing::Instrument; +use hyper::StatusCode; +use serde::Deserialize; +use windmill_api_auth::require_super_admin; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; -use windmill_common::auth::{fetch_authed_from_permissioned_as, TOKEN_PREFIX_LEN}; -use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; -use windmill_common::oauth2::InstanceEvent; -use windmill_common::users::COOKIE_NAME; -use windmill_common::users::{truncate_token, username_to_permissioned_as}; -use windmill_common::utils::paginate; -use windmill_common::worker::CLOUD_HOSTED; -use windmill_common::BASE_URL; +use windmill_common::usernames::VALID_USERNAME; use windmill_common::{ - auth::{get_folders_for_user, get_groups_for_user}, - db::UserDB, - error::{self, Error, JsonResult, Result}, - utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, + error::{self, Error, Result}, + DB, }; -use windmill_git_sync::handle_deployment_metadata; - -const COOKIE_PATH: &str = "/"; - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_users)) - .route("/list_usage", get(list_user_usage)) - .route("/list_usernames", get(list_usernames)) - .route("/exists", post(exists_username)) - .route("/get/:user", get(get_workspace_user)) - .route("/update/:user", post(update_workspace_user)) - .route("/delete/:user", delete(delete_workspace_user)) - .route("/convert_to_group/:user", post(convert_user_to_group)) - .route("/is_owner/*path", get(is_owner_of_path)) - .route("/whois/:username", get(whois)) - .route("/whoami", get(whoami)) - .route("/leave", post(leave_workspace)) - .route("/username_to_email/:username", get(username_to_email)) -} +/// Wraps the subcrate's global_service with routes that depend on windmill-api internals. pub fn global_service() -> Router { - Router::new() - .route("/exists/:email", get(exists_email)) - .route("/email", get(get_email)) - .route("/whoami", get(global_whoami)) - .route("/list_invites", get(list_invites)) - .route("/decline_invite", post(decline_invite)) - .route("/accept_invite", post(accept_invite)) - .route("/list_as_super_admin", get(list_users_as_super_admin)) + windmill_api_users::users::global_service() .route("/setpassword", post(set_password)) .route("/set_password_of/:user", post(set_password_of_user)) - .route("/set_login_type/:user", post(set_login_type)) .route("/create", post(create_user)) - .route("/update/:user", post(update_user)) - .route("/delete/:user", delete(delete_user)) - .route("/username_info/:user", get(get_instance_username_info)) .route("/rename/:user", post(rename_user)) - .route("/tokens/create", post(create_token)) - .route("/tokens/delete/:token_prefix", delete(delete_token)) - .route("/tokens/list", get(list_tokens)) - .route("/tokens/impersonate", post(impersonate)) - .route("/usage", get(get_usage)) - .route("/all_runnables", get(get_all_runnables)) - .route("/refresh_token", get(refresh_token)) - .route( - "/tutorial_progress", - post(update_tutorial_progress).get(get_tutorial_progress), - ) - .route("/leave_instance", post(leave_instance)) - .route("/export", get(export_global_users)) - .route("/overwrite", post(overwrite_global_users)) .route("/onboarding", post(submit_onboarding_data)) - - // .route("/list_invite_codes", get(list_invite_codes)) - // .route("/create_invite_code", post(create_invite_code)) - // .route("/signup", post(signup)) - // .route("/lost_password", post(lost_password)) - // .route("/use_magic_link", get(use_magic_link)) } +/// Wraps the subcrate's make_unauthed_service with routes that depend on windmill-api internals. pub fn make_unauthed_service() -> Router { - Router::new() - .route("/login", post(login)) - .route("/logout", post(logout).get(logout)) - .route("/is_first_time_setup", get(is_first_time_setup)) - .route("/request_password_reset", post(request_password_reset)) + windmill_api_users::users::make_unauthed_service() .route("/reset_password", post(reset_password)) - .route("/is_smtp_configured", get(is_smtp_configured)) -} - -pub async fn maybe_refresh_folders( - path: &str, - w_id: &str, - authed: ApiAuthed, - db: &DB, -) -> ApiAuthed { - if authed.is_admin { - return authed; - } - let splitted = path.split('/').collect::>(); - if splitted.len() >= 2 - && splitted[0] == "f" - && !authed.folders.iter().any(|(f, _, _)| f == splitted[1]) - { - let name = &authed.username; - let groups = get_groups_for_user(w_id, name, &authed.email, db) - .await - .ok() - .unwrap_or_default(); - - let folders = get_folders_for_user(w_id, name, &groups, db) - .await - .ok() - .unwrap_or_default(); - ApiAuthed { folders, ..authed } - } else { - authed - } -} - -pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { - authed.scopes.as_ref()?.iter().find_map(|s| { - if s.starts_with("if_jobs:filter_tags:") { - Some( - s.trim_start_matches("if_jobs:filter_tags:") - .split(",") - .collect::>(), - ) - } else { - None - } - }) -} - -#[derive(Clone, Debug)] -pub struct OptAuthed(pub Option); - -#[async_trait] -impl FromRequestParts for OptAuthed -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - ApiAuthed::from_request_parts(parts, state) - .await - .map(|authed| Self(Some(authed))) - .or_else(|_| Ok(Self(None))) - } -} - -#[allow(unused)] -pub async fn fetch_api_authed( - username: String, - email: String, - w_id: &str, - db: &DB, - username_override: Option, -) -> error::Result { - let permissioned_as = username_to_permissioned_as(username.as_str()); - fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await -} - -lazy_static::lazy_static! { - static ref API_AUTHED_CACHE: Cache<(String,String,String), ExpiringAuthCache> = Cache::new(300); -} - -#[allow(unused)] -pub async fn fetch_api_authed_from_permissioned_as( - permissioned_as: String, - email: String, - w_id: &str, - db: &DB, - username_override: Option, -) -> error::Result { - let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); - - let mut api_authed = match API_AUTHED_CACHE.get(&key) { - Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => { - tracing::debug!("API authed cache hit for user {}", email); - expiring_authed.authed - } - _ => { - tracing::debug!("API authed cache miss for user {}", email); - let authed = - fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; - - let api_authed = ApiAuthed { - username: authed.username, - email, - is_admin: authed.is_admin, - is_operator: authed.is_operator, - groups: authed.groups, - folders: authed.folders, - scopes: authed.scopes, - username_override: None, - token_prefix: authed.token_prefix, - }; - - API_AUTHED_CACHE.insert( - key, - ExpiringAuthCache { - authed: api_authed.clone(), - expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(), - job_id: None, - }, - ); - - api_authed - } - }; - - api_authed.username_override = username_override; - Ok(api_authed) -} - -#[derive(FromRow, Serialize)] -pub struct User { - pub workspace_id: String, - pub email: String, - pub username: String, - pub is_admin: bool, - pub created_at: chrono::DateTime, - pub operator: bool, - pub disabled: bool, - pub role: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub added_via: Option, -} - -#[derive(Serialize)] -pub struct UserWithUsage { - pub email: String, - pub executions: Option, -} - -#[derive(FromRow, Serialize, Debug)] -pub struct GlobalUserInfo { - email: String, - login_type: Option, - super_admin: bool, - devops: bool, - verified: bool, - name: Option, - company: Option, - username: Option, - #[serde(skip_serializing_if = "Option::is_none")] - operator_only: Option, - first_time_user: bool, -} - -#[derive(Serialize, Debug)] -pub struct UserInfo { - pub workspace_id: String, - pub email: String, - pub username: String, - pub is_admin: bool, - pub is_super_admin: bool, - pub created_at: chrono::DateTime, - pub groups: Vec, - pub operator: bool, - pub disabled: bool, - pub role: Option, - pub folders_read: Vec, - pub folders: Vec, - pub folders_owners: Vec, - pub name: Option, -} - -#[derive(FromRow, Serialize)] -pub struct WorkspaceInvite { - pub workspace_id: String, - pub email: String, - pub is_admin: bool, - pub operator: bool, - pub parent_workspace_id: Option, -} - -#[allow(dead_code)] -#[derive(Deserialize)] -pub struct NewUser { - pub email: String, - pub password: String, - pub super_admin: bool, - pub name: Option, - pub company: Option, - pub skip_email: Option, -} - -#[derive(Deserialize)] -pub struct AcceptInvite { - pub workspace_id: String, - pub username: Option, -} - -#[derive(Deserialize)] -pub struct DeclineInvite { - pub workspace_id: String, -} - -#[derive(Deserialize)] -pub struct EditUser { - pub is_super_admin: Option, - pub is_devops: Option, - pub name: Option, -} - -#[derive(Deserialize)] -pub struct EditWorkspaceUser { - pub is_admin: Option, - pub operator: Option, - pub disabled: Option, -} - -#[allow(dead_code)] -#[derive(Deserialize)] -pub struct EditPassword { - pub password: String, -} - -#[derive(Deserialize)] -pub struct EditLoginType { - pub login_type: String, -} - -#[derive(FromRow, Serialize)] -pub struct TruncatedToken { - pub label: Option, - pub token_prefix: Option, - pub expiration: Option>, - pub created_at: chrono::DateTime, - pub last_used_at: chrono::DateTime, - pub scopes: Option>, -} - -#[derive(Deserialize)] -pub struct NewToken { - pub label: Option, - pub expiration: Option>, - pub impersonate_email: Option, - pub scopes: Option>, - pub workspace_id: Option, -} - -#[cfg(feature = "native_trigger")] -impl NewToken { - pub fn new( - label: Option, - expiration: Option>, - impersonate_email: Option, - scopes: Option>, - workspace_id: Option, - ) -> NewToken { - NewToken { label, expiration, impersonate_email, scopes, workspace_id } - } -} - -#[derive(Deserialize)] -pub struct Login { - pub email: String, - pub password: String, -} - -lazy_static::lazy_static! { - static ref FIRST_TIME_SETUP: Arc = Arc::new(AtomicBool::new(true)); -} - -pub async fn is_first_time_setup(Extension(db): Extension) -> JsonResult { - if !FIRST_TIME_SETUP.load(std::sync::atomic::Ordering::Relaxed) { - return Ok(Json(false)); - } - let single_user = sqlx::query_scalar!("SELECT 1 FROM password LIMIT 2") - .fetch_all(&db) - .await - .ok() - .unwrap_or_default() - .len() - == 1; - if single_user { - let user_is_admin_and_password_changeme = sqlx::query_scalar!( - "SELECT 1 FROM password WHERE email = 'admin@windmill.dev' AND password_hash = '$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I'" - ).fetch_all(&db) - .await - .ok() - .unwrap_or_default() - .len() == 1; - if user_is_admin_and_password_changeme { - let base_url_is_not_set = - sqlx::query_scalar!("SELECT COUNT(*) FROM global_settings WHERE name = 'base_url'") - .fetch_optional(&db) - .await - .ok() - .flatten() - .flatten() - .unwrap_or(0) - == 0; - if base_url_is_not_set { - return Ok(Json(true)); - } - } - } - FIRST_TIME_SETUP.store(false, std::sync::atomic::Ordering::Relaxed); - Ok(Json(false)) -} - -#[derive(Deserialize)] -struct WorkspaceUsername { - pub username: String, -} - -async fn exists_username( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(WorkspaceUsername { username }): Json, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", - &w_id, - &username - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - tx.commit().await?; - Ok(Json(exists)) -} - -async fn list_users( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - if *CLOUD_HOSTED && w_id == "demo" { - require_admin(authed.is_admin, &authed.username)?; - } - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as!( - User, - " - SELECT * - FROM usr - WHERE workspace_id = $1 - ", - w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_user_usage( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - if *CLOUD_HOSTED && w_id == "demo" { - require_admin(authed.is_admin, &authed.username)?; - } - let mut tx = user_db.begin(&authed).await?; - let rows = tokio::time::timeout( - Duration::from_secs(300), - sqlx::query_as!( - UserWithUsage, - " - SELECT usr.email, usage.executions - FROM usr, LATERAL ( - SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions - FROM v2_job_completed c JOIN v2_job j USING (id) - WHERE j.workspace_id = $1 - AND j.kind NOT IN ('flow', 'flowpreview', 'flownode') - AND j.permissioned_as_email = usr.email - AND now() - '1 week'::interval < j.created_at - ) usage - WHERE workspace_id = $1 - ", - w_id - ) - .fetch_all(&mut *tx), - ) - .await - .map_err(|e| Error::internal_err(format!("Timed out while fetching user usage: {e:#}")))??; - tx.commit().await?; - Ok(Json(rows)) -} - -#[derive(Deserialize)] -struct ActiveUsersOnly { - active_only: Option, -} - -async fn list_users_as_super_admin( - authed: ApiAuthed, - Extension(db): Extension, - Query(pagination): Query, - Query(ActiveUsersOnly { active_only }): Query, -) -> JsonResult> { - require_super_admin(&db, &authed.email).await?; - let per_page = pagination.per_page.unwrap_or(10000).max(1); - let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; - - let rows = if active_only.is_some_and(|x| x) { - sqlx::query_as!( - GlobalUserInfo, - "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), - authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) - SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user - FROM password - WHERE email IN (SELECT email FROM active_users) - ORDER BY super_admin DESC, devops DESC - LIMIT $1 OFFSET $2", - per_page as i32, - offset as i32 - ) - .fetch_all(&db) - .await? - } else { - sqlx::query_as!( - GlobalUserInfo, - "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \ - $1 OFFSET $2", - per_page as i32, - offset as i32 - ) - .fetch_all(&db) - .await? - }; - - Ok(Json(rows)) -} - -#[derive(Serialize, Deserialize)] -struct Progress { - progress: u64, - skipped_all: bool, -} -async fn get_tutorial_progress( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult { - let row = sqlx::query!( - "SELECT progress::bigint as progress, skipped_all FROM tutorial_progress WHERE email = $1", - authed.email - ) - .fetch_optional(&db) - .await?; - - if let Some(row) = row { - Ok(Json(Progress { - progress: row.progress.unwrap_or_default() as u64, - skipped_all: row.skipped_all, - })) - } else { - Ok(Json(Progress { progress: 0, skipped_all: false })) - } -} - -async fn update_tutorial_progress( - authed: ApiAuthed, - Extension(db): Extension, - Json(progress): Json, -) -> Result { - sqlx::query!( - "INSERT INTO tutorial_progress (email, progress, skipped_all) VALUES ($2, $1::bigint::bit(64), $3) ON CONFLICT (email) DO UPDATE SET progress = EXCLUDED.progress, skipped_all = EXCLUDED.skipped_all", - progress.progress as i64, - authed.email, - progress.skipped_all - ) - .execute(&db) - .await?; - Ok("tutorial progress updated".to_string()) -} - -async fn list_usernames( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - if *CLOUD_HOSTED && w_id == "demo" { - return Ok(Json(vec![ - authed.username, - "other_usernames_redacted_in_demo_workspace".to_string(), - ])); - } - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_invites( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - let mut tx = db.begin().await?; - let rows = sqlx::query_as!( - WorkspaceInvite, - "SELECT - workspace_invite.workspace_id, - workspace_invite.email, - workspace_invite.is_admin, - workspace_invite.operator, - workspace.parent_workspace_id - FROM workspace_invite JOIN workspace ON workspace_invite.workspace_id = workspace.id WHERE email = $1", - authed.email - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -lazy_static::lazy_static! { - static ref INVALIDATE_ALL_SESSIONS_ON_LOGOUT: bool = std::env::var("INVALIDATE_ALL_SESSIONS_ON_LOGOUT") - .unwrap_or("false".to_string()) - .parse::() - .unwrap_or(false); -} - -#[derive(Deserialize)] -struct LogoutQuery { - rd: Option, -} -async fn logout( - Tokened { token }: Tokened, - cookies: Cookies, - Extension(db): Extension, - Query(LogoutQuery { rd }): Query, -) -> Result { - let mut cookie = Cookie::new(COOKIE_NAME, ""); - cookie.set_path(COOKIE_PATH); - if COOKIE_DOMAIN.is_some() { - cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); - } - cookies.remove(cookie); - let mut tx = db.begin().await?; - - let email = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT { - sqlx::query_scalar!( - "WITH email_lookup AS ( - SELECT email FROM token WHERE token = $1 - ) - DELETE FROM token - WHERE email = (SELECT email FROM email_lookup) AND label = 'session' - RETURNING email", - token - ) - .fetch_optional(&mut *tx) - .await? - } else { - sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token) - .fetch_optional(&mut *tx) - .await? - }; - - if let Some(email) = email { - let email = email.unwrap_or("noemail".to_string()); - let audit_message = if *INVALIDATE_ALL_SESSIONS_ON_LOGOUT { - "users.logout_all" - } else { - "users.logout" - }; - audit_log( - &mut *tx, - &AuditAuthor { - email: email.clone(), - username: email, - username_override: None, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }, - audit_message, - ActionKind::Delete, - "global", - Some(&truncate_token(&token)), - None, - ) - .await?; - } - tx.commit().await?; - if let Some(rd) = rd { - Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response()) - } else { - Ok((StatusCode::OK, "logged out successfully".to_string()).into_response()) - } -} - -async fn whoami( - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed, -) -> JsonResult { - let user = get_user(&w_id, &username, &db).await?; - if let Some(user) = user { - Ok(Json(user)) - } else { - Ok(Json(UserInfo { - workspace_id: w_id, - email: email.clone(), - username: email, - name: None, - is_admin, - is_super_admin: is_admin, - created_at: chrono::Utc::now(), - groups: groups, - operator: false, - disabled: false, - role: Some("superadmin".to_string()), - folders_read: folders.clone().into_iter().map(|x| x.0).collect(), - folders: folders - .clone() - .into_iter() - .filter_map(|x| if x.1 { Some(x.0) } else { None }) - .collect(), - folders_owners: folders - .into_iter() - .filter_map(|x| if x.2 { Some(x.0) } else { None }) - .collect(), - })) - } -} - -async fn global_whoami( - Extension(db): Extension, - ApiAuthed { email, .. }: ApiAuthed, - Tokened { token }: Tokened, -) -> JsonResult { - let user = sqlx::query_as!( - GlobalUserInfo, - "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user FROM password WHERE \ - email = $1", - email - ) - .fetch_one(&db) - .await - .map_err(|e| Error::internal_err(format!("fetching global identity: {e:#}"))); - - if let Ok(user) = user { - Ok(Json(user)) - } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) { - Ok(Json(GlobalUserInfo { - email: email.clone(), - login_type: Some("superadmin_secret".to_string()), - super_admin: true, - devops: false, - verified: true, - name: None, - company: None, - username: None, - operator_only: None, - first_time_user: false, - })) - } else { - Err(user.unwrap_err()) - } -} - -async fn exists_email(Extension(db): Extension, Path(email): Path) -> JsonResult { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", - email - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - Ok(Json(exists)) -} - -async fn get_email(ApiAuthed { email, .. }: ApiAuthed) -> Result { - Ok(email) -} - -async fn get_usage( - Extension(db): Extension, - ApiAuthed { email, .. }: ApiAuthed, -) -> Result { - let usage = sqlx::query_scalar!( - " - SELECT usage.usage FROM usage - WHERE is_workspace = false - AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) - AND id = $1", - email - ) - .fetch_optional(&db) - .await? - .unwrap_or(0); - Ok(usage.to_string()) -} - -#[derive(FromRow, Serialize)] -pub struct User2 { - pub workspace_id: String, - pub email: String, - pub username: String, - pub is_admin: bool, - pub created_at: chrono::DateTime, - pub operator: bool, - pub disabled: bool, - pub role: Option, - pub super_admin: bool, - pub name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub added_via: Option, -} - -async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { - let user = sqlx::query_as!( - User2, - "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2 - ", - username, - w_id - ) - .fetch_optional(db) - .await?; - let groups = get_groups_for_user( - &w_id, - username, - &user - .as_ref() - .map(|x| x.email.to_string()) - .unwrap_or_else(|| "".to_string()), - db, - ) - .await?; - let folders = get_folders_for_user(&w_id, username, &groups, db).await?; - - Ok(user.map(|usr| UserInfo { - groups, - workspace_id: usr.workspace_id, - email: usr.email, - username: usr.username, - name: usr.name, - is_admin: usr.is_admin, - is_super_admin: usr.super_admin, - created_at: usr.created_at, - operator: usr.operator, - disabled: usr.disabled, - role: usr.role, - folders_read: folders.clone().into_iter().map(|x| x.0).collect(), - folders: folders - .clone() - .into_iter() - .filter_map(|x| if x.1 { Some(x.0) } else { None }) - .collect(), - folders_owners: folders - .into_iter() - .filter_map(|x| if x.2 { Some(x.0) } else { None }) - .collect(), - })) -} - -pub async fn is_owner_of_path( - authed: ApiAuthed, - Path((_w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - if authed.is_admin { - Ok(Json(true)) - } else { - Ok(Json(require_owner_of_path(&authed, path).is_ok())) - } -} - -pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { - if authed.is_admin { - return Ok(()); - } - if !path.is_empty() { - let splitted = path.split("/").collect::>(); - if splitted[0] == "u" { - if splitted[1] == authed.username { - Ok(()) - } else { - Err(Error::BadRequest(format!( - "only the owner {} is authorized to perform this operation", - splitted[1] - ))) - } - } else if splitted[0] == "f" { - crate::folders::require_is_owner(authed, splitted[1]) - } else { - Err(Error::BadRequest(format!( - "Not recognized path kind: {}", - path - ))) - } - } else { - Err(Error::BadRequest(format!( - "Cannot be owner of an empty path" - ))) - } -} - -/// Checks that a user has at least read access to the path for preview jobs. -/// This prevents privilege escalation where a user could run preview code -/// under a path they don't have access to. -pub fn require_path_read_access_for_preview( - authed: &ApiAuthed, - path: &Option, -) -> Result<()> { - let Some(path) = path else { - return Ok(()); - }; - - if authed.is_admin { - return Ok(()); - } - - if path.is_empty() { - return Ok(()); - } - - let splitted: Vec<&str> = path.split('/').collect(); - if splitted.len() < 2 { - return Err(Error::BadRequest(format!( - "Invalid path format for preview job: {}", - path - ))); - } - - match splitted[0] { - "u" => { - if splitted[1] == authed.username { - Ok(()) - } else { - Err(Error::BadRequest(format!( - "You can only run preview jobs in your own namespace (u/{}) or in folders you have read access to", - authed.username - ))) - } - } - "f" => { - let folder = splitted[1]; - if authed.folders.iter().any(|(f, _, _)| f == folder) { - Ok(()) - } else { - Err(Error::BadRequest(format!( - "You do not have read access to folder '{}'. Preview jobs require at least read access to the target folder.", - folder - ))) - } - } - "hub" => Ok(()), - _ => Err(Error::BadRequest(format!( - "Invalid path format for preview job: {}. Path must start with 'u/' or 'f/'", - path - ))), - } -} - -pub fn get_perm_in_extra_perms_for_authed( - v: serde_json::Value, - authed: &ApiAuthed, -) -> Option { - match v { - serde_json::Value::Object(obj) => { - let mut keys = vec![format!("u/{}", authed.username)]; - for g in authed.groups.iter() { - keys.push(format!("g/{}", g)); - } - let mut res = None; - for k in keys { - if let Some(v) = obj.get(&k) { - if let Some(v) = v.as_bool() { - if v { - return Some(true); - } - res = Some(v); - } - } - } - res - } - _ => None, - } -} - -pub async fn require_is_writer( - authed: &ApiAuthed, - path: &str, - w_id: &str, - db: DB, - query: &str, - kind: &str, -) -> Result<()> { - if authed.is_admin { - return Ok(()); - } - if !path.is_empty() { - if require_owner_of_path(authed, path).is_ok() { - return Ok(()); - } - if path.starts_with("f/") && path.split('/').count() >= 2 { - let folder = path.split('/').nth(1).unwrap(); - let extra_perms = sqlx::query_scalar!( - "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", - folder, - w_id - ) - .fetch_optional(&db) - .await?; - if let Some(perms) = extra_perms { - let is_folder_writer = - get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); - if is_folder_writer { - return Ok(()); - } - } - } - let extra_perms = sqlx::query_scalar(query) - .bind(path) - .bind(w_id) - .fetch_optional(&db) - .await?; - if let Some(perms) = extra_perms { - let perm = get_perm_in_extra_perms_for_authed(perms, authed); - match perm { - Some(true) => Ok(()), - Some(false) => Err(Error::BadRequest(format!( - "User {} is not a writer of {kind} path {path}", - authed.username - ))), - None => Err(Error::BadRequest(format!( - "User {} has neither read or write permission on {kind} {path}", - authed.username - ))), - } - } else { - Err(Error::BadRequest(format!( - "{path} does not exist yet and user {} is not an owner of the parent folder", - authed.username - ))) - } - } else { - Err(Error::BadRequest(format!( - "Cannot be writer of an empty path" - ))) - } -} -async fn whois( - Extension(db): Extension, - Path((w_id, username)): Path<(String, String)>, -) -> JsonResult { - let user_o = get_user(&w_id, &username, &db).await?; - let user = not_found_if_none(user_o, "User", username)?; - Ok(Json(user)) -} - -// async fn create_invite_code( -// ApiAuthed { email, .. }: ApiAuthed, -// Extension(db): Extension, -// Json(nu): Json, -// ) -> Result<(StatusCode, String)> { - -// let mut tx = db.begin().await?; -// require_super_admin(&mut *tx, email).await?; - -// sqlx::query!( -// "INSERT INTO invite_code -// (code, seats_left) -// VALUES ($1, $2)", -// nu.code, -// nu.seats -// ) -// .execute(&mut *tx) -// .await?; - -// tx.commit().await?; - -// Ok(( -// StatusCode::CREATED, -// format!("new invite code {}", nu.code), -// )) -// } - -async fn decline_invite( - authed: ApiAuthed, - Extension(db): Extension, - Json(nu): Json, -) -> Result<(StatusCode, String)> { - let mut tx = db.begin().await?; - - let is_admin = sqlx::query_scalar!( - "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin", - nu.workspace_id, - authed.email, - ) - .fetch_optional(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.decline_invite", - ActionKind::Delete, - &nu.workspace_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - if is_admin.is_some() { - Ok(( - StatusCode::OK, - format!( - "user {} declined invite to workspace {}", - &authed.email, nu.workspace_id - ), - )) - } else { - Err(Error::NotFound(format!( - "invite for {} not found", - authed.email - ))) - } -} - -lazy_static! { - pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); -} - -async fn accept_invite( - authed: ApiAuthed, - Extension(webhook): Extension, - Extension(db): Extension, - Json(nu): Json, -) -> Result<(StatusCode, String)> { - let mut tx = db.begin().await?; - - let r = sqlx::query!( - "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin, operator", - nu.workspace_id, - authed.email, - ) - .fetch_optional(&mut *tx) - .await?; - - if let Some(r) = r { - let already_in_workspace = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", - &nu.workspace_id, - &authed.email, - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if already_in_workspace { - tx.commit().await?; - return Ok(( - StatusCode::CREATED, - format!( - "user {} accepted invite to workspace {}", - &authed.email, nu.workspace_id - ), - )); - } - let username; - (tx, username) = join_workspace( - &nu.workspace_id, - &authed, - nu.username, - r.is_admin, - r.operator, - tx, - ) - .await?; - - audit_log( - &mut *tx, - &ApiAuthed { username: username.clone(), ..authed.clone() }, - "users.accept_invite", - ActionKind::Create, - &nu.workspace_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &username, - &db, - &nu.workspace_id, - windmill_git_sync::DeployedObject::User { email: authed.email.clone() }, - Some(format!("User '{}' accepted invite", &authed.email)), - true, - None, - ) - .await?; - webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace { - email: authed.email.clone(), - workspace: nu.workspace_id.clone(), - username: username, - }); - Ok(( - StatusCode::CREATED, - format!( - "user {} accepted invite to workspace {}", - &authed.email, nu.workspace_id - ), - )) - } else { - Err(Error::NotFound(format!( - "invite for {} not found", - authed.email - ))) - } -} - -async fn join_workspace<'c>( - w_id: &str, - authed: &ApiAuthed, - username: Option, - is_admin: bool, - operator: bool, - mut tx: sqlx::Transaction<'c, sqlx::Postgres>, -) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, String)> { - let automate_username_creation = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = $1", - AUTOMATE_USERNAME_CREATION_SETTING, - ) - .fetch_optional(&mut *tx) - .await? - .map(|v| v.as_bool()) - .flatten() - .unwrap_or(false); - - let username = if automate_username_creation { - if username.is_some() && username.unwrap().len() > 0 { - return Err(Error::BadRequest( - "username is not allowed when username creation is automated".to_string(), - )); - } - get_instance_username_or_create_pending(&mut tx, &authed.email).await? - } else { - let username = username.ok_or(Error::BadRequest("username is required".to_string()))?; - let already_exists_username = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", - &w_id, - username, - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if already_exists_username { - return Err(Error::BadRequest(format!( - "user with username {} already exists in workspace {}", - username, w_id - ))); - } - - if !VALID_USERNAME.is_match(&username) { - return Err(windmill_common::error::Error::BadRequest(format!( - "Usermame can only contain alphanumeric characters and underscores and must start with a letter" - ))); - } - username.to_string() - }; - - let already_exists_email = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", - &w_id, - authed.email, - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if already_exists_email { - return Err(Error::BadRequest(format!( - "user with email {} already exists in workspace {}", - authed.email, w_id - ))); - } - - sqlx::query!( - "INSERT INTO usr - (workspace_id, email, username, is_admin, operator) - VALUES ($1, $2, $3, $4, $5)", - &w_id, - authed.email, - username, - is_admin, - operator - ) - .execute(&mut *tx) - .await?; - sqlx::query_as!( - Group, - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - &w_id, - username, - "all", - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &AuditAuthor { username: username.clone(), ..authed.into() }, - "users.add_to_workspace", - ActionKind::Create, - &w_id, - Some(&authed.email), - None, - ) - .await?; - Ok((tx, username)) -} - -async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Result { - let mut tx = db.begin().await?; - sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.leave", - ActionKind::Delete, - "global", - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Left instance",)) -} - -async fn get_workspace_user( - ApiAuthed { username, is_admin, .. }: ApiAuthed, - Extension(db): Extension, - Path((w_id, username_to_update)): Path<(String, String)>, -) -> Result> { - require_admin(is_admin, &username)?; - - let user = sqlx::query_as!( - User, - "SELECT * FROM usr WHERE username = $1 AND workspace_id = $2", - &username_to_update, - &w_id - ) - .fetch_optional(&db) - .await?; - - let user = not_found_if_none(user, "User", username_to_update)?; - - Ok(Json(user)) -} - -async fn update_workspace_user( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, username_to_update)): Path<(String, String)>, - Json(eu): Json, -) -> Result { - let mut tx = db.begin().await?; - - require_admin(authed.is_admin, &authed.username)?; - - update_workspace_user_internal( - &w_id, - &username_to_update, - eu.is_admin, - eu.operator, - eu.disabled, - &mut tx, - Some(&authed), - ) - .await?; - - let user_email = sqlx::query_scalar!( - "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", - &username_to_update, - &w_id - ) - .fetch_one(&mut *tx) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::User { email: user_email.clone() }, - Some(format!("Updated user '{}'", &user_email)), - true, - None, - ) - .await?; - - Ok(format!("user {} updated", user_email)) -} - -async fn convert_user_to_group( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, username_to_convert)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - let mut tx = db.begin().await?; - - // Get user email and current status - let user_info = sqlx::query!( - "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2", - username_to_convert, - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - let user_info = not_found_if_none(user_info, "User", &username_to_convert)?; - - // Check if user is already a group user - if let Some(added_via) = &user_info.added_via { - if added_via.get("source").and_then(|v| v.as_str()) == Some("instance_group") { - return Err(Error::BadRequest( - "User is already a group user".to_string(), - )); - } - } - - // Find which instance groups this user belongs to that are configured for auto-add in this workspace - let eligible_groups = sqlx::query!( - r#" - SELECT - eig.igroup as group_name, - ws.auto_invite->'instance_groups_roles' as instance_groups_roles - FROM email_to_igroup eig - INNER JOIN workspace_settings ws ON ws.workspace_id = $1 - WHERE eig.email = $2 - AND ws.auto_invite->'instance_groups' ? eig.igroup - "#, - &w_id, - &user_info.email - ) - .fetch_all(&mut *tx) - .await?; - - if eligible_groups.is_empty() { - return Err(Error::BadRequest( - "User is not a member of any instance groups configured for auto-add in this workspace" - .to_string(), - )); - } - - // Determine the group with highest precedence (same logic as process_instance_group_auto_adds) - let roles: std::collections::HashMap = - if let Some(roles_json) = &eligible_groups[0].instance_groups_roles { - serde_json::from_value(roles_json.clone()).unwrap_or_default() - } else { - std::collections::HashMap::new() - }; - - let mut best_group = &eligible_groups[0].group_name; - let mut best_precedence = 0u8; - - for group in &eligible_groups { - let default_role = "developer".to_string(); - let role = roles.get(&group.group_name).unwrap_or(&default_role); - - let precedence = match role.as_str() { - "admin" => 3, - "developer" => 2, - "operator" => 1, - _ => 2, - }; - - if precedence > best_precedence { - best_precedence = precedence; - best_group = &group.group_name; - } - } - - let primary_group_name = best_group; - - // Determine role from group configuration using the selected primary group - let default_role = "developer".to_string(); - let role = roles - .get(primary_group_name) - .unwrap_or(&default_role) - .as_str(); - - let (is_admin, is_operator) = match role { - "admin" => (true, false), - "operator" => (false, true), - _ => (false, false), - }; - - // Update user with instance group information - let instance_group_source = serde_json::json!({ - "source": "instance_group", - "group": primary_group_name - }); - - sqlx::query!( - "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5", - instance_group_source, - is_admin, - is_operator, - username_to_convert, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.convert_to_group", - ActionKind::Update, - &w_id, - Some(&username_to_convert), - Some([("group", primary_group_name.as_str()), ("role", role)].into()), - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::User { email: user_info.email.clone() }, - Some(format!( - "Converted user '{}' to group user (group: {}, role: {})", - &user_info.email, primary_group_name, role - )), - true, - None, - ) - .await?; - - Ok(format!( - "User {} converted to group user (group: {}, role: {})", - username_to_convert, primary_group_name, role - )) -} - -async fn update_user( - authed: ApiAuthed, - Path(email_to_update): Path, - Extension(db): Extension, - Json(eu): Json, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - - let mut new_super_admin: Option = None; - if let Some(sa) = eu.is_super_admin { - sqlx::query_scalar!( - "UPDATE password SET super_admin = $1 WHERE email = $2", - sa, - &email_to_update - ) - .execute(&mut *tx) - .await?; - new_super_admin = Some(sa); - } - - if let Some(dv) = eu.is_devops { - sqlx::query_scalar!( - "UPDATE password SET devops = $1 WHERE email = $2", - dv, - &email_to_update - ) - .execute(&mut *tx) - .await?; - // If super_admin wasn't explicitly set, we still need to refresh tokens - if new_super_admin.is_none() { - new_super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - &email_to_update - ) - .fetch_optional(&mut *tx) - .await?; - } - } - - if let Some(sa) = new_super_admin { - // Delete session tokens to force re-login with new privileges - sqlx::query!( - "DELETE FROM token WHERE email = $1 AND label = 'session'", - &email_to_update - ) - .execute(&mut *tx) - .await?; - // Update super_admin flag on non-session tokens (webhooks, API tokens, etc.) - sqlx::query!( - "UPDATE token SET super_admin = $1 WHERE email = $2 AND label != 'session'", - sa, - &email_to_update - ) - .execute(&mut *tx) - .await?; - } - - if let Some(n) = eu.name { - sqlx::query_scalar!( - "UPDATE password SET name = $1 WHERE email = $2", - n, - &email_to_update - ) - .execute(&mut *tx) - .await?; - } - - audit_log( - &mut *tx, - &authed, - "users.update", - ActionKind::Update, - "global", - Some(&email_to_update), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("email {} updated", &email_to_update)) -} - -async fn delete_user( - authed: ApiAuthed, - Path(email_to_delete): Path, - Extension(db): Extension, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - - sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) - .execute(&mut *tx) - .await?; - - let usernames = sqlx::query_scalar!( - "DELETE FROM usr WHERE email = $1 RETURNING username", - &email_to_delete - ) - .fetch_all(&mut *tx) - .await?; - - for username in usernames { - sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM usr_to_group WHERE usr = $1", &username) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM workspace_invite WHERE email = $1", - &email_to_delete - ) - .execute(&mut *tx) - .await?; - } - - // Remove user from all instance groups email_to_igroup - sqlx::query!( - "DELETE FROM email_to_igroup WHERE email = $1", - &email_to_delete - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.delete", - ActionKind::Delete, - "global", - Some(&email_to_delete), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("email {} deleted", &email_to_delete)) -} - -lazy_static::lazy_static! { - pub static ref NEW_USER_WEBHOOK: Option = std::env::var("NEW_USER_WEBHOOK").ok(); - } async fn create_user( authed: ApiAuthed, Extension(db): Extension, - Extension(webhook): Extension, + Extension(webhook): Extension, Extension(argon2): Extension>>, Json(nu): Json, ) -> Result<(StatusCode, String)> { @@ -1761,152 +65,6 @@ async fn submit_onboarding_data( crate::users_oss::submit_onboarding_data(authed, Extension(db), Json(data)).await } -/// Internal helper for updating workspace user permissions - used by both API and system operations -pub async fn update_workspace_user_internal( - w_id: &str, - username_to_update: &str, - is_admin: Option, - operator: Option, - disabled: Option, - tx: &mut Transaction<'_, Postgres>, - authed: Option<&ApiAuthed>, // None for system operations -) -> Result<()> { - if let Some(a) = is_admin { - sqlx::query_scalar!( - "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3", - a, - username_to_update, - w_id - ) - .execute(&mut **tx) - .await?; - } - if let Some(a) = operator { - sqlx::query_scalar!( - "UPDATE usr SET operator = $1 WHERE username = $2 AND workspace_id = $3", - a, - username_to_update, - w_id - ) - .execute(&mut **tx) - .await?; - } - if let Some(a) = disabled { - sqlx::query_scalar!( - "UPDATE usr SET disabled = $1 WHERE username = $2 AND workspace_id = $3", - a, - username_to_update, - w_id - ) - .execute(&mut **tx) - .await?; - } - - // Only audit if we have an authenticated user (API calls) - if let Some(auth) = authed { - audit_log( - &mut **tx, - auth, - "users.update", - ActionKind::Update, - w_id, - Some(username_to_update), - None, - ) - .await?; - } - - Ok(()) -} - -/// Internal helper for deleting workspace users - used by both API and system operations -pub async fn delete_workspace_user_internal( - w_id: &str, - username_to_delete: &str, - email_to_delete: &str, - tx: &mut Transaction<'_, Postgres>, - authed: Option<&ApiAuthed>, // None for system operations -) -> Result<()> { - sqlx::query_scalar!( - "DELETE FROM usr WHERE email = $1 AND workspace_id = $2", - email_to_delete, - w_id - ) - .execute(&mut **tx) - .await?; - - sqlx::query!( - "DELETE FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", - username_to_delete, - w_id - ) - .execute(&mut **tx) - .await?; - - // Only audit if we have an authenticated user (API calls) - if let Some(auth) = authed { - audit_log( - &mut **tx, - auth, - "users.delete", - ActionKind::Delete, - w_id, - Some(username_to_delete), - None, - ) - .await?; - } - - Ok(()) -} - -async fn delete_workspace_user( - authed: ApiAuthed, - Extension(db): Extension, - Path((w_id, username_to_delete)): Path<(String, String)>, -) -> Result { - let mut tx = db.begin().await?; - - require_admin(authed.is_admin, &authed.username)?; - - let email_to_delete_o = sqlx::query_scalar!( - "SELECT email FROM usr where username = $1 AND workspace_id = $2", - username_to_delete, - &w_id, - ) - .fetch_optional(&db) - .await?; - - let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; - - delete_workspace_user_internal( - &w_id, - &username_to_delete, - &email_to_delete, - &mut tx, - Some(&authed), - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::User { email: email_to_delete.clone() }, - Some(format!( - "Removed user '{}' from workspace", - &email_to_delete - )), - true, - None, - ) - .await?; - - Ok(format!("username {} deleted", username_to_delete)) -} - async fn set_password( Extension(db): Extension, Extension(argon2): Extension>>, @@ -1928,737 +86,6 @@ async fn set_password_of_user( crate::users_oss::set_password(db, argon2, authed, &email, ep).await } -async fn set_login_type( - Extension(db): Extension, - Path(email): Path, - authed: ApiAuthed, - Json(et): Json, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - - sqlx::query!( - "UPDATE password SET login_type = $1 WHERE email = $2", - et.login_type, - email - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.set_login_type", - ActionKind::Update, - "global", - Some(&email), - None, - ) - .await?; - - tx.commit().await?; - Ok(format!( - "login type of {} updated to {}", - email, et.login_type - )) -} - -async fn login( - cookies: Cookies, - Extension(db): Extension, - Extension(argon2): Extension>>, - Json(Login { email, password }): Json, -) -> Result { - let mut tx = db.begin().await?; - let email = email.to_lowercase(); - let audit_author = AuditAuthor { - email: email.clone(), - username: email.clone(), - username_override: None, - token_prefix: None, - }; - let email_w_h: Option<(String, String, bool)> = sqlx::query_as( - "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \ - 'password'", - ) - .bind(&email) - .fetch_optional(&mut *tx) - .await?; - - if let Some((email, hash, super_admin)) = email_w_h { - let parsed_hash = - PasswordHash::new(&hash).map_err(|e| Error::internal_err(e.to_string()))?; - if argon2 - .verify_password(password.as_bytes(), &parsed_hash) - .is_err() - { - audit_log( - &mut *tx, - &audit_author, - "users.login_failure", - ActionKind::Create, - "global", - None, - None, - ) - .await?; - Err(Error::BadRequest("Invalid login".to_string())) - } else { - let token = create_session_token(&email, super_admin, &mut tx, cookies).await?; - - let audit_author = AuditAuthor { - email: email.clone(), - username: email.clone(), - username_override: None, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }; - - audit_log( - &mut *tx, - &audit_author, - "users.login", - ActionKind::Create, - "global", - Some(&truncate_token(&token)), - None, - ) - .await?; - - tx.commit().await?; - Ok(token) - } - } else { - audit_log( - &mut *tx, - &audit_author, - "users.login_failure", - ActionKind::Create, - "global", - None, - None, - ) - .await?; - Err(Error::BadRequest("Invalid login".to_string())) - } -} - -#[derive(Deserialize)] -struct RefreshTokenQuery { - if_expiring_in_less_than_s: Option, -} -async fn refresh_token( - Extension(db): Extension, - Query(query): Query, - Tokened { token }: Tokened, - authed: ApiAuthed, - cookies: Cookies, -) -> Result { - let mut tx = db.begin().await?; - - if let Some(thresh_s) = query.if_expiring_in_less_than_s { - let not_expired = sqlx::query_scalar!("SELECT true FROM token WHERE token = $1 and expiration IS NOT NULL and expiration > now() + $2::int * '1 sec'::interval", &token, thresh_s) - .fetch_optional(&db) - .await? - .flatten() - .unwrap_or(false); - if not_expired { - return Ok("token expiry is far enough".to_string()); - } - } - - let super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - &authed.email - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(false); - - let new_token = create_session_token(&authed.email, super_admin, &mut tx, cookies).await?; - - audit_log( - &mut *tx, - &AuditAuthor { - email: authed.email.to_string(), - username: authed.email.to_string(), - username_override: None, - token_prefix: authed.token_prefix, - }, - "users.token.refresh", - ActionKind::Create, - &"global", - Some(&truncate_token(&new_token)), - None, - ) - .await?; - - tx.commit().await?; - Ok("token refreshed".to_string()) -} - -lazy_static::lazy_static! { - static ref MAX_SESSION_VALIDITY_SECONDS: i64 = std::env::var("MAX_SESSION_VALIDITY_SECONDS").ok().unwrap_or_else(|| String::new()).parse::().unwrap_or(3 * 24 * 60 * 60); - static ref INVALIDATE_OLD_SESSIONS: bool = std::env::var("INVALIDATE_OLD_SESSIONS").ok().unwrap_or_else(|| String::new()).parse::().unwrap_or(false); -} - -pub async fn create_session_token<'c>( - email: &str, - super_admin: bool, - tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, - cookies: Cookies, -) -> Result { - let token = rd_string(32); - - if *INVALIDATE_OLD_SESSIONS { - sqlx::query!( - "DELETE FROM token WHERE email = $1 AND label = 'session'", - email - ) - .execute(&mut **tx) - .await?; - - audit_log( - &mut **tx, - &AuditAuthor { - email: email.to_string(), - username: email.to_string(), - username_override: None, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }, - "users.token.invalidate_old_sessions", - ActionKind::Delete, - &"global", - None, - None, - ) - .instrument(tracing::info_span!("token", email)) - .await?; - } - - sqlx::query!( - "INSERT INTO token - (token, email, label, expiration, super_admin) - VALUES ($1, $2, $3, now() + ($4 || ' seconds')::interval, $5)", - token, - email, - "session", - &MAX_SESSION_VALIDITY_SECONDS.to_string(), - super_admin - ) - .execute(&mut **tx) - .await?; - - let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); - cookie.set_secure(IS_SECURE.read().await.clone()); - cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); - cookie.set_http_only(true); - cookie.set_path(COOKIE_PATH); - if COOKIE_DOMAIN.is_some() { - cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); - } - - let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); - expire += time::Duration::seconds(*MAX_SESSION_VALIDITY_SECONDS); - cookie.set_expires(expire); - cookies.add(cookie); - Ok(token) -} - -pub async fn create_token_internal( - tx: &mut PgConnection, - db: &DB, - authed: &ApiAuthed, - token_config: NewToken, -) -> Result { - let token = rd_string(32); - - let is_super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - authed.email - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(false); - if *CLOUD_HOSTED { - let nb_tokens = - sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email) - .fetch_one(db) - .await?; - if nb_tokens.unwrap_or(0) >= 10000 { - return Err(Error::BadRequest( - "You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - sqlx::query!( - "INSERT INTO token - (token, email, label, expiration, super_admin, scopes, workspace_id) - VALUES ($1, $2, $3, $4, $5, $6, $7)", - token, - authed.email, - token_config.label, - token_config.expiration, - is_super_admin, - token_config.scopes.as_ref().map(|x| x.as_slice()), - token_config.workspace_id, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - authed, - "users.token.create", - ActionKind::Create, - &"global", - Some(&token[0..10]), - None, - ) - .instrument(tracing::info_span!("token", email = &authed.email)) - .await?; - - Ok(token) -} - -async fn create_token( - Extension(db): Extension, - authed: ApiAuthed, - Json(token_config): Json, -) -> Result<(StatusCode, String)> { - let mut tx = db.begin().await?; - - let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?; - - tx.commit().await?; - Ok((StatusCode::CREATED, token)) -} - -async fn impersonate( - Extension(db): Extension, - authed: ApiAuthed, - Json(new_token): Json, -) -> Result<(StatusCode, String)> { - let token = rd_string(32); - require_super_admin(&db, &authed.email).await?; - - if new_token.impersonate_email.is_none() { - return Err(Error::BadRequest( - "impersonate_username is required".to_string(), - )); - } - - let impersonated = new_token.impersonate_email.unwrap(); - - let is_super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - impersonated - ) - .fetch_optional(&db) - .await? - .unwrap_or(false); - let mut tx = db.begin().await?; - - sqlx::query!( - "INSERT INTO token - (token, email, label, expiration, super_admin) - VALUES ($1, $2, $3, $4, $5)", - token, - impersonated, - new_token.label, - new_token.expiration, - is_super_admin - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.impersonate", - ActionKind::Delete, - &"global", - Some(&token[0..10]), - Some([("impersonated", &format!("{impersonated}")[..])].into()), - ) - .instrument(tracing::info_span!("token", email = &impersonated)) - .await?; - tx.commit().await?; - Ok((StatusCode::CREATED, token)) -} - -#[derive(Deserialize)] -struct ListTokenQuery { - exclude_ephemeral: Option, -} - -async fn list_tokens( - Extension(db): Extension, - ApiAuthed { email, .. }: ApiAuthed, - Query(query): Query, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - let rows = if query.exclude_ephemeral.unwrap_or(false) { - sqlx::query_as!( - TruncatedToken, - "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ - last_used_at, scopes FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL) - ORDER BY created_at DESC LIMIT $2 OFFSET $3", - email, - per_page as i64, - offset as i64, - ) - .fetch_all(&db) - .await? - } else { - sqlx::query_as!( - TruncatedToken, - "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ - last_used_at, scopes FROM token WHERE email = $1 - ORDER BY created_at DESC LIMIT $2 OFFSET $3", - email, - per_page as i64, - offset as i64, - ) - .fetch_all(&db) - .await? - }; - Ok(Json(rows)) -} - -async fn delete_token( - Extension(db): Extension, - authed: ApiAuthed, - Path(token_prefix): Path, -) -> Result { - let mut tx = db.begin().await?; - - let tokens_deleted: Vec = sqlx::query_scalar( - "DELETE FROM token - WHERE email = $1 - AND token LIKE concat($2::text, '%') - RETURNING concat(substring(token for 10), '*****')", - ) - .bind(&authed.email) - .bind(&token_prefix) - .fetch_all(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.token.delete", - ActionKind::Delete, - &"global", - Some(&token_prefix), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!( - "deleted {} tokens {:?} with prefix {}", - tokens_deleted.len(), - tokens_deleted, - token_prefix - )) -} - -async fn leave_workspace( - Extension(db): Extension, - Path(w_id): Path, - authed: ApiAuthed, -) -> Result { - let mut tx = db.begin().await?; - sqlx::query!( - "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", - &w_id, - authed.username - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.leave_workspace", - ActionKind::Delete, - &w_id, - None, - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("left workspace {w_id}")) -} - -#[derive(Serialize)] -struct Runnable { - workspace: String, - endpoint_async: String, - endpoint_sync: String, - summary: String, - description: String, - schema: Option, - kind: String, - path: String, -} - -async fn get_all_runnables( - Extension(db): Extension, - authed: ApiAuthed, - Tokened { token }: Tokened, - Extension(cache): Extension>, -) -> JsonResult> { - let mut tx = db.clone().begin(&authed).await?; - let mut runnables = Vec::new(); - let workspaces = sqlx::query_scalar!( - "SELECT workspace.id as id FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \ - usr.email = $1 AND deleted = false", - authed.email - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - for workspace in workspaces { - let nauthed = cache - .get_authed(Some(workspace.clone()), &token) - .await - .ok_or_else(|| { - Error::BadRequest(format!("not authorized to access workspace: {workspace}")) - })?; - let mut tx = db.clone().begin(&nauthed).await?; - let flows = sqlx::query!( - "SELECT flow.workspace_id as workspace, flow.path, summary, description, flow_version.schema - FROM flow - LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.workspace_id = $1", - workspace - ) - .fetch_all(&mut *tx) - .await?; - runnables.extend( - flows - .into_iter() - .map(|f| Runnable { - workspace: f.workspace.clone(), - endpoint_async: format!("/w/{}/jobs/run/f/{}", &f.workspace, &f.path), - endpoint_sync: format!( - "/w/{}/jobs/run_wait_result/f/{}", - &f.workspace, &f.path - ), - summary: f.summary, - description: f.description, - schema: f.schema, - kind: "flow".to_string(), - path: f.path, - }) - .collect::>(), - ); - let scripts = sqlx::query!( - "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o - WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) - AND workspace_id = $1 and archived = false", workspace - ) - .fetch_all(&mut *tx) - .await?; - runnables.extend( - scripts - .into_iter() - .map(|s| Runnable { - workspace: s.workspace.clone(), - endpoint_async: format!("/w/{}/jobs/run/p/{}", &s.workspace, &s.path), - endpoint_sync: format!( - "/w/{}/jobs/run_wait_result/p/{}", - &s.workspace, &s.path - ), - summary: s.summary, - description: s.description, - schema: s.schema, - kind: "script".to_string(), - path: s.path, - }) - .collect::>(), - ); - tx.commit().await?; - } - Ok(Json(runnables)) -} - -//used by oauth -#[allow(dead_code)] -#[derive(Deserialize, Debug, Clone)] -pub struct LoginUserInfo { - pub email: Option, - pub name: Option, - pub company: Option, - pub preferred_username: Option, - pub displayName: Option, -} - -#[derive(Serialize)] -struct InstanceUsernameInfo { - username: String, - workspace_usernames: Vec, -} - -#[derive(Serialize)] -struct WorkspaceUsernameInfo { - workspace_id: String, - username: String, -} -async fn get_instance_username_info( - ApiAuthed { email, .. }: ApiAuthed, - Path(user_email): Path, - Extension(db): Extension, -) -> JsonResult { - require_super_admin(&db, &email).await?; - let mut tx = db.begin().await?; - let instance_username = match sqlx::query_scalar!( - "SELECT username FROM password WHERE email = $1", - &user_email - ) - .fetch_one(&mut *tx) - .await? - { - Some(username) => username, - None => generate_instance_wide_unique_username(&mut tx, &user_email).await?, - }; - - let workspace_usernames = sqlx::query_as!( - WorkspaceUsernameInfo, - "SELECT workspace_id, username FROM usr WHERE email = $1", - &user_email - ) - .fetch_all(&mut *tx) - .await?; - - Ok(Json(InstanceUsernameInfo { - username: instance_username, - workspace_usernames: workspace_usernames, - })) -} - -async fn username_to_email( - Path((w_id, username)): Path<(String, String)>, - Extension(db): Extension, -) -> Result { - let email = sqlx::query_scalar!( - "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", - &username, - &w_id - ) - .fetch_optional(&db) - .await?; - - let email = not_found_if_none(email, "user", username)?; - - Ok(email) -} - -#[cfg(feature = "enterprise")] -#[derive(Serialize, Deserialize)] -struct ExportedGlobalUser { - email: String, - password_hash: Option, - login_type: String, - super_admin: bool, - verified: bool, - name: Option, - company: Option, - first_time_user: bool, - username: Option, -} - -#[cfg(feature = "enterprise")] -async fn export_global_users( - Extension(db): Extension, - authed: ApiAuthed, -) -> JsonResult> { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - let users = sqlx::query_as!( - ExportedGlobalUser, - "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password" - ) - .fetch_all(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.export_export", - ActionKind::Execute, - "global", - None, - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(users)) -} - -#[cfg(not(feature = "enterprise"))] -async fn export_global_users() -> JsonResult { - Err(Error::BadRequest( - "This feature is only available in the enterprise version".to_string(), - )) -} - -#[cfg(feature = "enterprise")] -async fn overwrite_global_users( - Extension(db): Extension, - authed: ApiAuthed, - Json(users): Json>, -) -> Result { - require_super_admin(&db, &authed.email).await?; - let mut tx = db.begin().await?; - sqlx::query!("DELETE FROM password") - .execute(&mut *tx) - .await?; - for user in users { - sqlx::query!( - "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - user.email, - user.password_hash, - user.login_type, - user.super_admin, - user.verified, - user.name, - user.company, - user.first_time_user, - user.username - ) - .execute(&mut *tx) - .await?; - } - audit_log( - &mut *tx, - &authed, - "users.import_global", - ActionKind::Create, - "global", - None, - None, - ) - .await?; - tx.commit().await?; - Ok("loaded global users".to_string()) -} - -#[cfg(not(feature = "enterprise"))] -async fn overwrite_global_users() -> JsonResult { - Err(Error::BadRequest( - "This feature is only available in the enterprise version".to_string(), - )) -} - #[derive(Deserialize)] struct RenameUser { new_username: String, @@ -3234,115 +661,6 @@ async fn update_username_in_workpsace<'c>( Ok(()) } -// Password Reset Types -#[derive(Deserialize)] -pub struct RequestPasswordReset { - pub email: String, -} - -#[derive(Deserialize)] -pub struct ResetPassword { - pub token: String, - pub new_password: String, -} - -#[derive(Serialize)] -pub struct PasswordResetResponse { - pub message: String, -} - -// Password Reset Functions - -/// Check if SMTP is configured -async fn is_smtp_configured(Extension(db): Extension) -> JsonResult { - let smtp = windmill_common::server::load_smtp_config(&db).await?; - Ok(Json(smtp.is_some())) -} - -/// Request a password reset email -async fn request_password_reset( - Extension(db): Extension, - Json(req): Json, -) -> Result> { - let email = req.email.to_lowercase(); - - // Check if SMTP is configured - let smtp = windmill_common::server::load_smtp_config(&db).await?; - let smtp = smtp.ok_or_else(|| { - Error::BadRequest("SMTP is not configured. Password reset is not available.".to_string()) - })?; - - // Check if user exists with password login type - let user_exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1 AND login_type = 'password')", - &email - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - // Always return success to prevent email enumeration - // But only send email if user exists - if user_exists { - // Generate a secure token - let token = rd_string(32); - - // Delete any existing tokens for this email - sqlx::query!("DELETE FROM magic_link WHERE email = $1", &email) - .execute(&db) - .await?; - - // Insert new token with 1 hour expiration - sqlx::query!( - "INSERT INTO magic_link (email, token, expiration) VALUES ($1, $2, NOW() + INTERVAL '1 hour')", - &email, - &token - ) - .execute(&db) - .await?; - - // Get the base URL for the reset link - let base_url = BASE_URL.read().await.clone(); - let base_url = if base_url.is_empty() { - std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string()) - } else { - base_url - }; - - let reset_link = format!("{}/user/reset-password?token={}", base_url, token); - - // Send the email - let subject = "Windmill Password Reset"; - let content = format!( - "You have requested a password reset for your Windmill account.\n\n\ - Click the link below to reset your password:\n\ - {}\n\n\ - This link will expire in 1 hour.\n\n\ - If you did not request this password reset, you can safely ignore this email.", - reset_link - ); - - // Send the email - don't fail the request if email fails - if let Err(e) = windmill_common::email_oss::send_email_plain_text( - subject, - &content, - vec![email.clone()], - smtp, - Some(Duration::from_secs(10)), - ) - .await - { - tracing::error!("Failed to send password reset email to {}: {:?}", email, e); - } - } - - // Always return success to prevent email enumeration - Ok(Json(PasswordResetResponse { - message: "If an account with that email exists, a password reset link has been sent." - .to_string(), - })) -} - /// Reset password using a token async fn reset_password( Extension(db): Extension, diff --git a/backend/windmill-api/src/users_oss.rs b/backend/windmill-api/src/users_oss.rs index 7cb643b84a..c2ac850874 100644 --- a/backend/windmill-api/src/users_oss.rs +++ b/backend/windmill-api/src/users_oss.rs @@ -62,13 +62,6 @@ pub fn hash_password(_argon2: Arc>, _password: String) -> Result, -} - -// Shared structs for bulk delete operations -#[derive(Deserialize)] -pub struct BulkDeleteRequest { - pub paths: Vec, -} - -pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { - let is_admin = is_super_admin_email(db, email).await?; - - if !is_admin { - Err(Error::NotAuthorized( - "This endpoint requires the caller to be a super admin".to_owned(), - )) - } else { - Ok(()) - } -} - -pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> -where - F: FnOnce() -> String, -{ - if let Some(scopes) = authed.scopes.as_ref() { - let mut is_scoped_token = false; - let required_scope = ScopeDefinition::from_scope_string(&required())?; - for scope in scopes { - if !scope.starts_with("if_jobs:filter_tags:") { - if !is_scoped_token { - is_scoped_token = true; - } - - match ScopeDefinition::from_scope_string(scope) { - Ok(scope) if scope.includes(&required_scope) => return Ok(()), - _ => {} - } - } - } - - if is_scoped_token { - return Err(Error::NotAuthorized(format!( - "Required scope: {}", - required_scope.as_string() - ))); - } - } - Ok(()) -} - -pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { - let is_devops = is_devops_email(db, email).await?; - - if is_devops { - Ok(()) - } else { - Err(Error::NotAuthorized( - "This endpoint requires the caller to have the `devops` role".to_string(), - )) - } -} - -lazy_static::lazy_static! { - pub static ref INVALID_USERNAME_CHARS: Regex = Regex::new(r"[^A-Za-z0-9_]").unwrap(); -} - -pub async fn generate_instance_wide_unique_username<'c>( - tx: &mut Transaction<'c, Postgres>, - email: &str, -) -> error::Result { - let mut username = email.split('@').next().unwrap().to_string(); - - username = INVALID_USERNAME_CHARS - .replace_all(&mut username, "") - .to_string(); - - if username.is_empty() { - username = "user".to_string() - } - - let base_username = username.clone(); - let mut username_conflict = true; - let mut i = 1; - while username_conflict { - if i > 1000 { - return Err(Error::internal_err(format!( - "too many username conflicts for {}", - email - ))); - } - if i > 1 { - username = format!("{}{}", base_username, i) - } - username_conflict = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", - &username, - &email - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - i += 1; - } - - Ok(username) -} +pub use windmill_common::usernames::generate_instance_wide_unique_username; +pub use windmill_common::utils::WithStarredInfoQuery; pub async fn generate_instance_username_for_all_users(db: &DB) -> error::Result<()> { let mut tx = db.begin().await?; @@ -174,45 +64,6 @@ pub async fn generate_instance_username_for_all_users(db: &DB) -> error::Result< Ok(()) } -pub async fn get_instance_username_or_create_pending<'c>( - tx: &mut Transaction<'c, Postgres>, - email: &str, -) -> error::Result { - let user = sqlx::query_scalar!("SELECT username FROM password WHERE email = $1", email) - .fetch_optional(&mut **tx) - .await?; - - if let Some(opt_username) = user { - if let Some(username) = opt_username { - Ok(username) - } else { - Err(Error::BadRequest(format!("No instance-wide username found for {email}. The user has different usernames for different workspaces. Ask the instance administrator to solve the conflict in the instance settings."))) - } - } else { - let pending_username = - sqlx::query_scalar!("SELECT username FROM pending_user WHERE email = $1", email) - .fetch_optional(&mut **tx) - .await?; - - if let Some(username) = pending_username { - Ok(username) - } else { - let username = generate_instance_wide_unique_username(&mut *tx, email).await?; - - sqlx::query!( - "INSERT INTO pending_user (email, username) VALUES ($1, $2)", - email, - username - ) - .execute(&mut **tx) - .await - .map_err(|e| Error::internal_err(format!("creating pending user: {e:#}")))?; - - Ok(username) - } - } -} - pub fn content_plain(body: Body) -> Response { use axum::http::header; Response::builder() diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 6fd97247d4..10e03b2146 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -1,920 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::{ - db::{ApiAuthed, DB}, - secret_backend_ext::{ - delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, - store_secret_value, - }, - users::{maybe_refresh_folders, require_owner_of_path}, - utils::{check_scopes, BulkDeleteRequest}, - webhook_util::{WebhookMessage, WebhookShared}, -}; - -use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, -}; -use futures::future::try_join_all; -use hyper::StatusCode; -use serde_json::Value; - -use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; -use windmill_audit::ActionKind; -use windmill_common::{ - db::{DbWithOptAuthed, UserDB}, - error::{Error, JsonResult, Result}, - scripts::ScriptHash, - utils::{not_found_if_none, paginate, Pagination, StripPath, WarnAfterExt}, - variables::{ - build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable, - }, - worker::CLOUD_HOSTED, -}; - -use crate::var_resource_cache::{cache_variable, get_cached_variable}; -use lazy_static::lazy_static; -use serde::Deserialize; -use sqlx::{Acquire, Postgres, Transaction}; -use windmill_common::variables::encrypt; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; - -lazy_static! { - pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); -} - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_variables)) - .route("/list_contextual", get(list_contextual_variables)) - .route("/get/*path", get(get_variable)) - .route("/get_value/*path", get(get_value)) - .route("/exists/*path", get(exists_variable)) - .route("/update/*path", post(update_variable)) - .route("/delete/*path", delete(delete_variable)) - .route("/delete_bulk", delete(delete_variables_bulk)) - .route("/create", post(create_variable)) - .route("/encrypt", post(encrypt_value)) -} - -async fn list_contextual_variables( - Path(w_id): Path, - ApiAuthed { username, email, .. }: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - Ok(Json( - get_reserved_variables( - &db.into(), - &w_id, - "q1A0qcPuO00yxioll7iph76N9CJDqn", - &email, - &username, - "017e0ad5-f499-73b6-5488-92a61c5196dd", - format!("u/{username}").as_str(), - Some("u/user/script_path".to_string()), - Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), - Some("u/user/encapsulating_flow_path".to_string()), - Some("u/user/triggering_flow_path".to_string()), - Some("c".to_string()), - Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), - Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), - Some(chrono::offset::Utc::now()), - Some(ScriptHash(1234567890)), - None, - ) - .await - .to_vec(), - )) -} - -#[derive(Deserialize)] -struct ListVariableQuery { - path_start: Option, -} - -async fn list_variables( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(lq): Query, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - - let mut tx = user_db.begin(&authed).await?; - - let rows = sqlx::query_as::<_, ListableVariable>( - "SELECT variable.workspace_id, variable.path, CASE WHEN is_secret IS TRUE THEN null ELSE variable.value::text END as value, - is_secret, variable.description, variable.extra_perms, account, is_oauth, (now() > account.expires_at) as is_expired, - account.refresh_error, - resource.path IS NOT NULL as is_linked, - account.refresh_token != '' as is_refreshed, - variable.expires_at - from variable - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $1 - LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $1 - WHERE variable.workspace_id = $1 AND variable.path NOT LIKE 'u/' || $2 || '/secret_arg/%' - AND variable.path LIKE $3 || '%' - ORDER BY path - LIMIT $4 OFFSET $5 -", - ) - .bind(&w_id) - .bind(&authed.username) - .bind(&lq.path_start.unwrap_or_default()) - .bind(per_page as i32) - .bind(offset as i32) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - Ok(Json(rows)) -} - -#[derive(Deserialize)] -struct GetVariableQuery { - decrypt_secret: Option, - include_encrypted: Option, -} - -async fn get_variable( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Query(q): Query, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("variables:read:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - - let variable_o = sqlx::query_as::<_, ListableVariable>( - "SELECT variable.*, (now() > account.expires_at) as is_expired, account.refresh_error, - resource.path IS NOT NULL as is_linked, - account.refresh_token != '' as is_refreshed - from variable - LEFT JOIN account ON variable.account = account.id - LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $2 - WHERE variable.path = $1 AND variable.workspace_id = $2 - LIMIT 1", - ) - .bind(&path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - let variable = if let Some(variable) = variable_o { - variable - } else { - explain_variable_perm_error(&path, &w_id, &db).await?; - unreachable!() - }; - - let decrypt_secret = q.decrypt_secret.unwrap_or(true); - - let r = if variable.is_secret { - if decrypt_secret { - audit_log( - &mut *tx, - &authed, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - } - - let value = variable.value.unwrap_or_else(|| "".to_string()); - ListableVariable { - value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() { - #[cfg(feature = "oauth2")] - { - Some( - crate::oauth2_oss::_refresh_token( - tx, - &variable.path, - &w_id, - variable.account.unwrap(), - &db, - ) - .await?, - ) - } - #[cfg(not(feature = "oauth2"))] - return Err(Error::internal_err("Require oauth2 feature".to_string())); - } else if !value.is_empty() && decrypt_secret { - let _ = tx.commit().await; - // Use secret backend for decryption (supports both DB and Vault) - Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) - } else if q.include_encrypted.unwrap_or(false) { - Some(value) - } else { - None - }, - ..variable - } - } else { - variable - }; - - Ok(Json(r)) -} - -#[derive(Deserialize)] -struct GetValueQuery { - allow_cache: Option, -} -async fn get_value( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(q): Query, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("variables:read:{}", path))?; - let userdb_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - - return get_value_internal(&userdb_authed, &w_id, &path, q.allow_cache.unwrap_or(false)) - .warn_after_seconds(10) - .await - .map(Json); -} - -async fn explain_variable_perm_error( - path: &str, - w_id: &str, - db: &sqlx::Pool, -) -> windmill_common::error::Result<()> { - let extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Variable {} not found", path)))?; - if path.starts_with("f/") { - let folder = path.split("/").nth(1).ok_or_else(|| { - Error::BadRequest(format!( - "path {} should have at least 2 components separated by /", - path - )) - })?; - let folder_extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", - folder, - w_id - ) - .fetch_optional(db) - .await?; - return Err(Error::NotAuthorized(format!( - "Variable exists but you don't have access to it:\nvariable perms: {}\nfolder perms: {}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() - ))); - } else { - return Err(Error::NotAuthorized(format!( - "Variable exists but you don't have access to it:\nvariable perms: {}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default() - ))); - } -} - -async fn exists_variable( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "Variable {} already exists", - path - ))); - } - return Ok(()); -} - -async fn create_variable( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Query(AlreadyEncrypted { already_encrypted }): Query, - Json(variable): Json, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || format!("variables:write:{}", variable.path))?; - if *CLOUD_HOSTED { - let nb_variables = sqlx::query_scalar!( - "SELECT COUNT(*) FROM variable WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - if nb_variables.unwrap_or(0) >= 10000 { - return Err(Error::BadRequest( - "You have reached the maximum number of variables (10000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await; - - check_path_conflict(&db, &w_id, &variable.path).await?; - let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { - // Use secret backend for encryption (supports both DB and Vault) - store_secret_value(&db, &w_id, &variable.path, &variable.value).await? - } else { - variable.value - }; - - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "INSERT INTO variable - (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - &w_id, - variable.path, - value, - variable.is_secret, - variable.description, - variable.account, - variable.is_oauth.unwrap_or(false), - variable.expires_at - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "variables.create", - ActionKind::Create, - &w_id, - Some(&variable.path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { path: variable.path.clone(), parent_path: None }, - Some(format!("Variable '{}' created", variable.path.clone())), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateVariable { workspace: w_id, path: variable.path.clone() }, - ); - - Ok(( - StatusCode::CREATED, - format!("variable {} created", variable.path), - )) -} - -async fn encrypt_value( - Extension(db): Extension, - Path(w_id): Path, - Json(variable): Json, -) -> Result { - let mc = build_crypt(&db, &w_id).await?; - let value = encrypt(&mc, &variable); - - Ok(value) -} - -async fn delete_variable( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - - check_scopes(&authed, || format!("variables:write:{}", path))?; - - // Check if variable is a secret before deleting (for Vault cleanup) - let is_secret = sqlx::query_scalar!( - "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", - path, - &w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(false); - - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - sqlx::query!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "variables.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - - tx.commit().await?; - - // If variable was a secret, also delete from Vault backend (if configured) - if is_secret { - delete_secret_from_backend(&db, &w_id, path).await?; - } - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { path: path.to_string(), parent_path: Some(path.to_string()) }, - Some(format!("Variable '{}' deleted", path)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("variable {} deleted", path)) -} - -async fn delete_variables_bulk( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(request): Json, -) -> JsonResult> { - for path in &request.paths { - check_scopes(&authed, || format!("variables:write:{}", path))?; - } - - // Query which paths are secrets before deletion (for Vault cleanup) - let secret_paths: Vec = sqlx::query_scalar!( - "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", - &request.paths, - &w_id - ) - .fetch_all(&db) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM variable WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id - ) - .fetch_all(&mut *tx) - .await?; - sqlx::query!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2", - &deleted_paths, - w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "variables.delete_bulk", - ActionKind::Delete, - &w_id, - Some(&deleted_paths.join(", ")), - None, - ) - .await?; - - tx.commit().await?; - - // Delete secrets from Vault backend (if configured) - for path in &secret_paths { - if deleted_paths.contains(path) { - delete_secret_from_backend(&db, &w_id, path).await?; - } - } - - try_join_all(deleted_paths.iter().map(|path| { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { - path: path.to_string(), - parent_path: Some(path.to_string()), - }, - Some(format!("Variable '{}' deleted", path)), - true, - None, - ) - })) - .await?; - - for path in &deleted_paths { - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteVariable { workspace: w_id.clone(), path: path.to_owned() }, - ); - } - - Ok(Json(deleted_paths)) -} - -#[derive(Deserialize)] -struct EditVariable { - path: Option, - value: Option, - is_secret: Option, - description: Option, - account: Option, -} - -#[derive(Deserialize)] -struct AlreadyEncrypted { - already_encrypted: Option, -} - -async fn update_variable( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(AlreadyEncrypted { already_encrypted }): Query, - Json(ns): Json, -) -> Result { - use sql_builder::prelude::*; - - let path = path.to_path(); - check_scopes(&authed, || format!("variables:write:{}", path))?; - let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; - - let mut sqlb = SqlBuilder::update_table("variable"); - sqlb.and_where_eq("path", "?".bind(&path)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - - if let Some(npath) = &ns.path { - sqlb.set_str("path", npath); - } - let ns_value_is_none = ns.value.is_none(); - // Determine the target path for storing secrets (use new path if provided) - let target_path = ns.path.as_deref().unwrap_or(path); - if let Some(nvalue) = ns.value.clone() { - let is_secret = if ns.is_secret.is_some() { - ns.is_secret.unwrap() - } else { - sqlx::query_scalar!( - "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(false) - }; - - let value = if is_secret && !already_encrypted.unwrap_or(false) { - // Use secret backend for encryption (supports both DB and Vault) - // Store at target_path (new path if renaming, otherwise current path) - store_secret_value(&db, &w_id, target_path, &nvalue).await? - } else { - nvalue - }; - sqlb.set_str("value", &value); - } - - if let Some(desc) = ns.description { - sqlb.set_str("description", &desc); - } - - if let Some(account_id) = ns.account { - sqlb.set_str("account", account_id); - } - - if let Some(nbool) = ns.is_secret { - let old_secret = sqlx::query_scalar!( - "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(false); - if old_secret != nbool && ns_value_is_none { - return Err(Error::BadRequest( - "cannot change is_secret without updating value too".to_string(), - )); - } - sqlb.set_str("is_secret", nbool); - } - sqlb.returning("path"); - - // Get old account_id if we're updating the account field - let old_account_id = if ns.account.is_some() { - sqlx::query_scalar!( - "SELECT account FROM variable WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .flatten() - } else { - None - }; - - let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - - if let Some(npath) = ns.path.clone() { - if npath != path { - check_path_conflict(&db, &w_id, &npath).await?; - require_owner_of_path(&authed, path)?; - - // Handle Vault secret rename if the variable is a secret stored in Vault - let current_var = sqlx::query!( - "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - if let Some(var) = current_var { - if var.is_secret && is_vault_stored_value(&var.value) { - if ns.value.is_some() { - // New value was provided and already stored at new path - // Just delete the old secret from Vault - delete_secret_from_backend(&db, &w_id, path).await?; - } else { - // No new value - rename the secret in Vault - if let Some(new_value) = - rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? - { - // Update the variable's value to point to the new Vault path - sqlb.set_str("value", &new_value); - } - } - } - } - - let mut v = sqlx::query_scalar!( - "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - if let Some(old_v) = v { - v = Some(replace_path( - old_v, - &format!("$var:{path}"), - &format!("$var:{npath}"), - )) - } - - sqlx::query!( - "UPDATE resource SET path = $1, value = $2, edited_at = now() WHERE path = $3 AND workspace_id = $4", - npath, - v, - path, - w_id - ) - .execute(&mut *tx) - .await?; - } - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - - let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; - - let npath = not_found_if_none(npath_o, "Variable", path)?; - - audit_log( - &mut *tx, - &authed, - "variables.update", - ActionKind::Update, - &w_id, - Some(path), - None, - ) - .await?; - - // Clean up old account if it's no longer referenced and different from new account - if let Some(old_acc_id) = old_account_id { - if ns.account.is_some() && ns.account != Some(old_acc_id) { - // Check if old account is still referenced by other variables or resources - let account_still_used = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM variable WHERE account = $1 AND workspace_id = $2)", - old_acc_id, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(true); - - if !account_still_used { - // Delete the orphaned account - sqlx::query!( - "DELETE FROM account WHERE id = $1 AND workspace_id = $2", - old_acc_id, - &w_id - ) - .execute(&mut *tx) - .await?; - } - } - } - - tx.commit().await?; - - // Detect if this was a rename operation - let old_path_if_renamed = if npath != path { Some(path) } else { None }; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { path: npath.clone(), parent_path: Some(path.to_string()) }, - None, - true, - old_path_if_renamed, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateVariable { - workspace: w_id, - old_path: path.to_owned(), - new_path: npath.clone(), - }, - ); - - Ok(format!("variable {} updated (npath: {:?})", path, npath)) -} - -fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { - match v { - Value::Object(v) => Value::Object( - v.into_iter() - .map(|(k, v)| (k, replace_path(v, path, npath))) - .collect(), - ), - Value::Array(arr) => Value::Array( - arr.into_iter() - .map(|v| replace_path(v, path, npath)) - .collect(), - ), - Value::String(s) if s == path => Value::String(npath.to_owned()), - _ => v, - } -} - -pub async fn get_value_internal<'a>( - db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, - w_id: &str, - path: &str, - allow_cache: bool, -) -> Result { - if allow_cache { - if let Some(cached_variable) = get_cached_variable(&w_id, &path) { - return Ok(cached_variable); - } - } - - let mut tx = db_with_opt_authed.begin().await?; - let variable_o = sqlx::query!( - "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable - LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id - ) - .fetch_optional(&mut *tx) - .warn_after_seconds(5) - .await?; - drop(tx); - - let variable = if let Some(variable) = variable_o { - variable - } else { - explain_variable_perm_error(path, w_id, &db_with_opt_authed.db()).await?; - unreachable!() - }; - - let r = if variable.is_secret { - // let audit_author = - let mut tx = db_with_opt_authed.db().begin().await?; - audit_log( - &mut *tx, - db_with_opt_authed, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - tx.commit().await?; - - let value = variable.value; - if variable.is_expired.unwrap_or(false) && variable.account.is_some() { - #[cfg(feature = "oauth2")] - { - let db = db_with_opt_authed.db(); - let tx = db.begin().await?; - crate::oauth2_oss::_refresh_token( - tx, - &variable.path, - &w_id, - variable.account.unwrap(), - db, - ) - .await? - } - #[cfg(not(feature = "oauth2"))] - return Err(Error::internal_err("Require oauth2 feature".to_string())); - } else if !value.is_empty() { - // Use secret backend for decryption (supports both DB and Vault) - get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? - } else { - "".to_string() - } - } else { - variable.value - }; - - // Cache the result when explicitly allowed and caching appropriate - if allow_cache { - cache_variable(&w_id, &path, db_with_opt_authed.email(), r.clone()); - } - - Ok(r) -} +pub use windmill_store::variables::*; diff --git a/backend/windmill-api/src/webhook_util.rs b/backend/windmill-api/src/webhook_util.rs index 362893a7a1..0287191efc 100644 --- a/backend/windmill-api/src/webhook_util.rs +++ b/backend/windmill-api/src/webhook_util.rs @@ -1,152 +1 @@ -use std::time::Duration; - -use quick_cache::sync::Cache; -use serde::Serialize; -use tokio::{select, sync::mpsc}; - -#[cfg(feature = "prometheus")] -use windmill_common::METRICS_ENABLED; - -use crate::db::DB; -use windmill_common::oauth2::InstanceEvent; -use windmill_common::utils::configure_client; - -#[cfg(feature = "prometheus")] -lazy_static::lazy_static! { - // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens. - static ref WEBHOOK_REQUEST_COUNT: prometheus::Histogram = prometheus::register_histogram!( - "webhook_request", - "Histogram of webhook requests made" - ) - .unwrap(); - -} - -lazy_static::lazy_static! { - - pub static ref INSTANCE_EVENTS_WEBHOOK: Option = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok(); - - pub static ref WEBHOOK_CACHE: Cache> = Cache::new(100); - -} - -pub enum WebhookPayload { - WorkspaceEvent(String, WebhookMessage), - InstanceEvent(InstanceEvent), -} - -#[derive(Serialize)] -#[serde(tag = "type")] -pub enum WebhookMessage { - // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON - CreateApp { workspace: String, path: String }, - DeleteApp { workspace: String, path: String }, - UpdateApp { workspace: String, old_path: String, new_path: String }, - CreateFlow { workspace: String, path: String }, - UpdateFlow { workspace: String, old_path: String, new_path: String }, - ArchiveFlow { workspace: String, path: String }, - DeleteFlow { workspace: String, path: String }, - CreateFolder { workspace: String, name: String }, - UpdateFolder { workspace: String, name: String }, - DeleteFolder { workspace: String, name: String }, - DeleteResource { workspace: String, path: String }, - CreateResource { workspace: String, path: String }, - UpdateResource { workspace: String, old_path: String, new_path: String }, - CreateResourceType { name: String }, - DeleteResourceType { name: String }, - UpdateResourceType { name: String }, - CreateScript { workspace: String, path: String, hash: String }, - UpdateScript { workspace: String, path: String, hash: String }, - DeleteScript { workspace: String, hash: String }, - DeleteScriptPath { workspace: String, path: String }, - CreateVariable { workspace: String, path: String }, - UpdateVariable { workspace: String, old_path: String, new_path: String }, - DeleteVariable { workspace: String, path: String }, -} - -#[derive(Clone)] -pub struct WebhookShared { - pub channel: mpsc::UnboundedSender, -} - -impl WebhookShared { - pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self { - let (tx, mut rx) = mpsc::unbounded_channel::(); - let _process = tokio::spawn(async move { - let client = configure_client(reqwest::Client::builder() - .connect_timeout(Duration::from_secs(5)) - // TODO: investigate pool timeouts and such if TCP load is high - .timeout(Duration::from_secs(5))) - .build() - .unwrap(); - - loop { - select! { - biased; - _ = shutdown_rx.recv() => break, - r = rx.recv() => match r { - Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => { - let webhook_opt = match WEBHOOK_CACHE.get(&workspace_id) { - Some(guard) => { - guard - }, - None => { - let Ok(mut webhook_opt) = - sqlx::query_scalar!( - "SELECT webhook FROM workspace_settings WHERE workspace_id = $1", - workspace_id - ) - .fetch_one( - &db, - ) - .await else { - tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}"); - continue; - }; - if webhook_opt.as_ref().is_some_and(|x| x.is_empty()) { - webhook_opt = None; - } - WEBHOOK_CACHE.insert(workspace_id, webhook_opt.clone()); - webhook_opt - } - }; - if let Some(url) = webhook_opt { - #[cfg(feature = "prometheus")] - let timer = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; - tracing::info!("Sending webhook message to {}", url); - let _ = client.post(url).json(&message).send().await; - #[cfg(feature = "prometheus")] - timer.map(|x| x.stop_and_record()); - } - }, - Some(WebhookPayload::InstanceEvent(event)) => { - #[cfg(feature = "prometheus")] - if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; - let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await; - if let Err(e) = r { - tracing::error!("Error sending instance event: {}", e); - } - }, - None => break, - }, - } - } - }); - - Self { channel: tx } - } - - pub fn send_message(&self, workspace_id: String, message: WebhookMessage) { - let _ = self.channel.send(WebhookPayload::WorkspaceEvent( - workspace_id.clone(), - message, - )); - } - - pub fn send_instance_event(&self, event: InstanceEvent) { - if INSTANCE_EVENTS_WEBHOOK.is_none() { - return; - } - let _ = self.channel.send(WebhookPayload::InstanceEvent(event)); - } -} +pub use windmill_common::webhook::*; diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 667131bb79..dc2480ce15 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -148,11 +148,7 @@ async fn exists_workers_with_tags( Query(tags_query): Query, ) -> JsonResult> { // Create a list of requested tags - let mut tags: Vec = tags_query - .tags - .split(',') - .map(|s| s.to_string()) - .collect(); + let mut tags: Vec = tags_query.tags.split(',').map(|s| s.to_string()).collect(); // When TAGS_ARE_SENSITIVE is enabled, filter tags based on workspace visibility if *TAGS_ARE_SENSITIVE { diff --git a/backend/windmill-api/src/workspace_dependencies.rs b/backend/windmill-api/src/workspace_dependencies.rs index 2330de200a..d887923f5b 100644 --- a/backend/windmill-api/src/workspace_dependencies.rs +++ b/backend/windmill-api/src/workspace_dependencies.rs @@ -13,7 +13,7 @@ use windmill_common::{ workspace_dependencies::WorkspaceDependencies, DB, }; -use windmill_worker::workspace_dependencies::{ +use windmill_dep_map::workspace_dependencies::{ trigger_dependents_to_recompute_dependencies_in_the_background, NewWorkspaceDependencies, }; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 93c524f5fc..50c3913b47 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -6,93 +6,40 @@ * LICENSE-AGPL for a copy of the license. */ +// Re-export everything from windmill-api-workspaces +pub use windmill_api_workspaces::workspaces::*; + use crate::ai::{AIConfig, AI_REQUEST_CACHE}; use crate::db::ApiAuthed; -use crate::users_oss::send_email_if_possible; -use crate::utils::get_instance_username_or_create_pending; -use crate::BASE_URL; -use crate::{ - db::DB, - users::{WorkspaceInvite, VALID_USERNAME}, - utils::require_super_admin, - webhook_util::WebhookShared, -}; - -use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, -}; -use chrono::Utc; - -use regex::Regex; - -use hex; -use sha2::{Digest, Sha256}; -use std::collections::{HashMap, HashSet}; -use uuid::Uuid; -use windmill_audit::audit_oss::audit_log; -use windmill_audit::ActionKind; -use windmill_common::db::UserDB; -use windmill_common::s3_helpers::LargeFileStorage; -use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; -use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; -#[cfg(feature = "enterprise")] -use windmill_common::workspaces::GitRepositorySettings; -#[cfg(feature = "enterprise")] -use windmill_common::workspaces::WorkspaceDeploymentUISettings; -use windmill_common::workspaces::{ - get_datatable_resource_from_db_unchecked, DataTable, DataTableCatalogResourceType, - WorkspaceGitSyncSettings, -}; -use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; -use windmill_common::PgDatabase; -use windmill_common::{ - error::{Error, JsonResult, Result}, - global_settings::AUTOMATE_USERNAME_CREATION_SETTING, - oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, - utils::{paginate, rd_string, require_admin, Pagination}, -}; -use windmill_git_sync::{handle_deployment_metadata, handle_fork_branch_creation, DeployedObject}; -use windmill_worker::scoped_dependency_map::{ - DependencyDependent, DependencyMap, ScopedDependencyMap, -}; - -#[cfg(feature = "enterprise")] -use windmill_common::utils::require_admin_or_devops; - -use hyper::StatusCode; -use serde::{Deserialize, Serialize}; -use sqlx::{FromRow, Postgres, Transaction}; -use windmill_common::oauth2::InstanceEvent; -use windmill_common::utils::not_found_if_none; - use crate::teams_oss::{ connect_teams, edit_teams_command, run_teams_message_test_job, workspaces_list_available_teams_channels, workspaces_list_available_teams_ids, }; -lazy_static::lazy_static! { - static ref WORKSPACE_KEY_REGEXP: Regex = Regex::new("^[a-zA-Z0-9]{64}$").unwrap(); -} +use axum::{ + extract::{Extension, Path}, + routing::{get, post}, + Json, Router, +}; +use windmill_audit::audit_oss::audit_log; +use windmill_audit::ActionKind; +use windmill_common::{ + error::{Error, JsonResult, Result}, + utils::require_admin, + DB, +}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +#[cfg(feature = "enterprise")] +use axum::extract::Query; +#[cfg(feature = "enterprise")] +use serde::Deserialize; +#[cfg(feature = "enterprise")] +use windmill_common::utils::require_admin_or_devops; + +/// Wraps the subcrate's workspaced_service with routes that depend on windmill-api internals. pub fn workspaced_service() -> Router { - let router = Router::new() - .route("/get_as_superadmin", get(get_workspace_as_superadmin)) - .route("/list_pending_invites", get(list_pending_invites)) - .route("/update", post(edit_workspace)) - .route("/archive", post(archive_workspace)) - .route("/invite_user", post(invite_user)) - .route("/add_user", post(add_user)) - .route("/delete_invite", post(delete_invite)) - .route("/rebuild_dependency_map", post(rebuild_dependency_map)) - .route("/get_dependency_map", get(get_dependency_map)) - .route("/get_dependents/*imported_path", get(get_dependents)) - .route("/get_dependents_amounts", post(get_dependents_amounts)) - .route("/get_settings", get(get_settings)) - .route("/get_deploy_to", get(get_deploy_to)) - .route("/edit_slack_command", post(edit_slack_command)) + let router = windmill_api_workspaces::workspaces::workspaced_service() .route("/edit_teams_command", post(edit_teams_command)) .route( "/available_teams_ids", @@ -103,69 +50,13 @@ pub fn workspaced_service() -> Router { get(workspaces_list_available_teams_channels), ) .route("/connect_teams", post(connect_teams)) - .route( - "/run_slack_message_test_job", - post(run_slack_message_test_job), - ) .route( "/run_teams_message_test_job", post(run_teams_message_test_job), ) - .route("/slack_oauth_config", get(get_slack_oauth_config)) - .route("/slack_oauth_config", post(set_slack_oauth_config)) - .route("/slack_oauth_config", delete(delete_slack_oauth_config)) - .route("/edit_webhook", post(edit_webhook)) - .route("/edit_auto_invite", post(edit_auto_invite)) - .route("/edit_instance_groups", post(edit_instance_groups)) - .route("/edit_deploy_to", post(edit_deploy_to)) - .route( - "/get_secondary_storage_names", - get(get_secondary_storage_names), - ) .route("/tarball", get(crate::workspaces_export::tarball_workspace)) - .route("/is_premium", get(is_premium)) .route("/edit_copilot_config", post(edit_copilot_config)) .route("/get_copilot_info", get(get_copilot_info)) - .route("/edit_error_handler", post(edit_error_handler)) - .route("/edit_success_handler", post(edit_success_handler)) - .route( - "/edit_large_file_storage_config", - post(edit_large_file_storage_config), - ) - .route("/edit_ducklake_config", post(edit_ducklake_config)) - .route("/list_ducklakes", get(list_ducklakes)) - .route("/list_datatables", get(list_datatables)) - .route("/list_datatable_schemas", get(list_datatable_schemas)) - .route("/edit_datatable_config", post(edit_datatable_config)) - .route("/edit_git_sync_config", post(edit_git_sync_config)) - .route("/edit_git_sync_repository", post(edit_git_sync_repository)) - .route( - "/delete_git_sync_repository", - delete(delete_git_sync_repository), - ) - .route("/edit_deploy_ui_config", post(edit_deploy_ui_config)) - .route("/edit_default_app", post(edit_default_app)) - .route("/default_app", get(get_default_app)) - .route( - "/default_scripts", - post(edit_default_scripts).get(get_default_scripts), - ) - .route("/set_environment_variable", post(set_environment_variable)) - .route( - "/encryption_key", - get(get_encryption_key).post(set_encryption_key), - ) - .route("/leave", post(leave_workspace)) - .route("/get_workspace_name", get(get_workspace_name)) - .route("/create_fork", post(create_workspace_fork)) - .route("/change_workspace_name", post(change_workspace_name)) - .route("/change_workspace_color", post(change_workspace_color)) - .route( - "/change_workspace_id", - post(crate::workspaces_extra::change_workspace_id), - ) - .route("/usage", get(get_usage)) - .route("/used_triggers", get(get_used_triggers)) .route("/critical_alerts", get(get_critical_alerts)) .route( "/critical_alerts/:id/acknowledge", @@ -175,18 +66,7 @@ pub fn workspaced_service() -> Router { "/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts), ) - .route("/critical_alerts/mute", post(mute_critical_alerts)) - .route("/public_app_rate_limit", post(edit_public_app_rate_limit)) - .route("/operator_settings", post(update_operator_settings)) - .route( - "/create_workspace_fork_branch", - post(create_workspace_fork_branch), - ) - .route( - "/reset_diff_tally/:fork_workspace_id", - post(reset_workspace_diffs), - ) - .route("/compare/:target_workspace_id", get(compare_workspaces)); + .route("/critical_alerts/mute", post(mute_critical_alerts)); #[cfg(all(feature = "stripe", feature = "enterprise"))] { @@ -196,894 +76,6 @@ pub fn workspaced_service() -> Router { #[cfg(not(feature = "stripe"))] router } -pub fn global_service() -> Router { - Router::new() - .route("/list_as_superadmin", get(list_workspaces_as_super_admin)) - .route("/list", get(list_workspaces)) - .route("/users", get(user_workspaces)) - .route("/create", post(create_workspace)) - .route("/create_fork", post(deprecated_create_workspace_fork)) - .route("/exists", post(exists_workspace)) - .route("/exists_username", post(exists_username)) - .route("/allowed_domain_auto_invite", get(is_allowed_auto_domain)) - .route("/unarchive/:workspace", post(unarchive_workspace)) - .route( - "/delete/:workspace", - delete(crate::workspaces_extra::delete_workspace), - ) - .route( - "/create_workspace_require_superadmin", - get(create_workspace_require_superadmin), - ) -} - -#[derive(FromRow, Serialize)] -struct Workspace { - id: String, - name: String, - owner: String, - deleted: bool, - premium: bool, - color: Option, - parent_workspace_id: Option, -} - -#[derive(FromRow, Serialize, Debug)] -pub struct WorkspaceSettings { - pub workspace_id: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub slack_team_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub teams_team_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub teams_team_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub teams_team_guid: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub slack_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub slack_command_script: Option, - pub teams_command_script: Option, - pub slack_email: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub slack_oauth_client_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub slack_oauth_client_secret: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub customer_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub plan: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub webhook: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub deploy_to: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ai_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub large_file_storage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ducklake: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub datatable: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub git_sync: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub deploy_ui: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub default_app: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub default_scripts: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mute_critical_alerts: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub color: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub operator_settings: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub git_app_installations: Option, - // Grouped config fields - #[serde(skip_serializing_if = "Option::is_none")] - pub auto_invite: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub success_handler: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub public_app_execution_limit_per_minute: Option, -} - -/// #[derive(sqlx::Type, Serialize, Deserialize, Debug)] -// #[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")] -// pub enum WorkspaceKeyKind { -// Cloud, -// } - -#[derive(Deserialize)] -struct EditCommandScript { - slack_command_script: Option, -} - -#[derive(Deserialize)] -struct RunSlackMessageTestJobRequest { - hub_script_path: String, - channel: String, - test_msg: String, -} - -#[derive(Serialize)] -struct RunSlackMessageTestJobResponse { - job_uuid: String, -} - -#[cfg(feature = "enterprise")] -#[derive(Deserialize)] -struct EditDeployTo { - deploy_to: Option, -} - -#[allow(dead_code)] -#[derive(Deserialize)] -pub struct EditAutoInvite { - pub operator: Option, - pub invite_all: Option, - pub auto_add: Option, -} - -#[derive(Deserialize)] -struct EditWebhook { - webhook: Option, -} - -#[derive(Deserialize, Serialize, Debug)] -struct LargeFileStorageWithSecondary { - #[serde(flatten)] - large_file_storage: LargeFileStorage, - #[serde(default)] - secondary_storage: HashMap, -} -#[derive(Deserialize, Debug)] -struct EditLargeFileStorageConfig { - large_file_storage: Option, -} - -#[derive(Deserialize, Debug)] -struct EditDucklakeConfig { - settings: DucklakeSettings, -} - -#[derive(Deserialize, Serialize, Debug)] -pub struct DucklakeSettings { - pub ducklakes: HashMap, -} - -#[derive(Deserialize, Debug)] -struct EditDataTableConfig { - settings: DataTableSettings, -} - -#[derive(Deserialize, Serialize, Debug)] -pub struct DataTableSettings { - pub datatables: HashMap, -} - -#[derive(Deserialize)] -struct CreateWorkspace { - id: String, - name: String, - username: Option, - color: Option, -} - -#[derive(Deserialize)] -struct CreateWorkspaceFork { - id: String, - name: String, - color: Option, -} - -#[derive(Deserialize)] -struct EditWorkspace { - name: String, - owner: String, -} - -#[derive(Serialize)] -struct WorkspaceList { - pub email: String, - pub workspaces: Vec, -} - -#[derive(Serialize)] -struct UserWorkspace { - pub id: String, - pub name: String, - pub username: String, - pub color: Option, - pub operator_settings: Option>, - pub parent_workspace_id: Option, - pub disabled: bool, -} - -#[derive(Deserialize)] -struct WorkspaceId { - pub id: String, -} - -#[derive(Deserialize)] -struct ValidateUsername { - pub id: String, - pub username: String, -} - -#[derive(Deserialize)] -pub struct NewWorkspaceInvite { - pub email: String, - pub is_admin: bool, - pub operator: bool, -} - -#[derive(Deserialize)] -pub struct NewWorkspaceUser { - pub email: String, - pub username: Option, - pub is_admin: bool, - pub operator: bool, -} - -// New format for error handler (grouped) -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EditErrorHandlerNew { - pub path: Option, - pub extra_args: Option, - #[serde(default)] - pub muted_on_cancel: bool, - #[serde(default)] - pub muted_on_user_path: bool, -} - -// Legacy format for error handler (flat fields from old CLI) -#[derive(Deserialize)] -pub struct EditErrorHandlerLegacy { - pub error_handler: Option, - pub error_handler_extra_args: Option, - #[serde(default)] - pub error_handler_muted_on_cancel: bool, -} - -// Accepts both old and new formats -#[derive(Deserialize)] -#[serde(untagged)] -pub enum EditErrorHandler { - New(EditErrorHandlerNew), - Legacy(EditErrorHandlerLegacy), -} - -impl EditErrorHandler { - pub fn into_normalized(self) -> EditErrorHandlerNew { - match self { - EditErrorHandler::New(new) => new, - EditErrorHandler::Legacy(legacy) => EditErrorHandlerNew { - path: legacy.error_handler, - extra_args: legacy.error_handler_extra_args, - muted_on_cancel: legacy.error_handler_muted_on_cancel, - muted_on_user_path: false, // Old format doesn't have this field - }, - } - } -} - -// New format for success handler (grouped) -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EditSuccessHandlerNew { - pub path: Option, - pub extra_args: Option, -} - -// Legacy format for success handler (flat fields from old CLI) -#[derive(Deserialize)] -pub struct EditSuccessHandlerLegacy { - pub success_handler: Option, - pub success_handler_extra_args: Option, -} - -// Accepts both old and new formats -#[derive(Deserialize)] -#[serde(untagged)] -pub enum EditSuccessHandler { - New(EditSuccessHandlerNew), - Legacy(EditSuccessHandlerLegacy), -} - -impl EditSuccessHandler { - pub fn into_normalized(self) -> EditSuccessHandlerNew { - match self { - EditSuccessHandler::New(new) => new, - EditSuccessHandler::Legacy(legacy) => EditSuccessHandlerNew { - path: legacy.success_handler, - extra_args: legacy.success_handler_extra_args, - }, - } - } -} - -lazy_static::lazy_static! { - pub static ref EMAIL_REGEXP: Regex = Regex::new(r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$").unwrap(); -} - -async fn list_pending_invites( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as!( - WorkspaceInvite, - "SELECT - workspace_invite.workspace_id, - workspace_invite.email, - workspace_invite.is_admin, - workspace_invite.operator, - workspace.parent_workspace_id - FROM workspace_invite JOIN workspace ON workspace_invite.workspace_id = workspace.id - WHERE workspace_id = $1", - w_id - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(rows)) -} - -async fn is_premium( - authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, -) -> JsonResult { - require_admin(authed.is_admin, &authed.username)?; - #[cfg(feature = "cloud")] - let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) - .await? - .premium; - #[cfg(not(feature = "cloud"))] - let premium = false; - Ok(Json(premium)) -} - -async fn exists_workspace( - authed: ApiAuthed, - Extension(user_db): Extension, - Json(WorkspaceId { id }): Json, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)", - id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - tx.commit().await?; - Ok(Json(exists)) -} - -async fn list_workspaces( - authed: ApiAuthed, - Extension(user_db): Extension, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let workspaces = sqlx::query_as!( - Workspace, - "SELECT workspace.id, workspace.name, workspace.owner, workspace.deleted, workspace.premium, workspace_settings.color, workspace.parent_workspace_id - FROM workspace - LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id - JOIN usr ON usr.workspace_id = workspace.id - WHERE usr.email = $1 AND workspace.deleted = false", - authed.email - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(workspaces)) -} - -async fn get_settings( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let settings = sqlx::query_as!( - WorkspaceSettings, - r#" - SELECT - workspace_id, - slack_team_id, - teams_team_id, - teams_team_name, - teams_team_guid, - slack_name, - slack_command_script, - teams_command_script, - slack_email, - slack_oauth_client_id, - slack_oauth_client_secret, - customer_id, - plan, - webhook, - deploy_to, - ai_config, - large_file_storage, - datatable, - ducklake, - git_sync, - deploy_ui, - default_app, - default_scripts, - mute_critical_alerts, - color, - operator_settings, - git_app_installations, - auto_invite, - error_handler, - success_handler, - public_app_execution_limit_per_minute - FROM - workspace_settings - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_optional(&mut *tx) - .await - .map_err(|e| Error::internal_err(format!("getting settings: {e:#}")))?; - - tx.commit().await?; - - let settings = not_found_if_none(settings, "workspace settings", &w_id)?; - Ok(Json(settings)) -} - -#[derive(Serialize)] -struct DeployTo { - deploy_to: Option, -} -async fn get_deploy_to( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let settings = sqlx::query_as!( - DeployTo, - "SELECT deploy_to FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&mut *tx) - .await - .map_err(|e| Error::internal_err(format!("getting deploy_to: {e:#}")))?; - - tx.commit().await?; - Ok(Json(settings)) -} - -async fn edit_slack_command( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(es): Json, -) -> Result { - require_admin(is_admin, &username)?; - let mut tx = db.begin().await?; - - if es.slack_command_script.is_some() { - let exists_slack_command_with_team_id = sqlx::query_scalar!( - r#" - SELECT EXISTS (SELECT 1 - FROM workspace_settings - WHERE workspace_id <> $1 - AND slack_command_script IS NOT NULL - AND slack_team_id IS NOT NULL - AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1)) - "#, - &w_id - ) - .fetch_one(&mut *tx) - .await?.unwrap_or(false); - - if exists_slack_command_with_team_id { - return Err(Error::BadRequest( - "A workspace connected to the same slack team already has a command script. Please remove it first." - .to_string(), - )); - } - } - - sqlx::query!( - "UPDATE workspace_settings SET slack_command_script = $1 WHERE workspace_id = $2", - es.slack_command_script, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.edit_command_script", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some( - [( - "script", - es.slack_command_script - .unwrap_or("NO_SCRIPT".to_string()) - .as_str(), - )] - .into(), - ), - ) - .await?; - tx.commit().await?; - - Ok(format!("Edit command script {}", &w_id)) -} - -async fn run_slack_message_test_job( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(req): Json, -) -> JsonResult { - let mut fake_result = HashMap::new(); - fake_result.insert("error".to_string(), to_raw_value(&req.test_msg)); - fake_result.insert("success_result".to_string(), to_raw_value(&req.test_msg)); - - let mut extra_args = HashMap::new(); - extra_args.insert("channel".to_string(), to_raw_value(&req.channel)); - extra_args.insert( - "slack".to_string(), - to_raw_value(&format!("$res:{WORKSPACE_SLACK_BOT_TOKEN_PATH}")), - ); - - let uuid = windmill_queue::push_error_handler( - &db, - Uuid::parse_str("00000000-0000-0000-0000-000000000000")?, - None, - Some("slack_message_test".to_string()), - false, - w_id.as_str(), - &format!("script/{}", req.hub_script_path.as_str()), - sqlx::types::Json(&fake_result), - None, - Some(Utc::now()), - Some(sqlx::types::Json(to_raw_value(&extra_args))), - authed.email.as_str(), - false, - false, - None, // Note: we could mark it as high priority to return result quickly to the user - ) - .await?; - - Ok(Json(RunSlackMessageTestJobResponse { - job_uuid: uuid.to_string(), - })) -} - -#[derive(Deserialize)] -struct SetSlackOAuthConfigRequest { - slack_oauth_client_id: String, - slack_oauth_client_secret: String, -} - -#[derive(Serialize)] -struct GetSlackOAuthConfigResponse { - slack_oauth_client_id: Option, - slack_oauth_client_secret: Option, -} - -async fn get_slack_oauth_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult { - require_admin(authed.is_admin, &authed.username)?; - - let settings = sqlx::query!( - "SELECT slack_oauth_client_id, slack_oauth_client_secret FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - - // Mask the secret if it exists - let masked_secret = settings - .slack_oauth_client_secret - .map(|_| "***".to_string()); - - Ok(Json(GetSlackOAuthConfigResponse { - slack_oauth_client_id: settings.slack_oauth_client_id, - slack_oauth_client_secret: masked_secret, - })) -} - -async fn set_slack_oauth_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(req): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - if req.slack_oauth_client_id.is_empty() || req.slack_oauth_client_secret.is_empty() { - return Err(Error::BadRequest( - "Both client ID and client secret are required".to_string(), - )); - } - - let mut tx = db.begin().await?; - - sqlx::query!( - "UPDATE workspace_settings - SET slack_oauth_client_id = $1, slack_oauth_client_secret = $2 - WHERE workspace_id = $3", - &req.slack_oauth_client_id, - &req.slack_oauth_client_secret, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.set_slack_oauth_config", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("client_id", req.slack_oauth_client_id.as_str())].into()), - ) - .await?; - - tx.commit().await?; - - Ok(format!("Slack OAuth config set for workspace {}", &w_id)) -} - -async fn delete_slack_oauth_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - - sqlx::query!( - "UPDATE workspace_settings - SET slack_oauth_client_id = NULL, slack_oauth_client_secret = NULL - WHERE workspace_id = $1", - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.delete_slack_oauth_config", - ActionKind::Delete, - &w_id, - Some(&authed.email), - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!( - "Slack OAuth config deleted for workspace {}", - &w_id - )) -} - -#[derive(Deserialize)] -struct GetSecondaryStorageNamesQuery { - #[serde(default)] - include_default: bool, -} - -async fn get_secondary_storage_names( - _authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Query(query): Query, -) -> JsonResult> { - let mut result: Vec = sqlx::query_scalar!( - "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') AS \"secondary_storage_name!: _\" - FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_all(&db) - .await?; - - // If include_default is true, check if primary storage is set and add "_default_" - if query.include_default { - let has_primary_storage: Option = sqlx::query_scalar!( - "SELECT (large_file_storage IS NOT NULL - AND large_file_storage != 'null'::jsonb - AND jsonb_typeof(large_file_storage) = 'object') AS \"has_primary!\" - FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_optional(&db) - .await?; - - if has_primary_storage.unwrap_or(false) { - result.insert(0, "_default_".to_string()); - } - } - - Ok(Json(result)) -} - -#[cfg(feature = "enterprise")] -async fn edit_deploy_to( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(es): Json, -) -> Result { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - sqlx::query!( - "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", - es.deploy_to, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.edit_deploy_to", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some( - [( - "script", - es.deploy_to.unwrap_or("NO_DEPLOY_TO".to_string()).as_str(), - )] - .into(), - ), - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Settings { setting_type: "deploy_to".to_string() }, - None, - false, - None, - ) - .await?; - - Ok(format!("Edit deploy to for {}", &w_id)) -} - -#[cfg(not(feature = "enterprise"))] -async fn edit_deploy_to() -> Result { - return Err(Error::BadRequest( - "Deploy to is only available on enterprise".to_string(), - )); -} - -pub const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt"); -pub const WM_FORK_PREFIX: &str = "wm-fork-"; -pub const MAX_CUSTOM_PROMPT_LENGTH: usize = 5000; - -async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult { - let domain = email.split('@').last().unwrap(); - return Ok(Json(!BANNED_DOMAINS.contains(domain))); -} - -async fn edit_auto_invite( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(ea): Json, -) -> Result { - crate::workspaces_oss::edit_auto_invite(authed, db, w_id, ea).await -} - -#[cfg(feature = "private")] -async fn edit_instance_groups( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(config): Json, -) -> Result { - crate::workspaces_ee::edit_instance_groups(authed, db, w_id, config).await -} - -#[cfg(not(feature = "private"))] -async fn edit_instance_groups( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_config): Json, -) -> Result { - Err(Error::BadRequest( - "Instance groups are only available on Windmill Enterprise Edition".to_string(), - )) -} - -async fn edit_webhook( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(ew): Json, -) -> Result { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - - if let Some(webhook) = &ew.webhook { - sqlx::query!( - "UPDATE workspace_settings SET webhook = $1 WHERE workspace_id = $2", - webhook, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET webhook = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - audit_log( - &mut *tx, - &authed, - "workspaces.edit_webhook", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("webhook", &format!("{:?}", ew.webhook)[..])].into()), - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Settings { setting_type: "webhook".to_string() }, - None, - false, - None, - ) - .await?; - - Ok(format!("Edit webhook for workspace {}", &w_id)) -} async fn edit_copilot_config( authed: ApiAuthed, @@ -1094,7 +86,6 @@ async fn edit_copilot_config( ) -> Result { require_admin(is_admin, &username)?; - // Validate custom prompts length if let Some(ref custom_prompts) = ai_config.custom_prompts { for (mode, prompt) in custom_prompts.iter() { if prompt.len() > MAX_CUSTOM_PROMPT_LENGTH { @@ -1136,13 +127,12 @@ async fn edit_copilot_config( .await?; tx.commit().await?; - // Trigger git sync for AI config changes handle_deployment_metadata( &authed.email, &authed.username, &db, &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "ai_config".to_string() }, + DeployedObject::Settings { setting_type: "ai_config".to_string() }, Some("AI configuration updated".to_string()), false, None, @@ -1183,3124 +173,6 @@ async fn get_copilot_info( } } -async fn edit_large_file_storage_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - - let args_for_audit = format!("{:?}", new_config.large_file_storage); - audit_log( - &mut *tx, - &authed, - "workspaces.edit_large_file_storage_config", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("large_file_storage", args_for_audit.as_str())].into()), - ) - .await?; - - if let Some(lfs_config) = new_config.large_file_storage { - let serialized_lfs_config = - serde_json::to_value::(lfs_config) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", - serialized_lfs_config, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET large_file_storage = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; - - // Trigger git sync for large file storage changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { - setting_type: "large_file_storage".to_string(), - }, - Some("Large file storage configuration updated".to_string()), - false, - None, - ) - .await?; - - Ok(format!( - "Edit large file storage config for workspace {}", - &w_id - )) -} - -async fn list_ducklakes( - _authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let ducklakes = sqlx::query_scalar!( - r#" - SELECT jsonb_object_keys(ws.ducklake->'ducklakes') AS ducklake_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|s| s) - .collect(); - - Ok(Json(ducklakes)) -} - -async fn list_datatables( - _authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let datatables = sqlx::query_scalar!( - r#" - SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|s| s) - .collect(); - - Ok(Json(datatables)) -} - -/// Compact column representation: "type" or "type?" for nullable, with "=default" suffix if has default -type CompactColumn = String; - -/// Columns mapped by name to their compact type -type ColumnMap = HashMap; - -/// Tables mapped by name to their columns -type TableMap = HashMap; - -/// Schemas mapped by name to their tables -type SchemaMap = HashMap; - -#[derive(Serialize, Debug)] -struct DataTableSchema { - datatable_name: String, - /// Hierarchical schema: schema_name -> table_name -> column_name -> "type[?][=default]" - schemas: SchemaMap, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, -} - -async fn list_datatable_schemas( - _authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - // Get all datatable names for this workspace - let datatable_names: Vec = sqlx::query_scalar!( - r#" - SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|s| s) - .collect(); - - let mut results = Vec::new(); - - for datatable_name in datatable_names { - let schema = match get_datatable_schema(&db, &w_id, &datatable_name).await { - Ok(schemas) => DataTableSchema { datatable_name, schemas, error: None }, - Err(e) => DataTableSchema { - datatable_name, - schemas: HashMap::new(), - error: Some(e.to_string()), - }, - }; - results.push(schema); - } - - Ok(Json(results)) -} - -async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Result { - // Get the datatable resource (connection credentials) - let db_resource = get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; - - // Parse the resource as PgDatabase - let pg_db: PgDatabase = serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; - - // Connect to the datatable database - let (client, connection) = pg_db.connect().await?; - - // Spawn the connection handler - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::error!("Datatable connection error: {}", e); - } - }); - - // First, get all non-system schemas (including empty ones) - let schema_rows = client - .query( - r#" - SELECT nspname::text AS schema_name - FROM pg_namespace - WHERE nspname NOT IN ('information_schema', 'pg_toast', 'pg_catalog') - AND nspname NOT LIKE 'pg_%' - ORDER BY nspname - "#, - &[], - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to query schemas: {}", e)))?; - - // Build hierarchical structure: schema -> table -> column -> compact_type - let mut schema_map: SchemaMap = HashMap::new(); - - // Collect schema names and initialize map - let schema_names: Vec = schema_rows - .iter() - .map(|row| { - let name: String = row.get(0); - schema_map.entry(name.clone()).or_default(); - name - }) - .collect(); - - // Query column information only for the schemas we found - let rows = client - .query( - r#" - SELECT - table_schema::text, - table_name::text, - column_name::text, - udt_name::text, - is_nullable::text, - column_default::text - FROM information_schema.columns - WHERE table_schema = ANY($1) - AND table_name IS NOT NULL - ORDER BY table_schema, table_name, ordinal_position - "#, - &[&schema_names], - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to query columns: {}", e)))?; - - for row in rows { - let table_schema: String = row.get(0); - let table_name: String = row.get(1); - let column_name: String = row.get(2); - let udt_name: String = row.get(3); - let is_nullable: String = row.get(4); - let column_default: Option = row.get(5); - - // Build compact type representation: "type[?][=default]" - let mut compact = udt_name; - if is_nullable == "YES" { - compact.push('?'); - } - if let Some(default) = column_default { - // Truncate long defaults for compactness - let short_default = if default.len() > 30 { - format!("{}...", &default[..27]) - } else { - default - }; - compact.push('='); - compact.push_str(&short_default); - } - - schema_map - .entry(table_schema) - .or_default() - .entry(table_name) - .or_default() - .insert(column_name, compact); - } - - Ok(schema_map) -} - -async fn edit_ducklake_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, email, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - require_admin(is_admin, &username)?; - let is_superadmin = require_super_admin(&db, &email).await.is_ok(); - - let mut tx = db.begin().await?; - - let args_for_audit = format!("{:?}", new_config.settings); - audit_log( - &mut *tx, - &authed, - "workspaces.edit_ducklake_config", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("ducklake", args_for_audit.as_str())].into()), - ) - .await?; - - // Check that non-superadmins are not abusing Instance databases - if !is_superadmin { - let old_ducklakes = sqlx::query_scalar!( - r#" - SELECT ws.ducklake->'ducklakes' AS ducklake_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(serde_json::Value::Null); - let old_ducklakes: HashMap = - serde_json::from_value(old_ducklakes).unwrap_or_default(); - for (name, dl) in new_config.settings.ducklakes.iter() { - if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { - let old_dl = old_ducklakes.get(name); - if old_dl.is_none() - || old_dl.unwrap().catalog.resource_type - != DucklakeCatalogResourceType::Instance - || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path - { - return Err(Error::BadRequest( - "Only superadmins can create or modify ducklakes with Instance databases" - .to_string(), - )); - } - } - } - } - - let config: serde_json::Value = serde_json::to_value(new_config.settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET ducklake = $1 WHERE workspace_id = $2", - config, - &w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(format!("Edit ducklake config for workspace {}", &w_id)) -} - -async fn edit_datatable_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, email, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - require_admin(is_admin, &username)?; - let is_superadmin = require_super_admin(&db, &email).await.is_ok(); - - let mut tx = db.begin().await?; - - let args_for_audit = format!("{:?}", new_config.settings); - audit_log( - &mut *tx, - &authed, - "workspaces.edit_datatable_config", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("datatable", args_for_audit.as_str())].into()), - ) - .await?; - - // Check that non-superadmins are not abusing Instance databases - if !is_superadmin { - let old_datatables = sqlx::query_scalar!( - r#" - SELECT ws.datatable->'datatables' AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(serde_json::Value::Null); - let old_datatables: HashMap = - serde_json::from_value(old_datatables).unwrap_or_default(); - for (name, dt) in new_config.settings.datatables.iter() { - if dt.database.resource_type == DataTableCatalogResourceType::Instance { - let old_dt = old_datatables.get(name); - if old_dt.is_none() - || old_dt.unwrap().database.resource_type - != DataTableCatalogResourceType::Instance - || old_dt.unwrap().database.resource_path != dt.database.resource_path - { - return Err(Error::BadRequest( - "Only superadmins can create or modify data tables with Instance databases" - .to_string(), - )); - } - } - } - } - - let config: serde_json::Value = serde_json::to_value(new_config.settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2", - config, - &w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(format!("Edit datatable config for workspace {}", &w_id)) -} - -#[derive(Deserialize)] -pub struct EditGitSyncConfig { - #[cfg(feature = "enterprise")] - pub git_sync_settings: Option, -} - -#[cfg(feature = "enterprise")] -#[derive(Deserialize, Debug)] -pub struct EditGitSyncRepository { - pub git_repo_resource_path: String, - pub repository: GitRepositorySettings, -} - -#[cfg(feature = "enterprise")] -#[derive(Deserialize, Debug)] -pub struct DeleteGitSyncRepositoryRequest { - pub git_repo_resource_path: String, -} - -#[cfg(feature = "enterprise")] -fn validate_git_repo_resource_path(path: &str) -> Result<()> { - // Resource paths should follow the pattern: $res:f// or $res:u// - if path.is_empty() { - return Err(Error::BadRequest( - "Resource path cannot be empty".to_string(), - )); - } - - // Must start with $res: prefix - if !path.starts_with("$res:") { - return Err(Error::BadRequest( - "Resource path must start with '$res:'".to_string(), - )); - } - - // Extract the actual path after $res: - let actual_path = &path[5..]; // Remove "$res:" prefix - - // Basic validation: must start with f/ or u/ and contain at least one slash - if !actual_path.starts_with("f/") && !actual_path.starts_with("u/") { - return Err(Error::BadRequest( - "Resource path must start with '$res:f/' or '$res:u/'".to_string(), - )); - } - - // Must have at least 3 parts (type, folder/user, name) - let parts: Vec<&str> = actual_path.split('/').collect(); - if parts.len() < 3 || parts.iter().any(|part| part.is_empty()) { - return Err(Error::BadRequest( - "Invalid resource path format".to_string(), - )); - } - - // Resource name validation (last part) - let resource_name = parts.last().unwrap(); - if !resource_name - .chars() - .all(|c| c.is_alphanumeric() || c == '_' || c == '-') - { - return Err(Error::BadRequest( - "Resource name can only contain alphanumeric characters, underscores, and hyphens" - .to_string(), - )); - } - - Ok(()) -} - -#[cfg(feature = "enterprise")] -fn cleanup_legacy_git_sync_settings_in_memory( - git_sync_settings: &mut windmill_common::workspaces::WorkspaceGitSyncSettings, - workspace_id: &str, -) { - // Check if all repositories are in new format (have settings field) - let all_repos_migrated = git_sync_settings - .repositories - .iter() - .all(|repo| repo.settings.is_some()); - - // If all repos are migrated and we still have legacy workspace-level settings - if all_repos_migrated - && (git_sync_settings.include_path.is_some() || git_sync_settings.include_type.is_some()) - { - tracing::info!( - workspace_id = workspace_id, - "All git sync repositories migrated to new format, cleaning up legacy workspace-level settings" - ); - - // Remove workspace-level legacy fields - git_sync_settings.include_path = None; - git_sync_settings.include_type = None; - } -} - -#[cfg(not(feature = "enterprise"))] -async fn edit_git_sync_config( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_new_config): Json, -) -> Result { - return Err(Error::BadRequest( - "Git sync is only available on Windmill Enterprise Edition".to_string(), - )); -} - -#[cfg(feature = "enterprise")] -async fn edit_git_sync_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - - let args_for_audit = format!("{:?}", new_config.git_sync_settings); - audit_log( - &mut *tx, - &authed, - "workspaces.edit_git_sync_config", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("git_sync_settings", args_for_audit.as_str())].into()), - ) - .await?; - - if let Some(mut git_sync_settings) = new_config.git_sync_settings { - // Clean up legacy workspace-level settings if all repos are migrated - cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); - - let serialized_config = serde_json::to_value::(git_sync_settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", - serialized_config, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET git_sync = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - - tx.commit().await?; - - // Trigger git sync for git sync settings changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() }, - Some("Git sync configuration updated".to_string()), - false, - None, - ) - .await?; - - Ok(format!("Edit git sync config for workspace {}", &w_id)) -} - -#[cfg(not(feature = "enterprise"))] -async fn edit_git_sync_repository( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_new_config): Json, -) -> Result { - return Err(Error::BadRequest( - "Git sync is only available on Windmill Enterprise Edition".to_string(), - )); -} - -#[cfg(feature = "enterprise")] -async fn edit_git_sync_repository( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - require_admin(is_admin, &username)?; - - // Validate the resource path format - validate_git_repo_resource_path(&new_config.git_repo_resource_path)?; - - let mut tx = db.begin().await?; - - // First, get the current git sync settings - let current_settings = sqlx::query!( - "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - let mut git_sync_settings = if let Some(row) = current_settings { - if let Some(git_sync) = row.git_sync { - serde_json::from_value::(git_sync) - .map_err(|err| Error::internal_err(err.to_string()))? - } else { - WorkspaceGitSyncSettings::default() - } - } else { - WorkspaceGitSyncSettings::default() - }; - - // Audit log before we move the repository - audit_log( - &mut *tx, - &authed, - "workspaces.edit_git_sync_repository", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some( - [ - ( - "repository_path", - new_config.git_repo_resource_path.as_str(), - ), - ("repository_data", &format!("{:?}", new_config.repository)), - ] - .into(), - ), - ) - .await?; - - // Check if repository exists before modifying - let repo_exists = git_sync_settings - .repositories - .iter() - .any(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path); - - // Find and update the specific repository, or add it if it doesn't exist - let repo_found = git_sync_settings - .repositories - .iter_mut() - .find(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path); - - if let Some(existing_repo) = repo_found { - // Update existing repository - *existing_repo = new_config.repository; - } else { - // Repository doesn't exist, add it as a new repository - git_sync_settings.repositories.push(new_config.repository); - } - - // Clean up legacy workspace-level settings if all repos are migrated - cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); - - // Save the updated configuration - let serialized_config = serde_json::to_value::(git_sync_settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", - serialized_config, - &w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - // Trigger git sync for individual repository update/add - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() }, - Some(format!( - "Git sync repository '{}' {}", - new_config.git_repo_resource_path, - if repo_exists { "updated" } else { "added" } - )), - false, - None, - ) - .await?; - - Ok(format!( - "{} git sync repository '{}' for workspace {}", - if repo_exists { "Updated" } else { "Added" }, - new_config.git_repo_resource_path, - &w_id - )) -} - -#[cfg(not(feature = "enterprise"))] -async fn delete_git_sync_repository( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_request): Json, -) -> Result { - return Err(Error::BadRequest( - "Git sync is only available on Windmill Enterprise Edition".to_string(), - )); -} - -#[cfg(feature = "enterprise")] -async fn delete_git_sync_repository( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(request): Json, -) -> Result { - require_admin(is_admin, &username)?; - - // For deletion, only validate that path is not empty to allow cleanup of malformed entries - if request.git_repo_resource_path.is_empty() { - return Err(Error::BadRequest( - "Resource path cannot be empty".to_string(), - )); - } - - let mut tx = db.begin().await?; - - // First, get the current git sync settings - let current_settings = sqlx::query!( - "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - let mut git_sync_settings = if let Some(row) = current_settings { - if let Some(git_sync) = row.git_sync { - serde_json::from_value::(git_sync) - .map_err(|err| Error::internal_err(err.to_string()))? - } else { - WorkspaceGitSyncSettings::default() - } - } else { - WorkspaceGitSyncSettings::default() - }; - - // Check if repository exists and remove it - let original_count = git_sync_settings.repositories.len(); - git_sync_settings - .repositories - .retain(|repo| repo.git_repo_resource_path != request.git_repo_resource_path); - - if git_sync_settings.repositories.len() == original_count { - return Err(Error::BadRequest(format!( - "Repository with path '{}' not found in git sync configuration", - request.git_repo_resource_path - ))); - } - - // Audit log - audit_log( - &mut *tx, - &authed, - "workspaces.delete_git_sync_repository", - ActionKind::Delete, - &w_id, - Some(&authed.email), - Some([("repository_path", request.git_repo_resource_path.as_str())].into()), - ) - .await?; - - // Clean up legacy workspace-level settings if all repos are migrated - cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); - - // Save the updated configuration - let serialized_config = serde_json::to_value::(git_sync_settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", - serialized_config, - &w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - // Trigger git sync for repository deletion - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "git_sync".to_string() }, - Some(format!( - "Git sync repository '{}' deleted", - request.git_repo_resource_path - )), - false, - None, - ) - .await?; - - Ok(format!( - "Deleted git sync repository '{}' from workspace {}", - request.git_repo_resource_path, &w_id - )) -} - -#[cfg(feature = "enterprise")] -#[derive(Debug, Deserialize)] -struct EditDeployUIConfig { - deploy_ui_settings: Option, -} - -#[cfg(not(feature = "enterprise"))] -async fn edit_deploy_ui_config( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, -) -> Result { - return Err(Error::BadRequest( - "Deployment UI is only available on Windmill Enterprise Edition".to_string(), - )); -} - -#[cfg(feature = "enterprise")] -async fn edit_deploy_ui_config( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - let args_for_audit = format!("{:?}", new_config.deploy_ui_settings); - audit_log( - &mut *tx, - &authed, - "workspaces.edit_deploy_ui_config", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("deployment_ui_settings", args_for_audit.as_str())].into()), - ) - .await?; - - if let Some(deploy_ui_settings) = new_config.deploy_ui_settings { - let serialized_config = - serde_json::to_value::(deploy_ui_settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET deploy_ui = $1 WHERE workspace_id = $2", - serialized_config, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET deploy_ui = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; - - Ok(format!("Edit deployment UI config for workspace {}", &w_id)) -} - -#[derive(Deserialize)] -pub struct EditDefaultApp { - #[cfg(feature = "enterprise")] - pub default_app_path: Option, -} - -#[cfg(not(feature = "enterprise"))] -async fn edit_default_app( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_new_config): Json, -) -> Result { - return Err(Error::BadRequest( - "Setting a workspace default app is only available on Windmill Enterprise Edition" - .to_string(), - )); -} - -async fn edit_default_scripts( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json>, -) -> Result { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.edit_default_scripts", - ActionKind::Update, - &w_id, - Some(&authed.email), - None, - ) - .await?; - - if let Some(config) = new_config { - sqlx::query!( - "UPDATE workspace_settings SET default_scripts = $1 WHERE workspace_id = $2", - config, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET default_scripts = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; - - // Trigger git sync for default scripts changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "default_scripts".to_string() }, - Some("Default scripts configuration updated".to_string()), - false, - None, - ) - .await?; - - Ok(format!("Edit default scripts for workspace {}", &w_id)) -} - -async fn get_default_scripts( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let mut tx = db.begin().await?; - let default_scripts = sqlx::query_scalar!( - "SELECT default_scripts FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_optional(&mut *tx) - .await - .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; - tx.commit().await?; - - Ok(Json(default_scripts.flatten())) -} - -#[cfg(feature = "enterprise")] -async fn edit_default_app( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json, -) -> Result { - #[cfg(not(feature = "enterprise"))] - { - return Err(Error::BadRequest( - "Setting a workspace default app is only available on Windmill Enterprise Edition" - .to_string(), - )); - } - - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - - let args_for_audit = format!("{:?}", new_config.default_app_path); - audit_log( - &mut *tx, - &authed, - "workspaces.edit_default_app", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("default_app", args_for_audit.as_str())].into()), - ) - .await?; - - if let Some(default_app_path) = new_config.default_app_path { - sqlx::query!( - "UPDATE workspace_settings SET default_app = $1 WHERE workspace_id = $2", - default_app_path, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET default_app = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - tx.commit().await?; - - // Trigger git sync for default app changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "default_app".to_string() }, - Some("Default app configuration updated".to_string()), - false, - None, - ) - .await?; - - Ok(format!("Edit default app for workspace {}", &w_id)) -} - -#[derive(Serialize)] -struct WorkspaceDefaultApp { - pub default_app_path: Option, -} -async fn get_default_app( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult { - let mut tx = db.begin().await?; - let default_app_path = sqlx::query_scalar!( - "SELECT default_app FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&mut *tx) - .await - .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; - tx.commit().await?; - - Ok(Json(WorkspaceDefaultApp { default_app_path })) -} - -async fn edit_error_handler( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(ee): Json, -) -> Result { - require_admin(is_admin, &username)?; - - // Normalize to new format (handles both old CLI and new CLI requests) - let ee = ee.into_normalized(); - - let mut tx = db.begin().await?; - - sqlx::query_as!( - Group, - "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", - w_id, - "error_handler", - "The group the error handler acts on behalf of", - serde_json::json!({username_to_permissioned_as(&authed.username): true}) - ) - .execute(&mut *tx) - .await?; - - if let Some(path) = &ee.path { - match ee.extra_args.as_ref() { - Some(extra_args) if extra_args.is_object() => { - let Ok(email_recipients) = serde_json::from_value::>>( - extra_args["email_recipients"].to_owned(), - ) else { - return Err(Error::BadRequest( - "Field `email_recipients` expected to be JSON array".to_string(), - )); - }; - - if let Some(email_recipients) = email_recipients { - for email in email_recipients { - if !EMAIL_REGEXP.is_match(&email) { - return Err(Error::BadRequest(format!( - "Invalid email format: {}", - email - ))); - } - } - } - } - None => {} - _ => { - return Err(Error::BadRequest( - "Field `extra_args` expected to be JSON object".to_string(), - )) - } - } - - let mut error_handler = serde_json::json!({ - "path": path, - }); - if let Some(extra_args) = &ee.extra_args { - error_handler["extra_args"] = extra_args.clone(); - } - if ee.muted_on_cancel { - error_handler["muted_on_cancel"] = serde_json::json!(true); - } - if ee.muted_on_user_path { - error_handler["muted_on_user_path"] = serde_json::json!(true); - } - - sqlx::query!( - "UPDATE workspace_settings SET error_handler = $1 WHERE workspace_id = $2", - error_handler, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET error_handler = NULL WHERE workspace_id = $1", - &w_id - ) - .execute(&mut *tx) - .await?; - } - - audit_log( - &mut *tx, - &authed, - "workspaces.edit_error_handler", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("error_handler", &format!("{:?}", ee.path)[..])].into()), - ) - .await?; - tx.commit().await?; - - // Trigger git sync for error handler changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "error_handler".to_string() }, - Some("Error handler configuration updated".to_string()), - false, - None, - ) - .await?; - - Ok(format!("Edit error_handler for workspace {}", &w_id)) -} - -async fn edit_success_handler( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(es): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - // Normalize to new format (handles both old CLI and new CLI requests) - let es = es.into_normalized(); - - let mut tx = db.begin().await?; - - sqlx::query_as!( - Group, - "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", - w_id, - "success_handler", - "The group the success handler acts on behalf of", - serde_json::json!({username_to_permissioned_as(&authed.username): true}) - ) - .execute(&mut *tx) - .await?; - - if let Some(path) = &es.path { - let mut success_handler = serde_json::json!({ - "path": path, - }); - if let Some(extra_args) = &es.extra_args { - success_handler["extra_args"] = extra_args.clone(); - } - - sqlx::query!( - "UPDATE workspace_settings SET success_handler = $1 WHERE workspace_id = $2", - success_handler, - &w_id - ) - .execute(&mut *tx) - .await?; - } else { - sqlx::query!( - "UPDATE workspace_settings SET success_handler = NULL WHERE workspace_id = $1", - &w_id - ) - .execute(&mut *tx) - .await?; - } - - audit_log( - &mut *tx, - &authed, - "workspaces.edit_success_handler", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("success_handler", &format!("{:?}", es.path)[..])].into()), - ) - .await?; - tx.commit().await?; - - // Trigger git sync for success handler changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "success_handler".to_string() }, - Some("Success handler configuration updated".to_string()), - false, - None, - ) - .await?; - - Ok(format!("Edit success_handler for workspace {}", &w_id)) -} - -#[derive(Deserialize)] -struct NewEnvironmentVariable { - name: String, - value: Option, -} - -async fn set_environment_variable( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(NewEnvironmentVariable { value, name }): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - - match value { - Some(value) => { - sqlx::query!( - "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = EXCLUDED.value", - &w_id, - name, - value - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspace.set_environment_variable", - ActionKind::Create, - &w_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Set environment variable {}", name)) - } - None => { - sqlx::query!( - "DELETE FROM workspace_env WHERE workspace_id = $1 AND name = $2", - &w_id, - name - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspace.delete_environment_variable", - ActionKind::Delete, - &w_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Deleted environment variable {}", name)) - } - } -} - -#[derive(Serialize)] -pub struct GetEncryptionKeyResponse { - key: String, -} - -async fn get_encryption_key( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult { - require_admin(authed.is_admin, &authed.username)?; - - let encryption_key_opt = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1", - w_id - ) - .fetch_optional(&db) - .await?; - - let encryption_key = not_found_if_none(encryption_key_opt, "workspace_encryption_key", w_id)?; - return Ok(Json(GetEncryptionKeyResponse { key: encryption_key })); -} - -#[derive(Deserialize)] -struct SetEncryptionKeyRequest { - new_key: String, - skip_reencrypt: Option, -} - -async fn set_encryption_key( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(request): Json, -) -> Result<()> { - require_super_admin(&db, &authed.email).await?; - - if !WORKSPACE_KEY_REGEXP.is_match(request.new_key.as_str()) { - return Err(Error::BadRequest( - "Encryption key should be an alphanumeric string of 64 characters".to_string(), - )); - } - - let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; - - sqlx::query!( - "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", - request.new_key.clone(), - w_id - ) - .execute(&db) - .await?; - - WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); - - if !request.skip_reencrypt.unwrap_or(false) { - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; - - let mut truncated_new_key = request.new_key.clone(); - truncated_new_key.truncate(8); - tracing::warn!( - "Re-encrypting all secrets for workspace {}. New key is {}***", - w_id, - truncated_new_key - ); - - let all_variables = sqlx::query!( - "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", - w_id - ) - .fetch_all(&db) - .await?; - - for variable in all_variables { - if !variable.is_secret { - continue; - } - let decrypted_value = - decrypt(&previous_encryption_key, variable.value).map_err(|e| { - Error::internal_err(format!( - "Error decrypting variable {}: {}", - variable.path, e - )) - })?; - let new_encrypted_value = encrypt(&new_encryption_key, decrypted_value.as_str()); - sqlx::query!( - "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3", - new_encrypted_value, - w_id, - variable.path - ) - .execute(&db) - .await?; - } - } - - // Trigger git sync for encryption key changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Key { key_type: "encryption_key".to_string() }, - Some("Encryption key updated".to_string()), - false, - None, - ) - .await?; - - return Ok(()); -} - -#[derive(Serialize)] -struct UsedTriggers { - pub websocket_used: bool, - pub http_routes_used: bool, - pub kafka_used: bool, - pub nats_used: bool, - pub postgres_used: bool, - pub mqtt_used: bool, - pub sqs_used: bool, - pub gcp_used: bool, - pub email_used: bool, - pub nextcloud_used: bool, -} - -async fn get_used_triggers( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let triggers_used = sqlx::query_as!( - UsedTriggers, - r#" - SELECT - EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS "websocket_used!", - EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS "http_routes_used!", - EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as "kafka_used!", - EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!", - EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!", - EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS "mqtt_used!", - EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!", - EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS "gcp_used!", - EXISTS(SELECT 1 FROM email_trigger WHERE workspace_id = $1) AS "email_used!", - EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS "nextcloud_used!" - "#, - w_id - ) - .fetch_one(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(triggers_used)) -} - -async fn get_workspace_as_superadmin( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult { - require_super_admin(&db, &authed.email).await?; - let workspace = sqlx::query_as!( - Workspace, - "SELECT - workspace.id AS \"id!\", - workspace.name AS \"name!\", - workspace.owner AS \"owner!\", - workspace.deleted AS \"deleted!\", - workspace.premium AS \"premium!\", - workspace_settings.color AS \"color\", - workspace.parent_workspace_id AS \"parent_workspace_id\" - FROM workspace - LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id - WHERE workspace.id = $1", - w_id - ) - .fetch_optional(&db) - .await?; - - let workspace = not_found_if_none(workspace, "workspace", w_id)?; - - Ok(Json(workspace)) -} - -async fn list_workspaces_as_super_admin( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Query(pagination): Query, - ApiAuthed { email, .. }: ApiAuthed, -) -> JsonResult> { - require_super_admin(&db, &email).await?; - let (per_page, offset) = paginate(pagination); - - let mut tx = user_db.begin(&authed).await?; - let workspaces = sqlx::query_as!( - Workspace, - "SELECT - workspace.id AS \"id!\", - workspace.name AS \"name!\", - workspace.owner AS \"owner!\", - workspace.deleted AS \"deleted!\", - workspace.premium AS \"premium!\", - workspace_settings.color AS \"color\", - workspace.parent_workspace_id AS \"parent_workspace_id\" - FROM workspace - LEFT JOIN workspace_settings ON workspace.id = workspace_settings.workspace_id - LIMIT $1 OFFSET $2", - per_page as i32, - offset as i32 - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(workspaces)) -} - -async fn user_workspaces( - Extension(db): Extension, - ApiAuthed { email, .. }: ApiAuthed, -) -> JsonResult { - let mut tx = db.begin().await?; - let workspaces = sqlx::query_as!( - UserWorkspace, - "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, - CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, - usr.disabled - FROM workspace - JOIN usr ON usr.workspace_id = workspace.id - JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id - WHERE usr.email = $1 AND workspace.deleted = false", - email - ) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - Ok(Json(WorkspaceList { email, workspaces })) -} - -pub async fn check_w_id_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> { - if w_id == "global" { - return Err(windmill_common::error::Error::BadRequest( - "'global' is not allowed as a workspace ID".to_string(), - )); - } - let exists = sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)", w_id) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(windmill_common::error::Error::BadRequest(format!( - "Workspace {} already exists", - w_id - ))); - } - return Ok(()); -} - -lazy_static::lazy_static! { - - pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = { - match std::env::var("CREATE_WORKSPACE_REQUIRE_SUPERADMIN") { - Ok(val) => val == "true", - Err(_) => true, - } - }; - - pub static ref DISABLE_WORKSPACE_FORK: bool = { - match std::env::var("DISABLE_WORKSPACE_FORK") { - Ok(val) => val == "true", - Err(_) => false, - } - }; - -} - -async fn create_workspace_require_superadmin() -> String { - format!("{}", *CREATE_WORKSPACE_REQUIRE_SUPERADMIN) -} - -async fn _check_nb_of_workspaces(db: &DB) -> Result<()> { - let nb_workspaces = sqlx::query_scalar!( - "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = false", - ) - .fetch_one(db) - .await?; - if nb_workspaces.unwrap_or(0) >= 2 { - return Err(Error::BadRequest( - "You have reached the maximum number of workspaces (2 outside of default workspace 'admins') without an enterprise license. Archive/delete another workspace to create a new one" - .to_string(), - )); - } - return Ok(()); -} - -async fn create_workspace( - authed: ApiAuthed, - Extension(db): Extension, - Json(nw): Json, -) -> Result { - if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN { - require_super_admin(&db, &authed.email).await?; - } - - #[cfg(not(feature = "enterprise"))] - _check_nb_of_workspaces(&db).await?; - - if *CLOUD_HOSTED { - let nb_workspaces = sqlx::query_scalar!( - "SELECT COUNT(*) FROM workspace WHERE owner = $1", - authed.email - ) - .fetch_one(&db) - .await?; - if nb_workspaces.unwrap_or(0) >= 10 { - return Err(Error::BadRequest( - "You have reached the maximum number of workspaces (10) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - - check_w_id_conflict(&mut tx, &nw.id).await?; - sqlx::query!( - "INSERT INTO workspace - (id, name, owner) - VALUES ($1, $2, $3)", - nw.id, - nw.name, - authed.email, - ) - .execute(&mut *tx) - .await?; - sqlx::query!( - "INSERT INTO workspace_settings - (workspace_id, color) - VALUES ($1, $2)", - nw.id, - nw.color, - ) - .execute(&mut *tx) - .await?; - let key = rd_string(64); - sqlx::query!( - "INSERT INTO workspace_key - (workspace_id, kind, key) - VALUES ($1, 'cloud', $2)", - nw.id, - &key - ) - .execute(&mut *tx) - .await?; - - // let mc = magic_crypt::new_magic_crypt!(key, 256); - // sqlx::query!( - // "INSERT INTO variable - // (workspace_id, path, value, is_secret, description) - // VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'), - // ($3, 'g/all/not_secret', $4, false, 'This item is not secret')", - // nw.id, - // crate::variables::encrypt(&mc, "pretty secret value"), - // nw.id, - // "finland does not actually exist", - // ) - // .execute(&mut *tx) - // .await?; - - let automate_username_creation = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = $1", - AUTOMATE_USERNAME_CREATION_SETTING, - ) - .fetch_optional(&mut *tx) - .await? - .map(|v| v.as_bool()) - .flatten() - .unwrap_or(false); - - let username = if automate_username_creation { - if nw.username.is_some() && nw.username.unwrap().len() > 0 { - return Err(Error::BadRequest( - "username is not allowed when username creation is automated".to_string(), - )); - } - get_instance_username_or_create_pending(&mut tx, &authed.email).await? - } else { - nw.username - .ok_or(Error::BadRequest("username is required".to_string()))? - }; - - sqlx::query!( - "INSERT INTO usr - (workspace_id, email, username, is_admin) - VALUES ($1, $2, $3, true)", - nw.id, - authed.email, - username, - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO group_ - VALUES ($1, 'all', 'The group that always contains all users of this workspace')", - nw.id - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO usr_to_group - VALUES ($1, 'all', $2)", - nw.id, - username - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.create", - ActionKind::Create, - &nw.id, - Some(nw.name.as_str()), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("Created workspace {}", &nw.id)) -} - -async fn clone_workspace_data( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - // Clone workspace settings (merge with existing basic settings) - update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; - - // Clone workspace environment variables - clone_workspace_env(tx, source_workspace_id, target_workspace_id).await?; - - // Clone folders - clone_folders(tx, source_workspace_id, target_workspace_id).await?; - - // Clone groups - clone_groups(tx, source_workspace_id, target_workspace_id).await?; - - // Clone resource types - clone_resource_types(tx, source_workspace_id, target_workspace_id).await?; - - // Clone resources - clone_resources(tx, source_workspace_id, target_workspace_id).await?; - - // Clone variables with re-encryption - clone_variables(tx, source_workspace_id, target_workspace_id).await?; - - // Clone scripts with new hashes - clone_scripts(tx, source_workspace_id, target_workspace_id).await?; - - // Clone flows with new versions - clone_flows(tx, source_workspace_id, target_workspace_id).await?; - - // Clone flow nodes - clone_flow_nodes(tx, source_workspace_id, target_workspace_id).await?; - - // Clone apps with new IDs and app scripts - let _app_id_mapping = clone_apps(tx, source_workspace_id, target_workspace_id).await?; - - // Clone raw apps - clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?; - - // Clone workspace runnable dependencies and dependency map - clone_workspace_runnable_dependencies(tx, source_workspace_id, target_workspace_id).await?; - - // TODO: Enable when git sync is implemented for workspace dependencies. - // // Clone workspace dependencies - // clone_workspace_dependencies(tx, source_workspace_id, target_workspace_id).await?; - Ok(()) -} - -async fn update_workspace_settings( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO workspace_key (workspace_id, kind, key) - SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - sqlx::query!( - r#" - UPDATE workspace_settings - SET - deploy_to = $1, - ai_config = source_ws.ai_config, - large_file_storage = source_ws.large_file_storage, - ducklake = source_ws.ducklake, - datatable = source_ws.datatable, - git_app_installations = source_ws.git_app_installations - FROM workspace_settings source_ws - WHERE source_ws.workspace_id = $1 - AND workspace_settings.workspace_id = $2 - "#, - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - let current_git_sync_settings = sqlx::query!( - "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", - source_workspace_id - ) - .fetch_optional(&mut **tx) - .await?; - - let mut git_sync_settings = if let Some(row) = current_git_sync_settings { - if let Some(git_sync) = row.git_sync { - serde_json::from_value::(git_sync) - .map_err(|err| Error::internal_err(err.to_string()))? - } else { - WorkspaceGitSyncSettings::default() - } - } else { - WorkspaceGitSyncSettings::default() - }; - - // We only keep the first git sync repo that is sync mode (use_individual_branch = false), since it is considered the main one - // Context: see WIN-1559 - git_sync_settings.repositories = git_sync_settings - .repositories - .into_iter() - .filter(|r| !r.use_individual_branch.unwrap_or(false)) - .take(1) - .collect(); - - let serialized_config = serde_json::to_value::(git_sync_settings) - .map_err(|err| Error::internal_err(err.to_string()))?; - - sqlx::query!( - "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", - serialized_config, - target_workspace_id - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_workspace_env( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO workspace_env (workspace_id, name, value) - SELECT $2, name, value - FROM workspace_env - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_folders( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, edited_at, created_by) - SELECT $2, name, display_name, owners, extra_perms, summary, edited_at, created_by - FROM folder - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_groups( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO group_ (workspace_id, name, summary, extra_perms) - SELECT $2, name, summary, extra_perms - FROM group_ - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - sqlx::query!( - "INSERT INTO usr_to_group (workspace_id, group_, usr) - SELECT $2, group_, usr - FROM usr_to_group - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_resource_types( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO resource_type (workspace_id, name, schema, description, edited_at, created_by, format_extension) - SELECT $2, name, schema, description, edited_at, created_by, format_extension - FROM resource_type - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_resources( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, edited_at, created_by) - SELECT $2, path, value, description, resource_type, extra_perms, edited_at, created_by - FROM resource - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_variables( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) - SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at - FROM variable - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_scripts( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - // Clone all scripts directly with a single query - sqlx::query!( - r#"INSERT INTO script ( - workspace_id, hash, path, parent_hashes, summary, description, content, - created_by, created_at, archived, schema, deleted, is_template, - extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, - envs, concurrent_limit, concurrency_time_window_s, cache_ttl, - dedicated_worker, ws_error_handler_muted, priority, timeout, - delete_after_use, restart_unless_cancelled, concurrency_key, - visible_to_runner_only, no_main_func, codebase, has_preprocessor, - on_behalf_of_email, assets - ) - SELECT - $1, hash, path, parent_hashes, summary, description, content, - created_by, created_at, archived, schema, deleted, is_template, - extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, - envs, concurrent_limit, concurrency_time_window_s, cache_ttl, - dedicated_worker, ws_error_handler_muted, priority, timeout, - delete_after_use, restart_unless_cancelled, concurrency_key, - visible_to_runner_only, no_main_func, codebase, has_preprocessor, - on_behalf_of_email, assets - FROM script - WHERE workspace_id = $2"#, - target_workspace_id, - source_workspace_id - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_flows( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - // First, clone flows without versions - sqlx::query!( - "INSERT INTO flow ( - workspace_id, path, summary, description, value, edited_by, edited_at, - archived, schema, extra_perms, dependency_job, draft_only, tag, - ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, - concurrency_key, versions, on_behalf_of_email, lock_error_logs - ) - SELECT $2, path, summary, description, value, edited_by, edited_at, - archived, schema, extra_perms, NULL, draft_only, tag, - ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, - concurrency_key, ARRAY[]::bigint[], on_behalf_of_email, lock_error_logs - FROM flow - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - // Then clone flow versions - let flow_versions = sqlx::query!( - "SELECT id, workspace_id, path, value, schema, created_by, created_at - FROM flow_version - WHERE workspace_id = $1 - ORDER BY path, created_at", - source_workspace_id - ) - .fetch_all(&mut **tx) - .await?; - - for version in flow_versions { - let new_version_id = sqlx::query_scalar!( - "INSERT INTO flow_version (workspace_id, path, value, schema, created_by, created_at) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id", - target_workspace_id, - version.path, - version.value, - version.schema, - version.created_by, - version.created_at, - ) - .fetch_one(&mut **tx) - .await?; - - // Update flow to include this version - sqlx::query!( - "UPDATE flow - SET versions = array_append(versions, $1) - WHERE workspace_id = $2 AND path = $3", - new_version_id, - target_workspace_id, - version.path, - ) - .execute(&mut **tx) - .await?; - } - - Ok(()) -} - -async fn clone_flow_nodes( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO flow_node (workspace_id, hash, path, lock, code, flow, hash_v2) - SELECT $2, - (SELECT COALESCE(MAX(hash), 0) FROM flow_node) + row_number() OVER () AS new_hash, - source_fn.path, source_fn.lock, source_fn.code, source_fn.flow, source_fn.hash_v2 - FROM flow_node source_fn - WHERE source_fn.workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_apps( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result> { - // Get all apps from source workspace - let apps = sqlx::query!( - "SELECT id, workspace_id, path, summary, policy, versions, extra_perms, draft_only, custom_path - FROM app - WHERE workspace_id = $1", - source_workspace_id - ) - .fetch_all(&mut **tx) - .await?; - - let mut app_id_mapping: HashMap = HashMap::new(); - - // Clone apps with new IDs - for app in apps { - let new_app_id = sqlx::query_scalar!( - "INSERT INTO app (workspace_id, path, summary, policy, versions, extra_perms, draft_only, custom_path) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - RETURNING id", - target_workspace_id, - app.path, - app.summary, - app.policy, - &Vec::::new(), // Start with empty versions array - app.extra_perms, - app.draft_only, - app.custom_path, - ) - .fetch_one(&mut **tx) - .await?; - - app_id_mapping.insert(app.id, new_app_id); - } - - { - // Clone app versions - let app_versions = sqlx::query!( - "SELECT app_id, value, created_by, created_at, raw_app - FROM app_version - WHERE app_id = ANY(SELECT id FROM app WHERE workspace_id = $1) - ORDER BY app_id, created_at", - source_workspace_id - ) - .fetch_all(&mut **tx) - .await?; - - for version in app_versions { - if let Some(&new_app_id) = app_id_mapping.get(&version.app_id) { - sqlx::query!( - "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) - VALUES ($1, $2, $3, $4, $5)", - new_app_id, - version.value, - version.created_by, - version.created_at, - version.raw_app, - ) - .execute(&mut **tx) - .await?; - } - } - } - - // Update app versions arrays - sqlx::query!( - "UPDATE app SET versions = ( - SELECT array_agg(av.id ORDER BY av.created_at) - FROM app_version av - WHERE av.app_id = app.id - ) WHERE workspace_id = $1", - target_workspace_id - ) - .execute(&mut **tx) - .await?; - - // Clone app scripts with recomputed hashes - let app_scripts = sqlx::query!( - "SELECT app, hash, lock, code, code_sha256 - FROM app_script - WHERE app = ANY(SELECT id FROM app WHERE workspace_id = $1)", - source_workspace_id - ) - .fetch_all(&mut **tx) - .await?; - - for app_script in app_scripts { - if let Some(&new_app_id) = app_id_mapping.get(&app_script.app) { - // Recompute hash using app_id, code_sha256, and lock - let mut hasher = Sha256::new(); - hasher.update(new_app_id.to_be_bytes()); - hasher.update(hex::decode(&app_script.code_sha256)?); - if let Some(lock) = &app_script.lock { - hasher.update(lock.as_bytes()); - } - let new_hash = hex::encode(hasher.finalize()); - - sqlx::query!( - "INSERT INTO app_script (app, hash, lock, code, code_sha256) - VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", - new_app_id, - new_hash, - app_script.lock, - app_script.code, - app_script.code_sha256, - ) - .execute(&mut **tx) - .await?; - } - } - - Ok(app_id_mapping) -} - -async fn clone_raw_apps( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - sqlx::query!( - "INSERT INTO raw_app (path, version, workspace_id, summary, edited_at, data, extra_perms) - SELECT path, version, $2, summary, edited_at, data, extra_perms - FROM raw_app - WHERE workspace_id = $1", - source_workspace_id, - target_workspace_id, - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn clone_workspace_runnable_dependencies( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - // Clone workspace_runnable_dependencies - sqlx::query!( - "INSERT INTO workspace_runnable_dependencies (flow_path, runnable_path, script_hash, runnable_is_flow, workspace_id, app_path) - SELECT flow_path, runnable_path, script_hash, runnable_is_flow, $1, app_path - FROM workspace_runnable_dependencies - WHERE workspace_id = $2", - target_workspace_id, - source_workspace_id - ) - .execute(&mut **tx) - .await?; - - // Clone dependency_map to preserve import relationships - sqlx::query!( - "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) - SELECT $1, importer_path, importer_kind, imported_path, importer_node_id - FROM dependency_map - WHERE workspace_id = $2", - target_workspace_id, - source_workspace_id - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -#[allow(dead_code)] -async fn clone_workspace_dependencies( - tx: &mut Transaction<'_, Postgres>, - source_workspace_id: &str, - target_workspace_id: &str, -) -> Result<()> { - // Clone workspace_runnable_dependencies - sqlx::query!( - "INSERT INTO workspace_dependencies (workspace_id, language, name, description, content, archived, created_at) - SELECT $1, language, name, description, content, archived, created_at - FROM workspace_dependencies - WHERE workspace_id = $2", - target_workspace_id, - source_workspace_id - ) - .execute(&mut **tx) - .await?; - - Ok(()) -} - -async fn deprecated_create_workspace_fork(_authed: ApiAuthed) -> Result { - return Err(Error::BadRequest("This API endpoint has been relocated. Your Windmill CLI version is outdated and needs to be updated.".to_string())); -} - -/// Return the uuids of the git sync jobs to create the branch before creating the fork -async fn create_workspace_fork_branch( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(nw): Json, -) -> JsonResult> { - if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); - } - - if *DISABLE_WORKSPACE_FORK { - require_super_admin(&db, &authed.email).await?; - } - - Ok(Json( - handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, - )) -} - -async fn create_workspace_fork( - authed: ApiAuthed, - Extension(db): Extension, - Path(parent_workspace_id): Path, - Json(nw): Json, -) -> Result { - if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); - } - - if *DISABLE_WORKSPACE_FORK { - require_super_admin(&db, &authed.email).await?; - } - - let mut tx: Transaction<'_, Postgres> = db.begin().await?; - - // Generate unique forked workspace ID with wm-fork prefix - if !nw.id.starts_with(WM_FORK_PREFIX) { - return Err(Error::BadRequest(format!( - "The id `{}` is invalid for a forked workspace. It should be prefixed by {}", - nw.id, WM_FORK_PREFIX - ))); - } - - let forked_id = nw.id; - - sqlx::query!( - "INSERT INTO workspace - (id, name, owner, parent_workspace_id) - VALUES ($1, $2, $3, $4)", - forked_id, - nw.name, - authed.email, - parent_workspace_id, - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO workspace_settings - (workspace_id, color) - VALUES ($1, $2)", - forked_id, - nw.color, - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO usr - (workspace_id, email, username, is_admin) - SELECT $1, email, username, is_admin FROM usr - WHERE workspace_id = $3 AND email = $2 - ", - forked_id, - authed.email, - parent_workspace_id, - ) - .execute(&mut *tx) - .await?; - - // Clone all data from the parent workspace using Rust implementation - clone_workspace_data(&mut tx, &parent_workspace_id, &forked_id).await?; - - sqlx::query!( - "INSERT INTO workspace_invite (workspace_id, email, is_admin, operator) - SELECT $1, email, is_admin, operator - FROM usr - WHERE workspace_id = $2", - &forked_id, - &parent_workspace_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.create_fork", - ActionKind::Create, - &forked_id, - Some(nw.name.as_str()), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("Created forked workspace {}", &forked_id)) -} - -async fn edit_workspace( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(ew): Json, -) -> Result { - require_admin(is_admin, &username)?; - let mut tx = db.begin().await?; - sqlx::query!( - "UPDATE workspace SET name = $1, owner = $2 WHERE id = $3", - ew.name, - ew.owner, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.update", - ActionKind::Update, - &w_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Updated workspace {}", &w_id)) -} - -/// Archive a workspace: disable schedules, cancel jobs, and mark as deleted. -/// Returns (schedules_disabled_count, jobs_canceled_count). -pub(crate) async fn archive_workspace_impl( - db: &DB, - w_id: &str, - username: &str, -) -> Result<(usize, usize)> { - // Step 1: Disable all schedules and clear their queued jobs - let mut tx = db.begin().await?; - let disabled_schedules = sqlx::query_scalar!( - "UPDATE schedule SET enabled = false WHERE workspace_id = $1 AND enabled = true RETURNING path", - w_id - ) - .fetch_all(&mut *tx) - .await?; - - let schedules_count = disabled_schedules.len(); - tracing::info!( - "Disabled {} schedules in workspace {}", - schedules_count, - w_id - ); - - // Clear all schedule-related jobs using the existing clear_schedule function - for schedule_path in &disabled_schedules { - crate::schedule::clear_schedule(&mut tx, schedule_path, w_id).await?; - } - - // Mark workspace as archived - sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", w_id) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - // Step 2: Get all remaining queued jobs for this workspace (non-schedule jobs) - let jobs_to_cancel = - sqlx::query_scalar!("SELECT id FROM v2_job_queue WHERE workspace_id = $1", w_id) - .fetch_all(db) - .await?; - - let jobs_count = jobs_to_cancel.len(); - tracing::info!( - "Found {} remaining jobs to cancel in workspace {}", - jobs_count, - w_id - ); - - // Step 3: Cancel all remaining jobs using the existing cancel_jobs function - let canceled_count = if !jobs_to_cancel.is_empty() { - let axum::Json(canceled_jobs) = crate::jobs::cancel_jobs( - jobs_to_cancel, - db, - username, - w_id, - false, // force_cancel - ) - .await?; - - let count = canceled_jobs.len(); - tracing::info!("Canceled {} jobs in workspace {}", count, w_id); - count - } else { - 0 - }; - - Ok((schedules_count, canceled_count)) -} - -async fn archive_workspace( - Extension(db): Extension, - Path(w_id): Path, - authed: ApiAuthed, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let (schedules_count, canceled_count) = - archive_workspace_impl(&db, &w_id, &authed.username).await?; - - // Audit log - let mut tx = db.begin().await?; - let mut audit_params = HashMap::new(); - audit_params.insert("disabled_schedules", schedules_count.to_string()); - audit_params.insert("canceled_jobs", canceled_count.to_string()); - let audit_params_refs: HashMap<&str, &str> = - audit_params.iter().map(|(k, v)| (*k, v.as_str())).collect(); - - audit_log( - &mut *tx, - &authed, - "workspaces.archive", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some(audit_params_refs), - ) - .await?; - tx.commit().await?; - - Ok(format!( - "Archived workspace {}, disabled {} schedules and canceled {} jobs", - &w_id, schedules_count, canceled_count - )) -} - -async fn leave_workspace( - Extension(db): Extension, - Path(w_id): Path, - authed: ApiAuthed, -) -> Result { - let mut tx = db.begin().await?; - sqlx::query!( - "DELETE FROM usr WHERE workspace_id = $1 AND email = $2", - &w_id, - &authed.email - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.leave", - ActionKind::Delete, - &w_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Left workspace {}", &w_id)) -} - -async fn unarchive_workspace( - Extension(db): Extension, - Path(w_id): Path, - authed: ApiAuthed, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - let mut tx = db.begin().await?; - sqlx::query!("UPDATE workspace SET deleted = false WHERE id = $1", &w_id) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspaces.unarchive", - ActionKind::Update, - &w_id, - Some(&authed.email), - None, - ) - .await?; - tx.commit().await?; - - Ok(format!("Unarchived workspace {}", &w_id)) -} - -async fn invite_user( - ApiAuthed { username, is_admin, .. }: ApiAuthed, - Extension(db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(mut nu): Json, -) -> Result<(StatusCode, String)> { - require_admin(is_admin, &username)?; - - nu.email = nu.email.to_lowercase(); - - let mut tx = db.begin().await?; - - let already_in_workspace = sqlx::query_scalar!( - "SELECT EXISTS (SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", - &w_id, - nu.email - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if already_in_workspace { - return Err(Error::BadRequest(format!( - "user with email {} already exists in workspace {}", - nu.email, w_id - ))); - } - - sqlx::query!( - "INSERT INTO workspace_invite - (workspace_id, email, is_admin, operator) - VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, email) - DO UPDATE SET is_admin = EXCLUDED.is_admin, operator = EXCLUDED.operator", - &w_id, - nu.email, - nu.is_admin, - nu.operator - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - send_email_if_possible( - &format!("Invited to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} - -If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - BASE_URL.read().await.clone() - ), - &nu.email, - ); - - webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace { - email: nu.email.clone(), - workspace: w_id, - }); - - Ok(( - StatusCode::CREATED, - format!("user with email {} invited", nu.email), - )) -} - -async fn add_user( - authed: ApiAuthed, - Extension(db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(mut nu): Json, -) -> Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - nu.email = nu.email.to_lowercase(); - - let mut tx = db.begin().await?; - - let already_exists_email = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", - &w_id, - nu.email, - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - if already_exists_email { - return Err(Error::BadRequest(format!( - "user with email {} already exists in workspace {}", - nu.email, w_id - ))); - } - - let automate_username_creation = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = $1", - AUTOMATE_USERNAME_CREATION_SETTING, - ) - .fetch_optional(&mut *tx) - .await? - .map(|v| v.as_bool()) - .flatten() - .unwrap_or(false); - - let username = if automate_username_creation { - if nu.username.is_some() && nu.username.unwrap().len() > 0 { - return Err(Error::BadRequest( - "username is not allowed when username creation is automated".to_string(), - )); - } - get_instance_username_or_create_pending(&mut tx, &nu.email).await? - } else { - let username = nu - .username - .ok_or(Error::BadRequest("username is required".to_string()))?; - - if !VALID_USERNAME.is_match(&username) { - return Err(windmill_common::error::Error::BadRequest(format!( - "Usermame can only contain alphanumeric characters and underscores and must start with a letter" - ))); - } - - username - }; - - sqlx::query!( - "INSERT INTO usr - (workspace_id, email, username, is_admin, operator) - VALUES ($1, $2, $3, $4, $5)", - &w_id, - nu.email, - username, - nu.is_admin, - nu.operator - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2", - &w_id, - nu.email - ) - .execute(&mut *tx) - .await?; - - sqlx::query_as!( - Group, - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - &w_id, - username, - "all", - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "users.add_to_workspace", - ActionKind::Create, - &w_id, - Some(&nu.email), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::User { email: nu.email.clone() }, - Some(format!("Added user '{}' to workspace", &nu.email)), - true, - None, - ) - .await?; - - send_email_if_possible( - &format!("Added to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} by {} - -If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - authed.email, - BASE_URL.read().await.clone() - ), - &nu.email, - ); - - webhook.send_instance_event(InstanceEvent::UserAddedWorkspace { - workspace: w_id.clone(), - email: nu.email.clone(), - }); - - Ok(( - StatusCode::CREATED, - format!("user with email {} added", nu.email), - )) -} - -async fn delete_invite( - ApiAuthed { username, is_admin, .. }: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(nu): Json, -) -> Result<(StatusCode, String)> { - require_admin(is_admin, &username)?; - - let mut tx = db.begin().await?; - - sqlx::query!( - "DELETE FROM workspace_invite WHERE - workspace_id = $1 AND email = $2 AND is_admin = $3 AND operator = $4", - &w_id, - nu.email, - nu.is_admin, - nu.operator - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(( - StatusCode::CREATED, - format!("invite to email {} deleted", nu.email), - )) -} - -async fn exists_username( - Extension(db): Extension, - Json(vu): Json, -) -> Result { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)", - vu.username, - vu.id - ) - .fetch_one(&db) - .await? - .unwrap_or(true); - - if exists { - return Err(Error::BadRequest("username already taken".to_string())); - } - - Ok("valid username".to_string()) -} - -async fn get_workspace_name( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - let workspace = sqlx::query_scalar!("SELECT name FROM workspace WHERE id = $1", &w_id) - .fetch_one(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(workspace) -} - -async fn get_dependency_map( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - let dmap = sqlx::query_as!( - DependencyMap, - " - SELECT workspace_id, importer_path, importer_kind::text, imported_path, importer_node_id - FROM dependency_map WHERE workspace_id = $1", - &w_id - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(dmap)) -} - -#[axum::debug_handler] -async fn rebuild_dependency_map( - Extension(db): Extension, - Path(w_id): Path, - authed: ApiAuthed, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - if *CLOUD_HOSTED { - return Err(Error::BadRequest("Disabled on Cloud".into())); - } - ScopedDependencyMap::rebuild_map(&w_id, &db).await -} - -#[axum::debug_handler] -async fn get_dependents( - Extension(db): Extension, - Path((w_id, imported_path)): Path<(String, String)>, - _authed: ApiAuthed, -) -> JsonResult> { - tracing::debug!( - workspace_id = %w_id, - imported_path = %imported_path, - "API: Getting dependents for imported path" - ); - - let dependents = ScopedDependencyMap::get_dependents(&imported_path, &w_id, &db).await?; - - tracing::debug!( - workspace_id = %w_id, - imported_path = %imported_path, - dependents_count = dependents.len(), - "API: Found dependents: {:?}", - dependents - ); - - Ok(Json(dependents)) -} - -#[derive(Serialize, Debug)] -struct DependentsAmount { - imported_path: String, - count: i64, -} - -#[axum::debug_handler] -async fn get_dependents_amounts( - Extension(db): Extension, - Path(w_id): Path, - Json(imported_paths): Json>, -) -> JsonResult> { - tracing::debug!( - workspace_id = %w_id, - imported_paths = ?imported_paths, - "API: Getting dependents amounts for imported paths" - ); - - let results = sqlx::query_as!( - DependentsAmount, - r#" - SELECT - imported_path, - COUNT(DISTINCT importer_path) as "count!" - FROM dependency_map - WHERE workspace_id = $1 AND imported_path = ANY($2) - GROUP BY imported_path - "#, - w_id, - &imported_paths - ) - .fetch_all(&db) - .await?; - - tracing::debug!( - workspace_id = %w_id, - results_count = results.len(), - "API: Found dependents amounts: {:?}", - results - ); - - Ok(Json(results)) -} - -#[derive(Deserialize)] -struct ChangeWorkspaceName { - new_name: String, -} - -#[derive(Deserialize)] -struct ChangeWorkspaceColor { - color: Option, -} - -async fn change_workspace_name( - authed: ApiAuthed, - Path(w_id): Path, - Extension(db): Extension, - Json(rw): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - - sqlx::query!( - "UPDATE workspace SET name = $1 WHERE id = $2", - &rw.new_name, - &w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "workspace.change_workspace_name", - ActionKind::Update, - &w_id, - Some(&authed.email), - None, - ) - .await?; - - tx.commit().await?; - - // Trigger git sync for workspace name changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { setting_type: "workspace_name".to_string() }, - Some(format!("Workspace name updated to {}", &rw.new_name)), - false, - None, - ) - .await?; - - Ok(format!("updated workspace name to {}", &rw.new_name)) -} - -async fn change_workspace_color( - authed: ApiAuthed, - Path(w_id): Path, - Extension(db): Extension, - Json(rw): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - - sqlx::query!( - "UPDATE workspace_settings SET color = $1 WHERE workspace_id = $2", - rw.color, - &w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Settings { setting_type: "workspace_color".to_string() }, - None, - false, - None, - ) - .await?; - - Ok(format!( - "updated workspace color to {}", - rw.color.as_deref().unwrap_or("no color") - )) -} - -async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Result { - let usage = sqlx::query_scalar!( - " - SELECT usage.usage FROM usage - WHERE is_workspace = true - AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) - AND id = $1", - w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(0); - Ok(usage.to_string()) -} - #[cfg(feature = "enterprise")] pub async fn get_critical_alerts( Extension(db): Extension, @@ -4404,887 +276,3 @@ async fn mute_critical_alerts( pub async fn mute_critical_alerts() -> Error { Error::NotFound("Critical Alerts require EE".to_string()) } - -#[derive(Deserialize)] -pub struct EditPublicAppRateLimitRequest { - pub public_app_execution_limit_per_minute: Option, -} - -async fn edit_public_app_rate_limit( - Extension(db): Extension, - Path(w_id): Path, - authed: ApiAuthed, - Json(req): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - sqlx::query!( - "UPDATE workspace_settings SET public_app_execution_limit_per_minute = $1 WHERE workspace_id = $2", - req.public_app_execution_limit_per_minute, - &w_id - ) - .execute(&db) - .await?; - - // Cache is invalidated via DB trigger -> notify_event -> polling in main.rs - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Settings { setting_type: "public_app_rate_limit".to_string() }, - None, - false, - None, - ) - .await?; - - Ok(format!( - "Updated public app rate limit for workspace: {}", - &w_id - )) -} - -// 5 minutes fallback TTL (in addition to event-based invalidation) -const PUBLIC_APP_RATE_LIMIT_CACHE_TTL_SECS: i64 = 300; - -pub async fn get_public_app_rate_limit(db: &DB, w_id: &str) -> Result> { - use windmill_common::workspaces::PUBLIC_APP_RATE_LIMIT_CACHE; - - let now = Utc::now().timestamp(); - - if let Some((rate_limit, cached_at)) = PUBLIC_APP_RATE_LIMIT_CACHE.get(w_id) { - if now - cached_at < PUBLIC_APP_RATE_LIMIT_CACHE_TTL_SECS { - return Ok(rate_limit); - } - } - - let result: Option> = sqlx::query_scalar( - "SELECT public_app_execution_limit_per_minute FROM workspace_settings WHERE workspace_id = $1", - ) - .bind(w_id) - .fetch_optional(db) - .await?; - let rate_limit = result.flatten(); - PUBLIC_APP_RATE_LIMIT_CACHE.insert(w_id.to_string(), (rate_limit, now)); - Ok(rate_limit) -} - -#[derive(Deserialize, Serialize)] -struct ChangeOperatorSettings { - #[serde(default)] - runs: bool, - #[serde(default)] - schedules: bool, - #[serde(default)] - resources: bool, - #[serde(default)] - variables: bool, - #[serde(default)] - assets: bool, - #[serde(default)] - triggers: bool, - #[serde(default)] - audit_logs: bool, - #[serde(default)] - groups: bool, - #[serde(default)] - folders: bool, - #[serde(default)] - workers: bool, -} - -async fn update_operator_settings( - authed: ApiAuthed, - Path(w_id): Path, - Extension(db): Extension, - Json(settings): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = db.begin().await?; - - let settings_json = serde_json::json!(settings); - - sqlx::query!( - "UPDATE workspace_settings SET operator_settings = $1 WHERE workspace_id = $2", - settings_json, - &w_id - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - // Trigger git sync for operator settings changes - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::Settings { - setting_type: "operator_settings".to_string(), - }, - Some("Operator settings updated".to_string()), - false, - None, - ) - .await?; - - Ok("Operator settings updated successfully".to_string()) -} - -#[derive(Serialize)] -pub struct WorkspaceComparison { - pub all_ahead_items_visible: bool, - pub all_behind_items_visible: bool, - pub skipped_comparison: bool, - pub diffs: Vec, - pub summary: CompareSummary, -} - -#[derive(Serialize, Default)] -pub struct CompareSummary { - pub total_diffs: usize, - pub total_ahead: usize, - pub total_behind: usize, - pub scripts_changed: usize, - pub flows_changed: usize, - pub apps_changed: usize, - pub resources_changed: usize, - pub variables_changed: usize, - pub resource_types_changed: usize, - pub folders_changed: usize, - pub conflicts: usize, // Items that are both ahead and behind -} - -async fn reset_workspace_diffs( - authed: ApiAuthed, - Path((w_id, target_workspace_id)): Path<(String, String)>, - Extension(db): Extension, - Extension(user_db): Extension, -) -> JsonResult<()> { - // Needed to compute the has_changes: Option. Otherwise it will be None, and the query will not hit the items - let _ = compare_workspaces( - authed, - Path((w_id.clone(), target_workspace_id.clone())), - Extension(db.clone()), - Extension(user_db), - ) - .await?; - - sqlx::query!( - "DELETE FROM workspace_diff WHERE has_changes = false AND ( - (source_workspace_id = $1 AND fork_workspace_id = $2) - OR (source_workspace_id = $2 AND fork_workspace_id =$1) - )", - target_workspace_id, - w_id, - ) - .execute(&db) - .await?; - - Ok(Json(())) -} - -#[derive(Serialize, Debug, Clone, Default)] -pub struct WorkspaceDiffRow { - kind: String, - path: String, - ahead: i32, - behind: i32, - has_changes: Option, - exists_in_source: Option, - exists_in_fork: Option, -} - -async fn compare_workspaces( - authed: ApiAuthed, - Path((source_workspace_id, fork_workspace_id)): Path<(String, String)>, - Extension(db): Extension, - Extension(user_db): Extension, -) -> JsonResult { - // require_admin(authed.is_admin, &authed.username)?; - - let skipped_comparison: bool = sqlx::query_scalar( - "SELECT EXISTS( - SELECT 1 FROM skip_workspace_diff_tally - WHERE workspace_id = $1 - )", - ) - .bind(&fork_workspace_id) - .fetch_one(&db) - .await?; - - if skipped_comparison { - return Ok(Json(WorkspaceComparison { - all_ahead_items_visible: true, - all_behind_items_visible: true, - skipped_comparison, - diffs: vec![], - summary: Default::default(), - })); - } - - let diff_items = sqlx::query_as!( - WorkspaceDiffRow, - "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff - WHERE source_workspace_id = $1 AND fork_workspace_id = $2", - source_workspace_id, - fork_workspace_id, - ) - .fetch_all(&db) - .await?; - - let mut confirmed_diffs = vec![]; - for item in diff_items { - if let Some(has_changes) = item.has_changes { - if has_changes { - confirmed_diffs.push(item); - } - continue; - } - - let item_comparison = match item.kind.as_str() { - "script" => Some( - compare_two_scripts(&db, &source_workspace_id, &fork_workspace_id, &item.path) - .await?, - ), - "flow" => Some( - compare_two_flows(&db, &source_workspace_id, &fork_workspace_id, &item.path) - .await?, - ), - "app" => Some( - compare_two_apps(&db, &source_workspace_id, &fork_workspace_id, &item.path).await?, - ), - "resource" => Some( - compare_two_resources(&db, &source_workspace_id, &fork_workspace_id, &item.path) - .await?, - ), - "variable" => Some( - compare_two_variables(&db, &source_workspace_id, &fork_workspace_id, &item.path) - .await?, - ), - "resource_type" => Some( - compare_two_resource_types( - &db, - &source_workspace_id, - &fork_workspace_id, - &item.path, - ) - .await?, - ), - "folder" => Some( - compare_two_folders(&db, &source_workspace_id, &fork_workspace_id, &item.path) - .await?, - ), - k => { - tracing::error!("Received unrecognized item kind `{k}` with path: `{}` while computing diff of {fork_workspace_id} and {source_workspace_id} workspaces. Skipping this item", item.path); - None - // Some(ItemComparison { - // has_changes: true, - // exists_in_source: true, - // exists_in_fork: true, - // }) - } - }; - - if let Some(item_comparison) = item_comparison { - if item_comparison.has_changes { - sqlx::query!( - "UPDATE workspace_diff SET has_changes = true, exists_in_source = $5, exists_in_fork = $6 - WHERE path = $3 AND kind = $4 AND ( - (source_workspace_id = $1 AND fork_workspace_id = $2) - OR (source_workspace_id = $2 AND fork_workspace_id =$1) - )", - source_workspace_id, - fork_workspace_id, - item.path, - item.kind, - item_comparison.exists_in_source, - item_comparison.exists_in_fork, - ) - .execute(&db) - .await?; - confirmed_diffs.push(WorkspaceDiffRow { - has_changes: Some(item_comparison.has_changes), - exists_in_source: Some(item_comparison.exists_in_source), - exists_in_fork: Some(item_comparison.exists_in_fork), - ..item - }); - } else { - sqlx::query!( - "DELETE FROM workspace_diff WHERE path = $3 AND kind = $4 AND ( - (source_workspace_id = $1 AND fork_workspace_id = $2) - OR (source_workspace_id = $2 AND fork_workspace_id =$1) - )", - source_workspace_id, - fork_workspace_id, - item.path, - item.kind, - ) - .execute(&db) - .await?; - } - } - } - - let visible_diffs = filter_visible_diffs( - &confirmed_diffs, - &source_workspace_id, - &fork_workspace_id, - user_db.begin(&authed).await?, - ) - .await?; - - let summary = CompareSummary { - total_diffs: visible_diffs.len(), - total_ahead: visible_diffs - .iter() - .map(|s| s.ahead) - .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)), - total_behind: visible_diffs - .iter() - .map(|s| s.behind) - .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)), - scripts_changed: visible_diffs.iter().filter(|s| s.kind == "script").count(), - flows_changed: visible_diffs.iter().filter(|s| s.kind == "flow").count(), - apps_changed: visible_diffs.iter().filter(|s| s.kind == "app").count(), - resources_changed: visible_diffs - .iter() - .filter(|s| s.kind == "resource") - .count(), - variables_changed: visible_diffs - .iter() - .filter(|s| s.kind == "variable") - .count(), - resource_types_changed: visible_diffs - .iter() - .filter(|s| s.kind == "resource_type") - .count(), - folders_changed: visible_diffs.iter().filter(|s| s.kind == "folder").count(), - conflicts: visible_diffs - .iter() - .filter(|s| s.ahead > 0 && s.behind > 0) - .count(), - }; - - let all_ahead_items_visible = summary.total_ahead - == confirmed_diffs - .iter() - .map(|s| s.ahead) - .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)); - let all_behind_items_visible = summary.total_behind - == confirmed_diffs - .iter() - .map(|s| s.behind) - .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)); - - return Ok(Json(WorkspaceComparison { - all_ahead_items_visible, - all_behind_items_visible, - skipped_comparison: false, - diffs: visible_diffs, - summary, - })); -} - -async fn filter_visible_diffs( - confirmed_diffs: &[WorkspaceDiffRow], - source_workspace_id: &str, - fork_workspace_id: &str, - mut tx: Transaction<'static, Postgres>, -) -> Result> { - // Step 1: Group paths by (workspace, kind) - let mut source_items: HashMap<&str, Vec<&str>> = HashMap::new(); - let mut fork_items: HashMap<&str, Vec<&str>> = HashMap::new(); - - for diff in confirmed_diffs { - if diff.exists_in_source.unwrap_or(false) { - source_items.entry(&diff.kind).or_default().push(&diff.path); - } - if diff.exists_in_fork.unwrap_or(false) { - fork_items.entry(&diff.kind).or_default().push(&diff.path); - } - } - - // Step 2: Batch query for each (workspace, kind) combination - let source_visible = query_visible_items(&mut tx, source_workspace_id, &source_items).await?; - let fork_visible = query_visible_items(&mut tx, fork_workspace_id, &fork_items).await?; - - // Step 3: Filter diffs based on visibility - let visible_diffs: Vec = confirmed_diffs - .iter() - .filter(|diff| { - let v = (diff.kind.to_string(), diff.path.to_string()); - let source_ok = !diff.exists_in_source.unwrap_or(false) || source_visible.contains(&v); - let fork_ok = !diff.exists_in_fork.unwrap_or(false) || fork_visible.contains(&v); - source_ok && fork_ok - }) - .cloned() - .collect(); - - Ok(visible_diffs) -} - -async fn query_visible_items<'c>( - tx: &mut Transaction<'c, Postgres>, - workspace_id: &str, - items_by_kind: &HashMap<&str, Vec<&str>>, -) -> Result> { - let mut visible = HashSet::new(); - - for (kind, paths) in items_by_kind { - let paths_vec: Vec = paths.iter().map(|s| s.to_string()).collect(); - - let results = match *kind { - "script" => { - sqlx::query_scalar!( - "SELECT path FROM script - WHERE workspace_id = $1 AND path = ANY($2) AND archived = false", - workspace_id, - &paths_vec - ) - .fetch_all(&mut **tx) - .await? - } - "flow" => { - sqlx::query_scalar!( - "SELECT path FROM flow - WHERE workspace_id = $1 AND path = ANY($2) AND archived = false", - workspace_id, - &paths_vec - ) - .fetch_all(&mut **tx) - .await? - } - "app" => { - sqlx::query_scalar!( - "SELECT path FROM app - WHERE workspace_id = $1 AND path = ANY($2)", - workspace_id, - &paths_vec - ) - .fetch_all(&mut **tx) - .await? - } - "resource" => { - sqlx::query_scalar!( - "SELECT path FROM resource - WHERE workspace_id = $1 AND path = ANY($2)", - workspace_id, - &paths_vec - ) - .fetch_all(&mut **tx) - .await? - } - "variable" => { - sqlx::query_scalar!( - "SELECT path FROM variable - WHERE workspace_id = $1 AND path = ANY($2)", - workspace_id, - &paths_vec - ) - .fetch_all(&mut **tx) - .await? - } - "folder" => { - let a: Vec = paths_vec - .iter() - .map(|p| p.strip_prefix("f/").unwrap_or(p.as_str()).to_string()) - .collect(); - sqlx::query_scalar!( - "SELECT name FROM folder - WHERE workspace_id = $1 AND name = ANY($2)", - workspace_id, - &a, - ) - .fetch_all(&mut **tx) - .await? - .into_iter() - .map(|p| format!("f/{p}")) - .collect() - } - "resource_type" => { - sqlx::query_scalar!( - "SELECT name FROM resource_type - WHERE workspace_id = $1 AND name = ANY($2)", - workspace_id, - &paths_vec - ) - .fetch_all(&mut **tx) - .await? - } - _ => vec![], // Unknown kind - }; - - for path in results { - visible.insert((kind.to_string(), path)); - } - } - - Ok(visible) -} - -#[derive(Debug)] -struct ItemComparison { - has_changes: bool, - exists_in_source: bool, - exists_in_fork: bool, -} - -async fn compare_two_scripts( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - path: &str, -) -> Result { - // Get latest script from each workspace - let source_script = sqlx::query!( - "SELECT hash, created_at, content, summary, description, lock, schema - FROM script - WHERE workspace_id = $1 AND path = $2 AND archived = false - ORDER BY created_at DESC - LIMIT 1", - source_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let target_script = sqlx::query!( - "SELECT hash, created_at, content, summary, description, lock, schema - FROM script - WHERE workspace_id = $1 AND path = $2 AND archived = false - ORDER BY created_at DESC - LIMIT 1", - fork_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata differences - if let (Some(source), Some(target)) = (&source_script, &target_script) { - if source.content != target.content - || source.summary != target.summary - || source.description != target.description - || source.lock != target.lock - || source.schema != target.schema - { - has_changes = true; - } - } else if source_script.is_some() || target_script.is_some() { - // The script exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_script.is_some(), - exists_in_fork: target_script.is_some(), - }); -} - -async fn compare_two_flows( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - path: &str, -) -> Result { - // Get latest flow from each workspace - let source_flow = sqlx::query!( - "SELECT value, summary, description, schema - FROM flow - WHERE workspace_id = $1 AND path = $2 AND archived = false", - source_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let target_flow = sqlx::query!( - "SELECT value, summary, description, schema - FROM flow - WHERE workspace_id = $1 AND path = $2 AND archived = false", - fork_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata differences - if let (Some(source), Some(target)) = (&source_flow, &target_flow) { - if source.value != target.value - || source.summary != target.summary - || source.description != target.description - || source.schema != target.schema - { - has_changes = true; - } - } else if source_flow.is_some() || target_flow.is_some() { - // The flow exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_flow.is_some(), - exists_in_fork: target_flow.is_some(), - }); -} - -async fn compare_two_apps( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - path: &str, -) -> Result { - // Get app with its latest version data from source workspace - let source_app = sqlx::query!( - "SELECT app.summary, app.policy, app_version.value - FROM app - JOIN app_version - ON app_version.id = app.versions[array_upper(app.versions, 1)] - WHERE app.workspace_id = $1 AND app.path = $2 AND COALESCE(app.draft_only, false) = false", - source_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let target_app = sqlx::query!( - "SELECT app.summary, app.policy, app_version.value - FROM app - JOIN app_version - ON app_version.id = app.versions[array_upper(app.versions, 1)] - WHERE app.workspace_id = $1 AND app.path = $2 AND COALESCE(app.draft_only, false) = false", - fork_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata and content differences - if let (Some(source), Some(target)) = (&source_app, &target_app) { - if source.summary != target.summary - || source.policy != target.policy - || source.value != target.value - { - has_changes = true; - } - } else if source_app.is_some() || target_app.is_some() { - // The app exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_app.is_some(), - exists_in_fork: target_app.is_some(), - }); -} - -async fn compare_two_resources( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - path: &str, -) -> Result { - // Get resource from each workspace - let source_resource = sqlx::query!( - "SELECT value, description, resource_type - FROM resource - WHERE workspace_id = $1 AND path = $2", - source_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let target_resource = sqlx::query!( - "SELECT value, description, resource_type - FROM resource - WHERE workspace_id = $1 AND path = $2", - fork_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata differences - if let (Some(source), Some(target)) = (&source_resource, &target_resource) { - if source.value != target.value - || source.description != target.description - || source.resource_type != target.resource_type - { - has_changes = true; - } - } else if source_resource.is_some() || target_resource.is_some() { - // The resource exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_resource.is_some(), - exists_in_fork: target_resource.is_some(), - }); -} - -async fn compare_two_variables( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - path: &str, -) -> Result { - // Get variable from each workspace - let source_variable = sqlx::query!( - "SELECT value, is_secret, description - FROM variable - WHERE workspace_id = $1 AND path = $2", - source_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let target_variable = sqlx::query!( - "SELECT value, is_secret, description - FROM variable - WHERE workspace_id = $1 AND path = $2", - fork_workspace_id, - path - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata differences - if let (Some(source), Some(target)) = (&source_variable, &target_variable) { - if source.is_secret != target.is_secret - || source.value != target.value - || source.description != target.description - { - has_changes = true; - } - } else if source_variable.is_some() || target_variable.is_some() { - // The variable exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_variable.is_some(), - exists_in_fork: target_variable.is_some(), - }); -} - -async fn compare_two_resource_types( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - name: &str, -) -> Result { - // Get resource type from each workspace - let source_resource_type = sqlx::query!( - "SELECT schema, description, format_extension - FROM resource_type - WHERE workspace_id = $1 AND name = $2", - source_workspace_id, - name - ) - .fetch_optional(db) - .await?; - - let target_resource_type = sqlx::query!( - "SELECT schema, description, format_extension - FROM resource_type - WHERE workspace_id = $1 AND name = $2", - fork_workspace_id, - name - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata differences - if let (Some(source), Some(target)) = (&source_resource_type, &target_resource_type) { - if source.schema != target.schema - || source.description != target.description - || source.format_extension != target.format_extension - { - has_changes = true; - } - } else if source_resource_type.is_some() || target_resource_type.is_some() { - // The resource type exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_resource_type.is_some(), - exists_in_fork: target_resource_type.is_some(), - }); -} - -async fn compare_two_folders( - db: &DB, - source_workspace_id: &str, - fork_workspace_id: &str, - name: &str, -) -> Result { - // Get folder from each workspace - let source_folder = sqlx::query!( - "SELECT display_name, owners, extra_perms, summary - FROM folder - WHERE workspace_id = $1 AND name = $2", - source_workspace_id, - name.strip_prefix("f/"), - ) - .fetch_optional(db) - .await?; - - let target_folder = sqlx::query!( - "SELECT display_name, owners, extra_perms, summary - FROM folder - WHERE workspace_id = $1 AND name = $2", - fork_workspace_id, - name.strip_prefix("f/"), - ) - .fetch_optional(db) - .await?; - - let mut has_changes = false; - - // Check metadata differences - if let (Some(source), Some(target)) = (&source_folder, &target_folder) { - if source.display_name != target.display_name - || source.owners != target.owners - || source.extra_perms != target.extra_perms - || source.summary != target.summary - { - has_changes = true; - } - } else if source_folder.is_some() || target_folder.is_some() { - // The folder exists in one of source or target, but not the other, this is considered as a change - has_changes = true - } - - return Ok(ItemComparison { - has_changes, - exists_in_source: source_folder.is_some(), - exists_in_fork: target_folder.is_some(), - }); -} diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 2631e22624..2234b4a0f1 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -426,7 +426,7 @@ pub(crate) async fn tarball_workspace( .await?; for script in scripts { - let script = script.prefetch_cached(&db).await?; + let script = windmill_common::scripts::prefetch_cached_script(script, &db).await?; let ext = match script.language { ScriptLang::Python3 => "py", ScriptLang::Deno => { @@ -651,7 +651,7 @@ pub(crate) async fn tarball_workspace( if include_triggers.unwrap_or(false) { #[cfg(feature = "http_trigger")] { - use crate::triggers::http::handler::HttpTrigger; + use crate::triggers::http::HttpTrigger; let handler = HttpTrigger; let http_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; @@ -810,7 +810,8 @@ pub(crate) async fn tarball_workspace( for service_name in ServiceName::iter() { let native_triggers = - list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None).await?; + list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None) + .await?; for trigger in native_triggers { let trigger_str = &to_string_without_metadata( @@ -927,7 +928,7 @@ pub(crate) async fn tarball_workspace( if include_settings.unwrap_or(false) { let row = sqlx::query_as::<_, SettingsRow>( - r#"SELECT + r#"SELECT auto_invite, webhook, deploy_to, @@ -945,10 +946,10 @@ pub(crate) async fn tarball_workspace( FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, - ) - .bind(&w_id) - .fetch_one(&mut *tx) - .await?; + ) + .bind(&w_id) + .fetch_one(&mut *tx) + .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility let settings_str = if settings_version.as_deref() == Some("v2") { @@ -977,11 +978,18 @@ pub(crate) async fn tarball_workspace( let (auto_invite_enabled, auto_invite_as, auto_invite_mode) = if let Some(ref ai) = row.auto_invite { let enabled = ai.get("enabled").and_then(|v| v.as_bool()).unwrap_or(false); - let operator = ai.get("operator").and_then(|v| v.as_bool()).unwrap_or(false); + let operator = ai + .get("operator") + .and_then(|v| v.as_bool()) + .unwrap_or(false); let mode = ai.get("mode").and_then(|v| v.as_str()).unwrap_or("invite"); ( enabled, - if operator { "operator".to_string() } else { "developer".to_string() }, + if operator { + "operator".to_string() + } else { + "developer".to_string() + }, mode.to_string(), ) } else { @@ -992,7 +1000,10 @@ pub(crate) async fn tarball_workspace( if let Some(ref eh) = row.error_handler { let path = eh.get("path").and_then(|v| v.as_str()).map(String::from); let extra_args = eh.get("extra_args").cloned(); - let muted_on_cancel = eh.get("muted_on_cancel").and_then(|v| v.as_bool()).unwrap_or(false); + let muted_on_cancel = eh + .get("muted_on_cancel") + .and_then(|v| v.as_bool()) + .unwrap_or(false); (path, extra_args, muted_on_cancel) } else { (None, None, false) diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index bdd84663f6..d399c08840 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -6,21 +6,23 @@ edition.workspace = true [features] default = [] -enterprise = [] +enterprise = ["dep:aws-config"] +local_reports = ["dep:rsa", "dep:aes-gcm"] private = ["dep:aws-sdk-rds"] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] prometheus = ["dep:prometheus"] benchmark = [] -parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"] -aws_auth = ["dep:aws-sdk-sts"] +parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion", "dep:aws-config", "dep:aws-credential-types"] +aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic", "dep:opentelemetry"] smtp = ["dep:mail-send"] scoped_cache = [] cloud = [] openidconnect = ["dep:openidconnect"] -bedrock = ["dep:aws-sdk-bedrockruntime"] +bedrock = ["dep:aws-sdk-bedrockruntime", "dep:aws-credential-types", "dep:aws-smithy-types", "dep:aws-config"] +python = ["dep:windmill-parser-py"] [lib] name = "windmill_common" @@ -63,10 +65,10 @@ cron.workspace = true magic-crypt.workspace = true object_store = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } -aws-config.workspace = true +aws-config = { workspace = true, optional = true } aws-sdk-sts = { workspace = true, optional = true } -aws-credential-types.workspace = true -aws-smithy-types.workspace = true +aws-credential-types = { workspace = true, optional = true } +aws-smithy-types = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } base64.workspace = true bitflags.workspace = true @@ -89,19 +91,22 @@ crc.workspace = true windmill-macros.workspace = true windmill-parser-sql.workspace = true windmill-parser-ts.workspace = true -windmill-parser-py.workspace = true +windmill-parser-py = { workspace = true, optional = true } windmill-parser.workspace = true jsonwebtoken.workspace = true backon.workspace = true openidconnect = { workspace = true, optional = true } strum.workspace = true strum_macros.workspace = true +windmill-types.workspace = true url.workspace = true urlencoding.workspace = true async-recursion.workspace = true pep440_rs.workspace = true systemstat.workspace = true size.workspace = true +rsa = { workspace = true, optional = true } +aes-gcm = { workspace = true, optional = true } semver.workspace = true croner.workspace = true diff --git a/backend/windmill-common/src/agent_workers.rs b/backend/windmill-common/src/agent_workers.rs index b501e78fd9..30c84f5807 100644 --- a/backend/windmill-common/src/agent_workers.rs +++ b/backend/windmill-common/src/agent_workers.rs @@ -22,8 +22,8 @@ pub struct AgentConfig { impl AgentConfig { pub fn from_env() -> Result { - let agent_token = std::env::var("AGENT_TOKEN") - .map_err(|_| AgentConfigError::MissingAgentToken)?; + let agent_token = + std::env::var("AGENT_TOKEN").map_err(|_| AgentConfigError::MissingAgentToken)?; let base_internal_url = std::env::var("BASE_INTERNAL_URL") .map_err(|_| AgentConfigError::MissingBaseInternalUrl)?; Ok(Self { agent_token, base_internal_url }) @@ -44,10 +44,16 @@ impl std::fmt::Display for AgentConfigError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { AgentConfigError::MissingAgentToken => { - write!(f, "AGENT_TOKEN environment variable is not set but required for agent mode") + write!( + f, + "AGENT_TOKEN environment variable is not set but required for agent mode" + ) } AgentConfigError::MissingBaseInternalUrl => { - write!(f, "BASE_INTERNAL_URL environment variable is not set but required for agent mode") + write!( + f, + "BASE_INTERNAL_URL environment variable is not set but required for agent mode" + ) } } } diff --git a/backend/windmill-common/src/ai_bedrock.rs b/backend/windmill-common/src/ai_bedrock.rs index 5a1ba8837c..ea817a1e41 100644 --- a/backend/windmill-common/src/ai_bedrock.rs +++ b/backend/windmill-common/src/ai_bedrock.rs @@ -20,8 +20,7 @@ use serde::{Deserialize, Serialize}; use crate::error::Error; use crate::ai_types::{ - ContentPart, OpenAIContent, OpenAIFunction, OpenAIMessage, - OpenAIToolCall, ToolDef, + ContentPart, OpenAIContent, OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, }; // ============================================================================ @@ -432,7 +431,10 @@ fn convert_message(msg: &OpenAIMessage) -> Result { "user" => ConversationRole::User, "assistant" => ConversationRole::Assistant, _ => { - return Err(Error::internal_err(format!("Unsupported role: {}", msg.role))); + return Err(Error::internal_err(format!( + "Unsupported role: {}", + msg.role + ))); } }; diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 2e657fd0ac..323e10a128 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -8,7 +8,9 @@ use serde::{Deserialize, Deserializer, Serialize}; /// Deserializes an Option where empty strings become None. /// Use with `#[serde(default, deserialize_with = "empty_string_as_none")]` -pub fn empty_string_as_none<'de, D>(deserializer: D) -> std::result::Result, D::Error> +pub fn empty_string_as_none<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> where D: Deserializer<'de>, { @@ -47,11 +49,7 @@ pub enum AIProvider { impl AIProvider { /// Get the base URL for the AI provider - pub async fn get_base_url( - &self, - resource_base_url: Option, - db: &DB, - ) -> Result { + pub async fn get_base_url(&self, resource_base_url: Option, db: &DB) -> Result { if let Some(base_url) = resource_base_url { return Ok(base_url); } diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index e27504a9f6..50657296c0 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -6,43 +6,19 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use serde::{Deserialize, Serialize}; use serde_json::{from_value, Value}; use tokio::sync::RwLock; use crate::{error, scripts::ScriptLang}; +pub use windmill_types::apps::*; + lazy_static::lazy_static! { pub static ref APP_WORKSPACED_ROUTE: Arc> = Arc::new(RwLock::new(false)); } -/// Id in the `app_script` table. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] -#[serde(transparent)] -pub struct AppScriptId(pub i64); - -#[derive(Deserialize)] -pub struct ListAppQuery { - pub starred_only: Option, - pub path_exact: Option, - pub path_start: Option, - pub include_draft_only: Option, - pub with_deployment_msg: Option, -} - -#[derive(Deserialize)] -pub struct RawAppValue { - pub files: HashMap, -} - -pub struct AppInlineScript { - pub language: Option, - pub content: String, - pub lock: Option, -} - /// Traverse FlowValue while invoking provided by caller callback on leafs // #[async_recursion::async_recursion(?Send)] pub fn traverse_app_inline_scripts< diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index eb4c482c67..669614d64a 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,53 +1,8 @@ -use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; -use std::collections::BTreeMap; use crate::{error, scripts::ScriptHash}; -#[derive( - Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, -)] -#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum AssetKind { - S3Object, - Resource, - // Avoid unnexpected crashes when deserializing old assets - Variable, // Deprecated - Ducklake, - DataTable, -} - -#[derive( - Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, -)] -#[sqlx(type_name = "ASSET_USAGE_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum AssetUsageKind { - Script, - Flow, - Job, -} - -#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] -#[sqlx(type_name = "ASSET_ACCESS_TYPE", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum AssetUsageAccessType { - R, - W, - RW, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] -pub struct AssetWithAltAccessType { - pub path: String, - pub kind: AssetKind, - pub access_type: Option, - pub alt_access_type: Option, - /// Map of column name to access type for column-level access tracking - #[serde(skip_serializing_if = "Option::is_none")] - pub columns: Option>, -} +pub use windmill_types::assets::*; pub async fn insert_static_asset_usage<'e>( executor: impl PgExecutor<'e>, @@ -111,59 +66,21 @@ pub async fn clear_static_asset_usage_by_script_hash<'e>( Ok(()) } -pub fn merge_asset_usage_access_types( - a: Option, - b: Option, -) -> Option { - use AssetUsageAccessType::*; - match (a, b) { - (None, _) | (_, None) => None, - (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), - (Some(RW), _) | (_, Some(RW)) => Some(RW), - (Some(R), Some(R)) => Some(R), - (Some(W), Some(W)) => Some(W), +pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetKind) -> AssetKind { + match parser_kind { + windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, + windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, + windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, + windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, } } -pub fn merge_asset_columns( - a: &Option>, - b: &Option>, -) -> Option> { - match (a, b) { - (None, None) => None, - (Some(cols), None) | (None, Some(cols)) => Some(cols.clone()), - (Some(cols_a), Some(cols_b)) => { - let mut merged = cols_a.clone(); - for (col, access_b) in cols_b { - let access_a = merged.get(col); - let merged_access = - merge_asset_usage_access_types(access_a.cloned(), Some(*access_b)); - if let Some(access) = merged_access { - merged.insert(col.clone(), access); - } - } - Some(merged) - } - } -} - -impl From for AssetKind { - fn from(parser_kind: windmill_parser::asset_parser::AssetKind) -> Self { - match parser_kind { - windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, - windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, - windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, - windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, - } - } -} - -impl From for AssetUsageAccessType { - fn from(parser_kind: windmill_parser::asset_parser::AssetUsageAccessType) -> Self { - match parser_kind { - windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, - windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, - windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, - } +pub fn asset_access_type_from_parser( + parser_kind: windmill_parser::asset_parser::AssetUsageAccessType, +) -> AssetUsageAccessType { + match parser_kind { + windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, + windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, + windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, } } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 89d4831e38..ccd9b5d2a0 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -244,17 +244,43 @@ pub fn permissioned_as_to_username(permissioned_as: &str) -> String { } } -pub async fn fetch_authed_from_permissioned_as( - permissioned_as: String, - email: String, +pub fn fetch_authed_from_permissioned_as<'a, A>( + permissioned_as: &'a str, + email: &'a str, + w_id: &'a str, + db: A, +) -> std::pin::Pin> + Send + 'a>> +where + A: sqlx::Acquire<'a, Database = sqlx::Postgres> + Send + 'a, +{ + Box::pin(async move { + let mut conn = db + .acquire() + .await + .map_err(|e| Error::internal_err(format!("acquiring connection: {e:#}")))?; + + fetch_authed_from_permissioned_as_inner(permissioned_as, email, w_id, &mut *conn).await + }) +} + +async fn fetch_authed_from_permissioned_as_inner( + permissioned_as: &str, + email: &str, w_id: &str, - db: &DB, + conn: &mut sqlx::PgConnection, ) -> Result { - let super_admin = - permissioned_as == SUPERADMIN_SYNC_EMAIL || is_super_admin_email(db, &email).await?; + let is_super_admin = permissioned_as == SUPERADMIN_SYNC_EMAIL + || email == SUPERADMIN_SECRET_EMAIL + || email == SUPERADMIN_NOTIFICATION_EMAIL + || sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) + .fetch_optional(&mut *conn) + .await + .map_err(|e| Error::internal_err(format!("fetching super admin: {e:#}")))? + .unwrap_or(false); + if let Some((prefix, name)) = permissioned_as.split_once('/') { if prefix == "u" { - let (is_admin, is_operator) = if super_admin { + let (is_admin, is_operator) = if is_super_admin { (true, false) } else { let r = sqlx::query!( @@ -263,7 +289,7 @@ pub async fn fetch_authed_from_permissioned_as( name, &w_id ) - .fetch_optional(db) + .fetch_optional(&mut *conn) .await?; if let Some(r) = r { (r.is_admin, r.operator) @@ -274,12 +300,12 @@ pub async fn fetch_authed_from_permissioned_as( } }; - let groups = get_groups_for_user(w_id, &name, &email, db).await?; + let groups = get_groups_for_user(w_id, &name, email, &mut *conn).await?; - let folders = get_folders_for_user(w_id, &name, &groups, db).await?; + let folders = get_folders_for_user(w_id, &name, &groups, &mut *conn).await?; Ok(Authed { - email, + email: email.to_string(), username: name.to_string(), is_admin, is_operator, @@ -290,9 +316,9 @@ pub async fn fetch_authed_from_permissioned_as( }) } else { let groups = vec![name.to_string()]; - let folders = get_folders_for_user(&w_id, "", &groups, db).await?; + let folders = get_folders_for_user(&w_id, "", &groups, &mut *conn).await?; Ok(Authed { - email, + email: email.to_string(), username: format!("group-{name}"), is_admin: false, groups, @@ -303,26 +329,24 @@ pub async fn fetch_authed_from_permissioned_as( }) } } else { - let groups = vec![]; - let folders = vec![]; Ok(Authed { - email, - username: permissioned_as, - is_admin: super_admin, + email: email.to_string(), + username: permissioned_as.to_string(), + is_admin: is_super_admin, is_operator: true, - groups, - folders, + groups: vec![], + folders: vec![], scopes: None, token_prefix: None, }) } } -pub async fn get_folders_for_user( +pub async fn get_folders_for_user<'e, E: sqlx::PgExecutor<'e>>( w_id: &str, username: &str, groups: &[String], - db: &DB, + db: E, ) -> Result> { let mut perms = groups .into_iter() @@ -344,11 +368,11 @@ pub async fn get_folders_for_user( Ok(folders) } -pub async fn get_groups_for_user( +pub async fn get_groups_for_user<'e, E: sqlx::PgExecutor<'e>>( w_id: &str, username: &str, email: &str, - db: &DB, + db: E, ) -> Result> { let groups = sqlx::query_scalar!( "SELECT group_ FROM usr_to_group where usr = $1 AND workspace_id = $2 UNION ALL SELECT igroup FROM email_to_igroup WHERE email = $3", @@ -400,7 +424,7 @@ pub async fn create_token_for_owner( Ok(Some(jp)) => jp.into(), _ => { tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly..."); - fetch_authed_from_permissioned_as(owner.to_string(), email.to_string(), w_id, db) + fetch_authed_from_permissioned_as(owner, email, w_id, db) .await .map_err(|e| { Error::internal_err(format!( diff --git a/backend/windmill-common/src/bench.rs b/backend/windmill-common/src/bench.rs index c851198b2a..cd475edaac 100644 --- a/backend/windmill-common/src/bench.rs +++ b/backend/windmill-common/src/bench.rs @@ -5,6 +5,31 @@ use crate::{ use serde::Serialize; use tokio::time::Instant; +#[derive(Serialize)] +pub struct PoolStats { + pub peak_active_conns: u32, + pub pool_saturation_histogram: Vec, +} + +impl PoolStats { + pub fn new(pool_size: u32) -> Self { + PoolStats { + peak_active_conns: 0, + pool_saturation_histogram: vec![0; pool_size as usize + 1], + } + } + + pub fn sample(&mut self, pool_size: u32, num_idle: u32) { + let active = pool_size.saturating_sub(num_idle); + if active > self.peak_active_conns { + self.peak_active_conns = active; + } + if let Some(bucket) = self.pool_saturation_histogram.get_mut(active as usize) { + *bucket += 1; + } + } +} + #[derive(Serialize)] pub struct BenchmarkInfo { #[serde(skip)] @@ -14,6 +39,7 @@ pub struct BenchmarkInfo { timings: Vec, pub iter_durations: Vec, pub total_duration: Option, + pub pool_stats: Option, } impl BenchmarkInfo { @@ -24,6 +50,17 @@ impl BenchmarkInfo { start: Instant::now(), iter_durations: vec![], total_duration: None, + pool_stats: None, + } + } + + pub fn init_pool_stats(&mut self, pool_size: u32) { + self.pool_stats = Some(PoolStats::new(pool_size)); + } + + pub fn sample_pool(&mut self, pool_size: u32, num_idle: u32) { + if let Some(stats) = self.pool_stats.as_mut() { + stats.sample(pool_size, num_idle); } } @@ -40,8 +77,11 @@ impl BenchmarkInfo { let total_duration = self.start.elapsed().as_millis() as u64; self.total_duration = Some(total_duration as u64); + let pool_info = self.pool_stats.as_ref().map_or(String::new(), |ps| { + format!(", peak active conns: {}", ps.peak_active_conns) + }); println!( - "Writing benchmark {path}, duration of benchmark: {total_duration}ms and RPS: {}", + "Writing benchmark {path}, duration of benchmark: {total_duration}ms and RPS: {}{pool_info}", self.iters as f64 / total_duration as f64 * 1000.0 ); write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling"); @@ -220,6 +260,7 @@ pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { .await .unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs (4)")); } + "none" => {} _ => { let uuids = sqlx::query_scalar!("INSERT INTO v2_job (id, runnable_id, runnable_path, kind, script_lang, tag, created_by, permissioned_as, permissioned_as_email, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9 FROM generate_series(1, $10)) RETURNING id", None::, diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 3cfc98dc98..11d6217fba 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -898,7 +898,7 @@ pub mod workspace_dependencies { time::{Duration, Instant}, }; - use crate::{error, scripts::ScriptLang, workspace_dependencies::WorkspaceDependencies, DB}; + use crate::{error, workspace_dependencies::WorkspaceDependencies, DB}; make_static! { /// Workspace Dependencies by id and workspace cache. @@ -908,7 +908,7 @@ pub mod workspace_dependencies { /// Cache for checking if default/unnamed workspace dependencies exist for a workspace and language. /// Cache key: (workspace_id, language) /// Cache value: (exists: bool, cached_at timestamp) - static ref DEFAULT_WD_EXISTS_CACHE: quick_cache::sync::Cache<(String, ScriptLang), (bool, Instant)> = quick_cache::sync::Cache::new(500); + static ref DEFAULT_WD_EXISTS_CACHE: quick_cache::sync::Cache<(String, String), (bool, Instant)> = quick_cache::sync::Cache::new(500); } /// Cache timeout for existence checks (10 seconds) pub const EXISTS_CACHE_TIMEOUT: Duration = Duration::from_secs(10); @@ -926,15 +926,15 @@ pub mod workspace_dependencies { } pub fn get_cached_is_unnamed_workspace_dependencies_exists<'c>( - language: ScriptLang, + dependencies_filename: String, workspace_id: String, ) -> Option { - let exists_key = (workspace_id.to_string(), language); + let exists_key = (workspace_id.to_string(), dependencies_filename); + if let Some((exists, cached_at)) = DEFAULT_WD_EXISTS_CACHE.get(&exists_key) { if cached_at.elapsed() < EXISTS_CACHE_TIMEOUT { tracing::debug!( workspace_id = %workspace_id, - ?language, exists, "cache hit for unnamed workspace dependencies existence" ); @@ -942,7 +942,6 @@ pub mod workspace_dependencies { } else { tracing::debug!( workspace_id = %workspace_id, - ?language, "cache expired for unnamed workspace dependencies existence" ); DEFAULT_WD_EXISTS_CACHE.remove(&exists_key); @@ -950,24 +949,22 @@ pub mod workspace_dependencies { } else { tracing::debug!( workspace_id = %workspace_id, - ?language, "cache miss for unnamed workspace dependencies existence" ); } None } pub fn set_cached_is_unnamed_workspace_dependencies_exists<'c>( - language: ScriptLang, + dependencies_filename: String, workspace_id: String, exists: bool, ) { tracing::debug!( workspace_id = %workspace_id, - ?language, exists, "setting cache for unnamed workspace dependencies existence" ); - let exists_key = (workspace_id.to_string(), language); + let exists_key = (workspace_id.to_string(), dependencies_filename); DEFAULT_WD_EXISTS_CACHE.insert(exists_key, (exists, Instant::now())); } } diff --git a/backend/windmill-common/src/email_oss.rs b/backend/windmill-common/src/email_oss.rs index 05eb6bf243..2813ebe458 100644 --- a/backend/windmill-common/src/email_oss.rs +++ b/backend/windmill-common/src/email_oss.rs @@ -37,3 +37,10 @@ pub async fn send_email_plain_text( ) -> crate::error::Result<()> { Ok(()) } + +#[cfg(not(feature = "private"))] +pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) { + tracing::warn!( + "send_email_if_possible is not implemented in Windmill's Open Source repository" + ); +} diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 472c9c264c..33056f59fa 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -1,7 +1,9 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use sqlx; +use sqlx::{self, FromRow}; use uuid::Uuid; +use crate::db::DB; use crate::error::Result; #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, sqlx::Type)] @@ -14,6 +16,66 @@ pub enum MessageType { Tool, } +#[derive(Serialize, FromRow, Debug)] +pub struct FlowConversation { + pub id: Uuid, + pub workspace_id: String, + pub flow_path: String, + pub title: Option, + pub created_at: DateTime, + pub updated_at: DateTime, + pub created_by: String, +} + +pub async fn get_or_create_conversation_with_id( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + flow_path: &str, + username: &str, + title: &str, + conversation_id: Uuid, +) -> Result { + // Check if conversation already exists + let existing_conversation = sqlx::query_as!( + FlowConversation, + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + FROM flow_conversation + WHERE id = $1 AND workspace_id = $2", + conversation_id, + w_id + ) + .fetch_optional(&mut **tx) + .await?; + + if let Some(existing) = existing_conversation { + return Ok(existing); + } + + // Truncate title to 25 char characters max + let title = if title.len() > 25 { + format!("{}...", &title[..25]) + } else { + title.to_string() + }; + + // Create new conversation with provided ID + let conversation = sqlx::query_as!( + FlowConversation, + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) + VALUES ($1, $2, $3, $4, $5) + RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + conversation_id, + w_id, + flow_path, + username, + title + ) + .fetch_one(&mut **tx) + .await?; + + Ok(conversation) +} + /// Add a message to a conversation using an existing transaction /// If the conversation doesn't exist, logs a warning and returns Ok (no error thrown) /// This allows memory_id to be used for agent memory without requiring a conversation @@ -67,3 +129,20 @@ pub async fn add_message_to_conversation_tx( Ok(()) } + +/// Delete all memory for a conversation from the database +pub async fn delete_conversation_memory( + db: &DB, + workspace_id: &str, + conversation_id: Uuid, +) -> Result<()> { + sqlx::query!( + "DELETE FROM ai_agent_memory WHERE workspace_id = $1 AND conversation_id = $2", + workspace_id, + conversation_id + ) + .execute(db) + .await?; + + Ok(()) +} diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index d1d33f67ef..c9aa706ac9 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -1,541 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use std::collections::HashMap; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use uuid::Uuid; - -use crate::flows::FlowValue; - -const MINUTES: Duration = Duration::from_secs(60); -const HOURS: Duration = MINUTES.saturating_mul(60); - -pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX; -pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); - -pub fn is_retry_default(v: &RetryStatus) -> bool { - v.fail_count == 0 && v.failed_jobs.is_empty() -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlowStatus { - pub step: i32, - pub modules: Vec, - pub failure_module: Box, - pub preprocessor_module: Option, - - #[serde(skip_serializing_if = "HashMap::is_empty")] - #[serde(default)] - pub user_states: HashMap, - #[serde(default)] - pub cleanup_module: FlowCleanupModule, - #[serde(default)] - #[serde(skip_serializing_if = "is_retry_default")] - pub retry: RetryStatus, - #[serde(skip_serializing_if = "Option::is_none")] - pub approval_conditions: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub restarted_from: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_job: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_input_enabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub memory_id: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct RetryStatus { - pub fail_count: u32, - pub failed_jobs: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct ApprovalConditions { - pub user_auth_required: bool, - pub user_groups_required: Vec, - pub self_approval_disabled: bool, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -#[serde(default)] -pub struct RestartedFrom { - pub flow_job_id: Uuid, - pub step_id: String, - pub branch_or_iteration_n: Option, - pub flow_version: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Iterator { - pub index: usize, - #[serde(skip_serializing_if = "Option::is_none")] - pub itered: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub itered_len: Option, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct BranchAllStatus { - pub branch: usize, - pub len: usize, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum BranchChosen { - Default, - Branch { branch: usize }, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Approval { - pub resume_id: u16, - pub approver: String, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct FlowStatusModuleWParent { - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_module: Option, - #[serde(flatten)] - pub module_status: FlowStatusModule, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -pub struct FlowCleanupModule { - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - pub flow_jobs_to_clean: Vec, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct FlowJobsDuration { - pub started_at: Vec>>, - pub duration_ms: Vec>, -} - -impl FlowJobsDuration { - pub fn set(&mut self, position: Option, value: &Option) { - if let Some(position) = position { - if position >= self.started_at.len() - || position >= self.duration_ms.len() - || value.is_none() - { - return; - } - let value = value.clone().unwrap(); - self.started_at[position] = Some(value.started_at); - self.duration_ms[position] = Some(value.duration_ms); - } - } - - pub fn push(&mut self, value: &Option) { - self.started_at.push(value.as_ref().map(|x| x.started_at)); - self.duration_ms.push(value.as_ref().map(|x| x.duration_ms)); - } - - pub fn new(n: usize) -> Self { - Self { started_at: vec![None; n], duration_ms: vec![None; n] } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct FlowJobDuration { - pub started_at: chrono::DateTime, - pub duration_ms: i64, -} - -impl FlowJobsDuration { - pub fn truncate(&mut self, n: usize) { - self.started_at.truncate(n); - self.duration_ms.truncate(n); - } -} - -#[derive(Deserialize)] -struct UntaggedFlowStatusModule { - #[serde(rename = "type")] - type_: String, - id: Option, - count: Option, - progress: Option, - job: Option, - iterator: Option, - flow_jobs: Option>, - flow_jobs_success: Option>>, - flow_jobs_duration: Option, - branch_chosen: Option, - branchall: Option, - parallel: Option, - while_loop: Option, - approvers: Option>, - failed_retries: Option>, - skipped: Option, - agent_actions: Option>, - agent_actions_success: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum AgentAction { - ToolCall { - job_id: uuid::Uuid, - function_name: String, - module_id: String, - }, - McpToolCall { - call_id: uuid::Uuid, - function_name: String, - resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - arguments: Option, - }, - Message {}, - WebSearch {}, -} - -#[derive(Serialize, Debug, Clone)] -#[serde(tag = "type")] -pub enum FlowStatusModule { - WaitingForPriorSteps { - id: String, - }, - WaitingForEvents { - id: String, - count: u16, - job: Uuid, - }, - WaitingForExecutor { - id: String, - job: Uuid, - }, - InProgress { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - progress: Option, - #[serde(skip_serializing_if = "Option::is_none")] - iterator: Option, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_success: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_duration: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branchall: Option, - #[serde(skip_serializing_if = "std::ops::Not::not")] - parallel: bool, - #[serde(skip_serializing_if = "std::ops::Not::not")] - while_loop: bool, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions_success: Option>, - }, - Success { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_success: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_duration: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(default)] - #[serde(skip_serializing_if = "Vec::is_empty")] - approvers: Vec, - #[serde(skip_serializing_if = "Vec::is_empty")] - failed_retries: Vec, - skipped: bool, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions_success: Option>, - }, - Failure { - id: String, - job: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_success: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - flow_jobs_duration: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch_chosen: Option, - #[serde(skip_serializing_if = "Vec::is_empty")] - failed_retries: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - agent_actions_success: Option>, - }, -} - -impl<'de> Deserialize<'de> for FlowStatusModule { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let untagged: UntaggedFlowStatusModule = - UntaggedFlowStatusModule::deserialize(deserializer)?; - - match untagged.type_.as_str() { - "WaitingForPriorSteps" => Ok(FlowStatusModule::WaitingForPriorSteps { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - }), - "WaitingForEvents" => Ok(FlowStatusModule::WaitingForEvents { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - count: untagged - .count - .ok_or_else(|| serde::de::Error::missing_field("count"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - }), - "WaitingForExecutor" => Ok(FlowStatusModule::WaitingForExecutor { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - }), - "InProgress" => Ok(FlowStatusModule::InProgress { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - iterator: untagged.iterator, - flow_jobs: untagged.flow_jobs, - flow_jobs_success: untagged.flow_jobs_success, - flow_jobs_duration: untagged.flow_jobs_duration, - branch_chosen: untagged.branch_chosen, - branchall: untagged.branchall, - parallel: untagged.parallel.unwrap_or(false), - while_loop: untagged.while_loop.unwrap_or(false), - progress: untagged.progress, - agent_actions: untagged.agent_actions, - agent_actions_success: untagged.agent_actions_success, - }), - "Success" => Ok(FlowStatusModule::Success { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - flow_jobs: untagged.flow_jobs, - flow_jobs_success: untagged.flow_jobs_success, - flow_jobs_duration: untagged.flow_jobs_duration, - branch_chosen: untagged.branch_chosen, - approvers: untagged.approvers.unwrap_or_default(), - failed_retries: untagged.failed_retries.unwrap_or_default(), - skipped: untagged.skipped.unwrap_or(false), - agent_actions: untagged.agent_actions, - agent_actions_success: untagged.agent_actions_success, - }), - "Failure" => Ok(FlowStatusModule::Failure { - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - job: untagged - .job - .ok_or_else(|| serde::de::Error::missing_field("job"))?, - flow_jobs: untagged.flow_jobs, - flow_jobs_success: untagged.flow_jobs_success, - flow_jobs_duration: untagged.flow_jobs_duration, - branch_chosen: untagged.branch_chosen, - failed_retries: untagged.failed_retries.unwrap_or_default(), - agent_actions: untagged.agent_actions, - agent_actions_success: untagged.agent_actions_success, - }), - other => Err(serde::de::Error::unknown_variant( - other, - &[ - "WaitingForPriorSteps", - "WaitingForEvents", - "WaitingForExecutor", - "InProgress", - "Success", - "Failure", - ], - )), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum JobResult { - SingleJob(Uuid), - ListJob(Vec), -} - -impl FlowStatusModule { - pub fn job(&self) -> Option { - match self { - FlowStatusModule::WaitingForPriorSteps { .. } => None, - FlowStatusModule::WaitingForEvents { job, .. } => Some(*job), - FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job), - FlowStatusModule::InProgress { job, .. } => Some(*job), - FlowStatusModule::Success { job, .. } => Some(*job), - FlowStatusModule::Failure { job, .. } => Some(*job), - } - } - - pub fn flow_jobs(&self) -> Option> { - match self { - FlowStatusModule::InProgress { flow_jobs, .. } => flow_jobs.clone(), - FlowStatusModule::Success { flow_jobs, .. } => flow_jobs.clone(), - FlowStatusModule::Failure { flow_jobs, .. } => flow_jobs.clone(), - _ => None, - } - } - - pub fn branch_chosen(&self) -> Option { - match self { - FlowStatusModule::InProgress { branch_chosen, .. } => branch_chosen.clone(), - FlowStatusModule::Success { branch_chosen, .. } => branch_chosen.clone(), - FlowStatusModule::Failure { branch_chosen, .. } => branch_chosen.clone(), - _ => None, - } - } - - pub fn flow_jobs_success(&self) -> Option>> { - match self { - FlowStatusModule::InProgress { flow_jobs_success, .. } => flow_jobs_success.clone(), - FlowStatusModule::Success { flow_jobs_success, .. } => flow_jobs_success.clone(), - FlowStatusModule::Failure { flow_jobs_success, .. } => flow_jobs_success.clone(), - _ => None, - } - } - - pub fn flow_jobs_duration(&self) -> Option { - match self { - FlowStatusModule::InProgress { flow_jobs_duration, .. } => flow_jobs_duration.clone(), - FlowStatusModule::Success { flow_jobs_duration, .. } => flow_jobs_duration.clone(), - FlowStatusModule::Failure { flow_jobs_duration, .. } => flow_jobs_duration.clone(), - _ => None, - } - } - - pub fn job_result(&self) -> Option { - self.flow_jobs() - .map(JobResult::ListJob) - .or_else(|| self.job().map(JobResult::SingleJob)) - } - - pub fn id(&self) -> String { - match self { - FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(), - FlowStatusModule::WaitingForEvents { id, .. } => id.clone(), - FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(), - FlowStatusModule::InProgress { id, .. } => id.clone(), - FlowStatusModule::Success { id, .. } => id.clone(), - FlowStatusModule::Failure { id, .. } => id.clone(), - } - } - - pub fn is_failure(&self) -> bool { - match self { - FlowStatusModule::Failure { .. } => true, - _ => false, - } - } - - pub fn agent_actions(&self) -> Option> { - match self { - FlowStatusModule::InProgress { agent_actions, .. } => agent_actions.clone(), - FlowStatusModule::Success { agent_actions, .. } => agent_actions.clone(), - FlowStatusModule::Failure { agent_actions, .. } => agent_actions.clone(), - _ => None, - } - } - - pub fn agent_actions_success(&self) -> Option> { - match self { - FlowStatusModule::InProgress { agent_actions_success, .. } => { - agent_actions_success.clone() - } - FlowStatusModule::Success { agent_actions_success, .. } => { - agent_actions_success.clone() - } - FlowStatusModule::Failure { agent_actions_success, .. } => { - agent_actions_success.clone() - } - _ => None, - } - } -} - -impl FlowStatus { - pub fn new(f: &FlowValue) -> Self { - Self { - step: if f.preprocessor_module.is_some() { - -1 - } else { - 0 - }, - approval_conditions: None, - modules: f - .modules - .iter() - .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() }) - .collect(), - failure_module: Box::new(FlowStatusModuleWParent { - parent_module: None, - module_status: FlowStatusModule::WaitingForPriorSteps { - id: f - .failure_module - .as_ref() - .map(|x| x.id.clone()) - .unwrap_or_else(|| "failure".to_string()), - }, - }), - preprocessor_module: if f.preprocessor_module.is_some() { - Some(FlowStatusModule::WaitingForPriorSteps { - id: f.preprocessor_module.as_ref().unwrap().id.clone(), - }) - } else { - None - }, - cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, - retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, - restarted_from: None, - user_states: HashMap::new(), - stream_job: None, - chat_input_enabled: f.chat_input_enabled, - memory_id: None, - } - } - - /// current module status ... excluding failure_module - pub fn current_step(&self) -> Option<&FlowStatusModule> { - let i = usize::try_from(self.step).ok()?; - self.modules.get(i) - } -} +pub use windmill_types::flow_status::*; diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 420530ecf3..9c10bf62ce 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -6,1177 +6,18 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{ - collections::{BTreeMap, HashMap}, - time::Duration, - u8, -}; +pub use windmill_types::flows::*; -use anyhow::Context; -use rand::Rng; -use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; -use serde_json::value::RawValue; use sqlx::types::Json; use sqlx::types::JsonRawValue; use crate::{ - assets::AssetWithAltAccessType, cache, db::DB, - error::{Error, Result as WindmillResult}, - more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, - runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, - scripts::{Schema, ScriptHash, ScriptLang}, + error::Error, worker::{to_raw_value, Connection}, }; -#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] -pub struct Flow { - pub workspace_id: String, - pub path: String, - pub summary: String, - pub description: String, - pub value: Json>, - pub edited_by: String, - pub edited_at: chrono::DateTime, - pub archived: bool, - pub schema: Option, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dedicated_worker: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tag: Option, - #[serde(skip_serializing_if = "is_none_or_false")] - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "is_none_or_false")] - pub visible_to_runner_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_behalf_of_email: Option, -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct FlowWithStarred { - #[sqlx(flatten)] - #[serde(flatten)] - pub flow: Flow, - #[serde(skip_serializing_if = "Option::is_none")] - pub starred: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub lock_error_logs: Option, - pub version_id: i64, -} - -fn is_none_or_false(b: &Option) -> bool { - b.is_none() || !b.unwrap() -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct ListableFlow { - pub workspace_id: String, - pub path: String, - pub summary: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub edited_by: Option, - pub edited_at: Option>, - pub archived: bool, - pub extra_perms: serde_json::Value, - pub starred: bool, - pub has_draft: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_error_handler_muted: Option, - #[sqlx(default)] - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, -} - -fn validate_retry(retry: &Retry, module_id: &str) -> WindmillResult<()> { - if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { - return Err(Error::BadRequest(format!( - "Module '{}': Exponential backoff base (seconds) must be greater than 0. A base of 0 would cause immediate retries.", - module_id - ))); - } - Ok(()) -} - -fn validate_flow_value<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let raw_value = Box::::deserialize(deserializer)?; - - let flow_value: FlowValue = serde_json::from_str(raw_value.get()) - .map_err(|e| serde::de::Error::custom(format!("Invalid flow value: {}", e)))?; - - FlowModule::traverse_modules(&flow_value.modules, &mut |module| { - if let Some(ref retry) = module.retry { - validate_retry(retry, &module.id)?; - } - return Ok(()); - }) - .map_err(|e| serde::de::Error::custom(e.to_string()))?; - - if let Some(ref _failure_module) = flow_value.failure_module { - //add validation logic here for failure module - } - - if let Some(ref _preprocessor_module) = flow_value.preprocessor_module { - //add validation logic here for preprocessor module - } - - Ok(raw_value) -} - -#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] -pub struct NewFlow { - pub path: String, - pub summary: String, - pub description: Option, - #[serde(deserialize_with = "validate_flow_value")] - pub value: Box, - pub schema: Option, - pub draft_only: Option, - pub tag: Option, - pub dedicated_worker: Option, - pub timeout: Option, - pub deployment_message: Option, - pub visible_to_runner_only: Option, - pub on_behalf_of_email: Option, - pub ws_error_handler_muted: Option, -} - -impl NewFlow { - pub fn parse_flow_value(&self) -> crate::error::Result { - serde_json::from_str(self.value.get()).map_err(|e| { - crate::error::Error::InternalErr(format!("Failed to parse flow value: {}", e)) - }) - } -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default)] -pub struct FlowValue { - pub modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - pub failure_module: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde(default)] - pub preprocessor_module: Option>, - #[serde(default)] - #[serde(skip_serializing_if = "is_default")] - pub same_worker: bool, - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettings, - #[serde(flatten)] - pub debouncing_settings: DebouncingSettings, - #[serde(skip_serializing_if = "Option::is_none")] - pub skip_expr: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub early_return: Option, - #[serde(skip_serializing_if = "Option::is_none")] - // Priority at the flow level - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub chat_input_enabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_env: Option>>, -} - -impl FlowValue { - pub fn get_flow_module_at_step(&self, step: Step) -> anyhow::Result<&FlowModule> { - let flow_module = match step { - Step::PreprocessorStep => self - .preprocessor_module - .as_deref() - .with_context(|| format!("no preprocessor module")), - Step::Step { idx, .. } => self - .modules - .get(idx) - .with_context(|| format!("no module found at index: {idx}")), - Step::FailureStep => self - .failure_module - .as_deref() - .with_context(|| format!("no failure module")), - }; - - flow_module - } - - /// Traverse FlowValue while invoking provided by caller callback on leafs - // #[async_recursion::async_recursion(?Send)] - // TODO: We may be want this async. - pub fn traverse_leafs crate::error::Result<()>>( - modules: Vec<&FlowModule>, - cb: &mut C, - ) -> crate::error::Result<()> { - use FlowModuleValue::*; - for module in modules { - match serde_json::from_str::(module.value.get())? { - s @ (Script { .. } - | RawScript { .. } - | Flow { .. } - | FlowScript { .. } - | Identity) => cb(&s, &module.id)?, - ForloopFlow { modules, .. } | WhileloopFlow { modules, .. } => { - Self::traverse_leafs(modules.iter().collect(), cb)? - } - AIAgent { tools, .. } => { - for tool in tools { - match &tool.value { - ToolValue::FlowModule(module_value) => cb(module_value, &tool.id)?, - ToolValue::Mcp(_) => { - // MCP tools don't have a FlowModuleValue to traverse - } - ToolValue::Websearch(_) => { - // Websearch tools don't have a FlowModuleValue to traverse - } - } - } - } - BranchOne { default, branches, .. } => { - Self::traverse_leafs(default.iter().collect(), cb)?; - for branch in branches { - Self::traverse_leafs(branch.modules.iter().collect(), cb)?; - } - } - BranchAll { branches, .. } => { - for branch in branches { - Self::traverse_leafs(branch.modules.iter().collect(), cb)?; - } - } - } - } - Ok(()) - } -} - -#[derive(Debug, Copy, Clone)] -pub enum Step { - Step { idx: usize, len: usize }, - PreprocessorStep, - FailureStep, -} - -impl Step { - pub fn from_i32_and_len(step: i32, len: usize) -> Self { - if step < 0 { - Step::PreprocessorStep - } else if (step as usize) < len { - Step::Step { idx: step as usize, len } - } else { - Step::FailureStep - } - } - - pub fn get_step_index(&self) -> Option { - match self { - Step::Step { idx, .. } => Some(*idx), - _ => None, - } - } - - pub fn is_index_step(&self) -> bool { - matches!(self, Step::Step { .. }) - } - - pub fn is_preprocessor_step(&self) -> bool { - matches!(self, Step::PreprocessorStep) - } - - pub fn is_failure_step(&self) -> bool { - matches!(self, Step::FailureStep) - } - - pub fn is_last_step(&self) -> bool { - matches!(self, Step::Step { idx, len } if *idx == len - 1) - } -} - -#[derive(Default, Deserialize, Serialize, Debug, Clone)] -pub struct StopAfterIf { - pub expr: String, - pub skip_if_stopped: bool, - pub error_message: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -pub struct RetryIf { - pub expr: String, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct Retry { - pub constant: ConstantDelay, - pub exponential: ExponentialDelay, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry_if: Option, -} - -impl Retry { - /// Takes the number of previous retries and returns the interval until the next retry if any. - /// - /// May return [`Duration::ZERO`] to retry immediately. - pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option { - let Self { constant, exponential, .. } = self; - - if previous_attempts < constant.attempts { - Some(Duration::from_secs(constant.seconds as u64)) - } else if previous_attempts - constant.attempts < exponential.attempts { - let exp = previous_attempts.saturating_add(1) as u32; - let mut secs = exponential.multiplier * exponential.seconds.saturating_pow(exp); - if let Some(random_factor) = exponential.random_factor { - if random_factor > 0 { - let random_component = - rand::rng().random_range(0..(std::cmp::min(random_factor, 100) as u16)); - secs = match rand::rng().random_bool(1.0 / 2.0) { - true => secs.saturating_add(secs * random_component / 100), - false => secs.saturating_sub(secs * random_component / 100), - }; - } - } - if !silent { - tracing::warn!("Rescheduling job in {} seconds due to failure", secs); - } - Some(Duration::from_secs(secs as u64)) - } else { - None - } - } - - pub fn has_attempts(&self) -> bool { - self.constant.attempts != 0 || self.exponential.attempts != 0 - } - - pub fn max_attempts(&self) -> u32 { - self.constant - .attempts - .saturating_add(self.exponential.attempts) - } - - pub fn max_interval(&self) -> Option { - self.max_attempts() - .checked_sub(1) - .and_then(|p| self.interval(p, true)) - } -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] -#[serde(default)] -pub struct ConstantDelay { - pub attempts: u32, - pub seconds: u16, -} - -/// multiplier * seconds ^ failures (+/- jitter of the previous value, if any) -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] -#[serde(default)] -pub struct ExponentialDelay { - pub attempts: u32, - pub multiplier: u16, - pub seconds: u16, - pub random_factor: Option, // percentage, defaults to 0 for no jitter -} - -impl Default for ExponentialDelay { - fn default() -> Self { - Self { attempts: 0, multiplier: 1, seconds: 0, random_factor: None } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct Suspend { - #[serde(skip_serializing_if = "Option::is_none")] - pub required_events: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub resume_form: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_auth_required: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub user_groups_required: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub self_approval_disabled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub hide_cancel: Option, - #[serde(skip_serializing_if = "false_or_empty")] - pub continue_on_disapprove_timeout: Option, -} - -fn false_or_empty(v: &Option) -> bool { - v.is_none() || v.as_ref().is_some_and(|x| !x) -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct Mock { - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub return_value: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone, Default)] -pub struct FlowModule { - #[serde(default = "default_id")] - pub id: String, - pub value: Box, - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_after_if: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stop_after_all_iters_if: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub suspend: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mock: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub sleep: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde( - default, - deserialize_with = "raw_value_to_input_transform::<_, i32>", - skip_serializing_if = "Option::is_none" - )] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - // Priority at the flow step level - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub continue_on_error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub skip_if: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub apply_preprocessor: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub pass_flow_input_directly: Option, -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct SkipIf { - pub expr: String, -} - -#[derive(Deserialize)] -pub struct FlowModuleValueWithParallel { - #[serde(rename = "type")] - pub type_: String, - pub parallel: Option, - #[serde( - default, - deserialize_with = "raw_value_to_input_transform::<_, u16>", - skip_serializing_if = "Option::is_none" - )] - pub parallelism: Option, -} - -#[derive(Deserialize)] -pub struct FlowModuleValueWithSkipFailures { - pub skip_failures: Option, - pub parallel: Option, - #[serde( - default, - deserialize_with = "raw_value_to_input_transform::<_, u16>", - skip_serializing_if = "Option::is_none" - )] - pub parallelism: Option, -} - -#[derive(Deserialize)] -pub struct BranchWithSkipFailures { - pub skip_failure: Option, -} - -#[derive(Deserialize)] -pub struct FlowModuleWithBranches { - pub branches: Vec, -} - -impl FlowModule { - pub fn id_append(&mut self, s: &str) { - self.id = format!("{}-{}", self.id, s); - } - pub fn get_value(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()).map_err(crate::error::to_anyhow) - } - - pub fn get_value_with_skip_failures(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - } - - pub fn get_branches_skip_failures(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - } - - pub fn is_flow(&self) -> bool { - self.get_type().is_ok_and(|x| x == "flow") - } - - pub fn get_value_with_parallel(&self) -> anyhow::Result { - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - } - - pub fn is_ai_agent(&self) -> bool { - self.get_type().is_ok_and(|x| x == "aiagent") - } - - pub fn is_simple(&self) -> bool { - //todo: flow modules could also be simple execpt for the fact that the case of having single parallel flow approval step is not handled well (Create SuspendedTimeout) - self.get_type() - .is_ok_and(|x| x == "script" || x == "rawscript" || x == "flowscript") - } - - pub fn get_type(&self) -> anyhow::Result<&str> { - #[derive(Deserialize)] - pub struct FlowModuleValueType<'a> { - pub r#type: &'a str, - } - - serde_json::from_str::(self.value.get()) - .map_err(crate::error::to_anyhow) - .map(|x| x.r#type) - } - - pub fn traverse_modules crate::error::Result<()>>( - modules: &Vec, - cb: &mut C, - ) -> crate::error::Result<()> { - for module in modules { - cb(module)?; - match module - .get_value() - .map_err(|e| Error::BadRequest(format!("Module '{}': {}", module.id, e)))? - { - FlowModuleValue::ForloopFlow { modules, .. } - | FlowModuleValue::WhileloopFlow { modules, .. } => { - Self::traverse_modules(&modules, cb)?; - } - FlowModuleValue::BranchOne { branches, default, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - Self::traverse_modules(&default, cb)?; - } - FlowModuleValue::BranchAll { branches, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - } - FlowModuleValue::AIAgent { tools, .. } => { - for tool in tools { - match &tool.value { - ToolValue::FlowModule(module_value) => match module_value { - FlowModuleValue::ForloopFlow { modules, .. } - | FlowModuleValue::WhileloopFlow { modules, .. } => { - Self::traverse_modules(&modules, cb)?; - } - FlowModuleValue::BranchOne { branches, default, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - Self::traverse_modules(&default, cb)?; - } - FlowModuleValue::BranchAll { branches, .. } => { - for branch in branches { - Self::traverse_modules(&branch.modules, cb)?; - } - } - _ => {} - }, - ToolValue::Mcp(_) => { - // MCP tools don't have a FlowModule to traverse - } - ToolValue::Websearch(_) => { - // Websearch tools don't have a FlowModule to traverse - } - } - } - } - _ => {} - } - } - Ok(()) - } -} - -#[derive(Deserialize)] -pub struct UntaggedInputTransform { - #[serde(rename = "type")] - pub type_: String, - pub value: Option>, - pub expr: Option, -} - -impl<'de> Deserialize<'de> for InputTransform { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let untagged: UntaggedInputTransform = UntaggedInputTransform::deserialize(deserializer)?; - - let input_transform = TryInto::::try_into(untagged) - .map_err(|e| serde::de::Error::custom(e))?; - - Ok(input_transform) - } -} - -#[derive(Serialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum InputTransform { - Static { - #[serde(default = "default_null")] - value: Box, - }, - Javascript { - #[serde(default = "default_empty_string")] - expr: String, - }, - Ai, -} - -impl InputTransform { - pub fn new_static_value(value: Box) -> InputTransform { - InputTransform::Static { value } - } - - pub fn new_javascript_expr(expr: &str) -> InputTransform { - InputTransform::Javascript { expr: expr.to_owned() } - } -} - -impl TryFrom for InputTransform { - type Error = anyhow::Error; - fn try_from(value: UntaggedInputTransform) -> Result { - let input_transform = match value.type_.as_str() { - "static" => InputTransform::new_static_value(value.value.unwrap_or_else(default_null)), - "javascript" => InputTransform::new_javascript_expr(&value.expr.unwrap_or_default()), - "ai" => InputTransform::Ai, - other => { - return Err(anyhow::anyhow!( - "got value: {other} for field `type`, expected value: `static` or `javascript`" - )) - } - }; - - Ok(input_transform) - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum RawValueOrFormatted { - RawValue(T), - Formatted { r#type: String, value: Option, expr: Option }, -} - -fn raw_value_to_input_transform<'de, D, T>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, - T: DeserializeOwned + Serialize, -{ - let val = Option::>::deserialize(deserializer)?; - let input_tranform = match val { - Some(RawValueOrFormatted::RawValue(v)) => { - Some(InputTransform::new_static_value(to_raw_value(&v))) - } - Some(RawValueOrFormatted::Formatted { r#type, expr, value }) => { - let untaged_input_transform = UntaggedInputTransform { - type_: r#type, - expr, - value: value.map(|val| to_raw_value(&val)), - }; - let input_transform = TryInto::::try_into(untaged_input_transform) - .map_err(|e| serde::de::Error::custom(e))?; - Some(input_transform) - } - _ => None, - }; - Ok(input_tranform) -} - -/// Id in the `flow_node` table. -#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] -#[serde(transparent)] -pub struct FlowNodeId(pub i64); - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Branch { - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(default = "default_empty_string")] - pub expr: String, - pub modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub modules_node: Option, - #[serde(default = "default_true")] - pub skip_failure: bool, - #[serde(default = "default_true")] - pub parallel: bool, -} - -// Tool types for AI Agent -#[derive(Serialize, Debug, Clone, Deserialize)] -pub struct AgentTool { - pub id: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - pub value: ToolValue, -} - -// Convert FlowModule -> AgentTool -impl From for AgentTool { - fn from(flow_module: FlowModule) -> Self { - let module_value = serde_json::from_str::(flow_module.value.get()) - .unwrap_or(FlowModuleValue::Identity); - - AgentTool { - id: flow_module.id, - summary: flow_module.summary, - value: ToolValue::FlowModule(module_value), - } - } -} - -// Convert AgentTool -> FlowModule (only for FlowModule type tools) -impl From<&AgentTool> for Option { - fn from(tool: &AgentTool) -> Self { - match &tool.value { - ToolValue::FlowModule(module_value) => Some(FlowModule { - id: tool.id.clone(), - value: to_raw_value(module_value), - summary: tool.summary.clone(), - ..Default::default() - }), - ToolValue::Mcp(_) => None, // MCP tools can't be converted to FlowModule - ToolValue::Websearch(_) => None, // Websearch tools can't be converted to FlowModule - } - } -} - -#[derive(Serialize, Debug, Clone)] -#[serde(tag = "tool_type", rename_all = "lowercase")] -pub enum ToolValue { - FlowModule(FlowModuleValue), - Mcp(McpToolValue), - Websearch(WebsearchToolValue), -} - -// Custom deserializer for backward compatibility with old flows -impl<'de> Deserialize<'de> for ToolValue { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - use serde::de::Error; - - let content = serde_json::Value::deserialize(deserializer)?; - - // First, try to deserialize as the new tagged format (with tool_type field) - #[derive(Deserialize)] - #[serde(tag = "tool_type", rename_all = "lowercase")] - enum TaggedToolValue { - FlowModule(FlowModuleValue), - Mcp(McpToolValue), - Websearch(WebsearchToolValue), - } - - if let Ok(tagged) = TaggedToolValue::deserialize(&content) { - return Ok(match tagged { - TaggedToolValue::FlowModule(v) => ToolValue::FlowModule(v), - TaggedToolValue::Mcp(v) => ToolValue::Mcp(v), - TaggedToolValue::Websearch(v) => ToolValue::Websearch(v), - }); - } - - // Fall back to legacy format (direct FlowModuleValue without tool_type) - FlowModuleValue::deserialize(&content) - .map(ToolValue::FlowModule) - .map_err(|_| { - D::Error::custom( - "expected ToolValue with tool_type field or legacy FlowModuleValue", - ) - }) - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct McpToolValue { - pub resource_path: String, - #[serde(default)] - pub include_tools: Vec, - #[serde(default)] - pub exclude_tools: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default)] -pub struct WebsearchToolValue { - // WebSearch tools don't need additional configuration - // The tool is enabled just by adding it to the agent -} - -fn is_none_or_empty_vec(expr: &Option>) -> bool { - expr.is_none() || expr.as_ref().unwrap().is_empty() -} - -#[derive(Serialize, Debug, Clone)] -#[serde( - tag = "type", - rename_all(serialize = "lowercase", deserialize = "lowercase") -)] -pub enum FlowModuleValue { - /// Reference to another script on the workspace - Script { - #[serde(default)] - #[serde(alias = "input_transform")] - input_transforms: HashMap, - path: String, - #[serde(skip_serializing_if = "Option::is_none")] - hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - tag_override: Option, - #[serde(skip_serializing_if = "Option::is_none")] - is_trigger: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pass_flow_input_directly: Option, - }, - - /// Reference to another flow on the workspace - Flow { - #[serde(default)] - #[serde(alias = "input_transform")] - input_transforms: HashMap, - path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pass_flow_input_directly: Option, - }, - - /// For loop node - ForloopFlow { - iterator: InputTransform, - modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - modules_node: Option, - #[serde(default = "default_true")] - skip_failures: bool, - parallel: bool, - #[serde(skip_serializing_if = "Option::is_none")] - parallelism: Option, - #[serde(skip_serializing_if = "Option::is_none")] - squash: Option, - }, - - /// While loop node - WhileloopFlow { - modules: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - modules_node: Option, - #[serde(default = "default_false")] - skip_failures: bool, - #[serde(skip_serializing_if = "Option::is_none")] - squash: Option, - }, - - /// Branch-one node - BranchOne { - branches: Vec, - default: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - default_node: Option, - }, - - /// Branch-all node - BranchAll { - branches: Vec, - #[serde(default = "default_true")] - parallel: bool, - }, - - /// Inline script node - /// Only exists if parsed from value from `flow_version` | `flow` table. - RawScript { - #[serde(default)] - #[serde(alias = "input_transform", serialize_with = "ordered_map")] - input_transforms: HashMap, - content: String, - #[serde(skip_serializing_if = "Option::is_none")] - lock: Option, - #[serde(skip_serializing_if = "Option::is_none")] - path: Option, - #[serde(skip_serializing_if = "is_none_or_empty")] - tag: Option, - language: ScriptLang, - #[serde(flatten)] - concurrency_settings: ConcurrencySettingsWithCustom, - #[serde(skip_serializing_if = "Option::is_none")] - is_trigger: Option, - #[serde(skip_serializing_if = "is_none_or_empty_vec")] - assets: Option>, - }, - - /// Just a placeholder - Identity, - - /// Also Inline script node, but instead of being baked into flow, it references `flow_node` - /// Internal only, never exposed to the frontend. - /// Only exists if parsed from value from `flow_version_lite` table. - FlowScript { - #[serde(default)] - #[serde(alias = "input_transform", serialize_with = "ordered_map")] - input_transforms: HashMap, - id: FlowNodeId, - #[serde(skip_serializing_if = "is_none_or_empty")] - tag: Option, - language: ScriptLang, - #[serde(flatten)] - concurrency_settings: ConcurrencySettingsWithCustom, - #[serde(skip_serializing_if = "Option::is_none")] - is_trigger: Option, - #[serde(skip_serializing_if = "is_none_or_empty_vec")] - assets: Option>, - }, - - // AI agent node - AIAgent { - input_transforms: HashMap, - tools: Vec, - }, -} - -fn is_none_or_empty(expr: &Option) -> bool { - expr.is_none() || expr.as_ref().unwrap().is_empty() -} - -#[derive(Deserialize)] -struct UntaggedFlowModuleValue { - #[serde(rename = "type")] - type_: String, - #[serde(alias = "input_transform")] - input_transforms: Option>, - path: Option, - hash: Option, - tag_override: Option, - iterator: Option, - modules: Option>, - skip_failures: Option, - parallel: Option, - #[serde(default, deserialize_with = "raw_value_to_input_transform::<_, u16>")] - parallelism: Option, - branches: Option>, - default: Option>, - content: Option, - lock: Option, - tag: Option, - language: Option, - is_trigger: Option, - id: Option, - default_node: Option, - modules_node: Option, - assets: Option>, - tools: Option>, - pass_flow_input_directly: Option, - squash: Option, - #[serde(flatten)] - concurrency_settings: ConcurrencySettingsWithCustom, -} - -impl<'de> Deserialize<'de> for FlowModuleValue { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let untagged: UntaggedFlowModuleValue = UntaggedFlowModuleValue::deserialize(deserializer)?; - - match untagged.type_.as_str() { - "script" => Ok(FlowModuleValue::Script { - input_transforms: untagged.input_transforms.unwrap_or_default(), - path: untagged - .path - .ok_or_else(|| serde::de::Error::missing_field("path"))?, - hash: untagged.hash, - tag_override: untagged.tag_override, - is_trigger: untagged.is_trigger, - pass_flow_input_directly: untagged.pass_flow_input_directly, - }), - "flow" => Ok(FlowModuleValue::Flow { - input_transforms: untagged.input_transforms.unwrap_or_default(), - path: untagged - .path - .ok_or_else(|| serde::de::Error::missing_field("path"))?, - pass_flow_input_directly: untagged.pass_flow_input_directly, - }), - "forloopflow" => Ok(FlowModuleValue::ForloopFlow { - iterator: untagged - .iterator - .ok_or_else(|| serde::de::Error::missing_field("iterator"))?, - modules: untagged - .modules - .ok_or_else(|| serde::de::Error::missing_field("modules"))?, - modules_node: untagged.modules_node, - skip_failures: untagged.skip_failures.unwrap_or(true), - parallel: untagged.parallel.unwrap_or(false), - parallelism: untagged.parallelism, - squash: untagged.squash, - }), - "whileloopflow" => Ok(FlowModuleValue::WhileloopFlow { - modules: untagged - .modules - .ok_or_else(|| serde::de::Error::missing_field("modules"))?, - modules_node: untagged.modules_node, - skip_failures: untagged.skip_failures.unwrap_or(false), - squash: untagged.squash, - }), - "branchone" => Ok(FlowModuleValue::BranchOne { - branches: untagged - .branches - .ok_or_else(|| serde::de::Error::missing_field("branches"))?, - default: untagged - .default - .ok_or_else(|| serde::de::Error::missing_field("default"))?, - default_node: untagged.default_node, - }), - "branchall" => Ok(FlowModuleValue::BranchAll { - branches: untagged - .branches - .ok_or_else(|| serde::de::Error::missing_field("branches"))?, - parallel: untagged.parallel.unwrap_or(true), - }), - "rawscript" => Ok(FlowModuleValue::RawScript { - input_transforms: untagged.input_transforms.unwrap_or_default(), - content: untagged - .content - .ok_or_else(|| serde::de::Error::missing_field("content"))?, - lock: untagged.lock, - path: untagged.path, - tag: untagged.tag, - language: untagged - .language - .ok_or_else(|| serde::de::Error::missing_field("language"))?, - concurrency_settings: untagged.concurrency_settings, - is_trigger: untagged.is_trigger, - assets: untagged.assets, - }), - "flowscript" => Ok(FlowModuleValue::FlowScript { - input_transforms: untagged.input_transforms.unwrap_or_default(), - id: untagged - .id - .ok_or_else(|| serde::de::Error::missing_field("id"))?, - tag: untagged.tag, - language: untagged - .language - .ok_or_else(|| serde::de::Error::missing_field("language"))?, - concurrency_settings: untagged.concurrency_settings, - is_trigger: untagged.is_trigger, - assets: untagged.assets, - }), - "identity" => Ok(FlowModuleValue::Identity), - "aiagent" => Ok(FlowModuleValue::AIAgent { - input_transforms: untagged.input_transforms.unwrap_or_default(), - tools: untagged - .tools - .ok_or_else(|| serde::de::Error::missing_field("tools"))?, - }), - other => Err(serde::de::Error::unknown_variant( - other, - &[ - "script", - "flow", - "forloopflow", - "whileloopflow", - "branchone", - "branchall", - "rawscript", - "identity", - "aiagent", - ], - )), - } - } -} - -impl Into> for FlowModuleValue { - fn into(self) -> Box { - to_raw_value(&self) - } -} - -fn ordered_map(value: &HashMap, serializer: S) -> Result -where - S: Serializer, -{ - let ordered: BTreeMap<_, _> = value.iter().collect(); - ordered.serialize(serializer) -} - -#[derive(Deserialize)] -pub struct ListFlowQuery { - pub without_description: Option, - pub path_start: Option, - pub path_exact: Option, - pub edited_by: Option, - pub show_archived: Option, - pub order_by: Option, - pub order_desc: Option, - pub starred_only: Option, - pub include_draft_only: Option, - pub with_deployment_msg: Option, - pub dedicated_worker: Option, -} - -pub fn add_virtual_items_if_necessary(modules: &mut Vec) { - if modules.len() > 0 - && (modules[modules.len() - 1].sleep.is_some() - || modules[modules.len() - 1].suspend.is_some()) - { - modules.push(FlowModule { - id: format!("{}-v", modules[modules.len() - 1].id), - value: crate::worker::to_raw_value(&FlowModuleValue::Identity), - stop_after_if: None, - stop_after_all_iters_if: None, - summary: Some("Virtual module needed for suspend/sleep when last module".to_string()), - mock: None, - retry: None, - sleep: None, - suspend: None, - cache_ttl: None, - cache_ignore_s3_path: None, - timeout: None, - priority: None, - delete_after_use: None, - continue_on_error: None, - skip_if: None, - apply_preprocessor: None, - pass_flow_input_directly: None, - }); - } -} - /// Resolve the value of a flow if any. pub async fn resolve_maybe_value( e: &sqlx::PgPool, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index b1497c3d3e..9362f7ff02 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -4,557 +4,24 @@ use bytes::Bytes; use futures_core::Stream; use indexmap::IndexMap; use once_cell::sync::OnceCell; -use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use sqlx::types::Json; use tokio::io::AsyncReadExt; -use uuid::Uuid; -pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; -pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; - -pub const EMAIL_ERROR_HANDLER_USER_EMAIL: &str = "email_error_handler@windmill.dev"; +pub use windmill_types::jobs::*; use crate::{ - apps::AppScriptId, auth::is_super_admin_email, client::AuthedClient, db::{AuthedRef, UserDbWithAuthed, DB}, error::{self, to_anyhow, Error}, - flow_status::{FlowStatus, RestartedFrom}, - flows::{FlowNodeId, FlowValue, Retry}, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, - runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, TMP_DIR}, - FlowVersionInfo, ScriptHashInfo, + FlowVersionInfo, ScriptHashInfo, Tag, }; -#[derive(Debug, Deserialize, Clone)] -pub struct DynamicInput { - #[serde(rename = "x-windmill-dyn-select-code")] - pub x_windmill_dyn_select_code: String, - #[serde(rename = "x-windmill-dyn-select-lang")] - pub x_windmill_dyn_select_lang: ScriptLang, -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] -#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum JobTriggerKind { - Webhook, - Http, - Websocket, - Kafka, - Email, - Nats, - Mqtt, - Sqs, - Postgres, - Schedule, - Gcp, - Nextcloud, -} - -impl std::fmt::Display for JobTriggerKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let kind = match self { - JobTriggerKind::Webhook => "webhook", - JobTriggerKind::Http => "http", - JobTriggerKind::Websocket => "websocket", - JobTriggerKind::Kafka => "kafka", - JobTriggerKind::Email => "email", - JobTriggerKind::Nats => "nats", - JobTriggerKind::Mqtt => "mqtt", - JobTriggerKind::Sqs => "sqs", - JobTriggerKind::Postgres => "postgres", - JobTriggerKind::Schedule => "schedule", - JobTriggerKind::Gcp => "gcp", - JobTriggerKind::Nextcloud => "nextcloud", - }; - write!(f, "{}", kind) - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] -#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum JobKind { - Script, - #[allow(non_camel_case_types)] - Script_Hub, - Preview, - Dependencies, - Flow, - FlowPreview, - SingleStepFlow, - Identity, - FlowDependencies, - AppDependencies, - #[default] - Noop, - DeploymentCallback, - FlowScript, - FlowNode, - AppScript, - AIAgent, - #[serde(rename = "unassigned_script")] - #[sqlx(rename = "unassigned_script")] - UnassignedScript, - #[serde(rename = "unassigned_flow")] - #[sqlx(rename = "unassigned_flow")] - UnassignedFlow, - #[serde(rename = "unassigned_singlestepflow")] - #[sqlx(rename = "unassigned_singlestepflow")] - UnassignedSinglestepFlow, -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] -#[sqlx(type_name = "JOB_STATUS", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum JobStatus { - Success, - Failure, - Canceled, - Skipped, -} - -impl JobKind { - pub fn is_flow(&self) -> bool { - matches!( - self, - JobKind::Flow | JobKind::FlowPreview | JobKind::SingleStepFlow | JobKind::FlowNode - ) - } - - pub fn is_dependency(&self) -> bool { - matches!( - self, - JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies - ) - } -} - -#[derive(sqlx::FromRow, Debug, Serialize, Clone)] -pub struct QueuedJob { - pub workspace_id: String, - pub id: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub started_at: Option>, - pub scheduled_for: chrono::DateTime, - pub running: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_path: Option, - pub script_entrypoint_override: Option, - pub args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub logs: Option, - pub canceled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_reason: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_ping: Option>, - pub job_kind: JobKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub schedule_path: Option, - pub permissioned_as: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub workflow_as_code_status: Option>>, - pub is_flow_step: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option, - pub same_worker: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub pre_run_error: Option, - pub email: String, - pub visible_to_owner: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub suspend: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub root_job: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub leaf_jobs: Option, - pub tag: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_step_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub preprocessed: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub runnable_settings_handle: Option, -} - -impl QueuedJob { - pub fn script_path(&self) -> &str { - self.script_path - .as_ref() - .map(String::as_str) - .unwrap_or("tmp/main") - } - pub fn is_flow(&self) -> bool { - self.job_kind.is_flow() - } - - pub fn full_path_with_workspace(&self) -> String { - format!( - "{}/{}/{}", - self.workspace_id, - if self.is_flow() { "flow" } else { "script" }, - self.script_path() - ) - } - - pub fn parse_flow_status(&self) -> Option { - self.flow_status - .as_ref() - .and_then(|v| serde_json::from_str::((**v).get()).ok()) - } -} - -impl Default for QueuedJob { - fn default() -> Self { - Self { - workspace_id: "".to_string(), - id: Uuid::default(), - parent_job: None, - created_by: "".to_string(), - created_at: chrono::Utc::now(), - started_at: None, - scheduled_for: chrono::Utc::now(), - running: false, - script_hash: None, - script_path: None, - args: None, - logs: None, - canceled: false, - canceled_by: None, - canceled_reason: None, - last_ping: None, - job_kind: JobKind::Identity, - schedule_path: None, - permissioned_as: "".to_string(), - workflow_as_code_status: None, - flow_status: None, - is_flow_step: false, - language: None, - script_entrypoint_override: None, - same_worker: false, - pre_run_error: None, - email: "".to_string(), - visible_to_owner: false, - suspend: None, - mem_peak: None, - root_job: None, - leaf_jobs: None, - tag: "deno".to_string(), - concurrent_limit: None, - concurrency_time_window_s: None, - timeout: None, - flow_step_id: None, - cache_ttl: None, - cache_ignore_s3_path: None, - priority: None, - preprocessed: None, - runnable_settings_handle: None, - } - } -} - -#[derive(Debug, sqlx::FromRow, Serialize, Clone)] -pub struct CompletedJob { - pub workspace_id: String, - pub id: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub completed_at: Option>, - pub duration_ms: i64, - pub success: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_path: Option, - pub args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub result: Option>>, - pub result_columns: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub logs: Option, - pub deleted: bool, - pub canceled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_reason: Option, - pub job_kind: JobKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub schedule_path: Option, - pub permissioned_as: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub workflow_as_code_status: Option>>, - pub is_flow_step: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option, - pub is_skipped: bool, - pub email: String, - pub visible_to_owner: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - pub tag: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub labels: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub preprocessed: Option, -} - -impl CompletedJob { - pub fn json_result(&self) -> Option { - self.result - .as_ref() - .map(|r| serde_json::from_str(r.get()).ok()) - .flatten() - } - - pub fn parse_flow_status(&self) -> Option { - self.flow_status - .as_ref() - .and_then(|v| serde_json::from_str::((**v).get()).ok()) - } -} - -#[derive(Debug, Clone)] -pub enum JobPayload { - /// Execute Hub Script - ScriptHub { - path: String, - apply_preprocessor: bool, - }, - - /// Execute script - ScriptHash { - hash: ScriptHash, - path: String, - cache_ttl: Option, - cache_ignore_s3_path: Option, - dedicated_worker: Option, - language: ScriptLang, - priority: Option, - apply_preprocessor: bool, - concurrency_settings: ConcurrencySettings, - debouncing_settings: DebouncingSettings, - }, - - /// Execute flow step (can be subflow only). - FlowNode { - id: FlowNodeId, // flow_node(id). - path: String, // flow node inner path (e.g. `outer/branchall-42`). - }, - - /// Execute flow step - FlowScript { - id: FlowNodeId, // flow_node(id). - path: String, - language: ScriptLang, - cache_ttl: Option, - cache_ignore_s3_path: Option, - dedicated_worker: Option, - concurrency_settings: ConcurrencySettings, - }, - - /// Inline App Script - AppScript { - id: AppScriptId, // app_script(id). - path: Option, - language: ScriptLang, - cache_ttl: Option, - }, - - /// Script/App/FlowAsCode Preview - Code(RawCode), - - /// Script Dependency Job - Dependencies { - path: String, - hash: ScriptHash, - language: ScriptLang, - dedicated_worker: Option, - debouncing_settings: DebouncingSettings, - }, - - /// Flow Dependency Job - FlowDependencies { - path: String, - dedicated_worker: Option, - version: i64, - debouncing_settings: DebouncingSettings, - }, - - /// App Dependency Job - AppDependencies { - path: String, - version: i64, - debouncing_settings: DebouncingSettings, - }, - - /// Flow Dependency Job, exposed with API. Requirements can be partially or fully predefined - RawFlowDependencies { - path: String, - flow_value: FlowValue, - }, - - /// Dependency Job, exposed with API. Requirements can be predefined - RawScriptDependencies { - script_path: String, - /// Will reflect raw requirements content (e.g. requirements.in) - content: String, - language: ScriptLang, - }, - - /// Flow Job - Flow { - path: String, - dedicated_worker: Option, - apply_preprocessor: bool, - version: i64, - }, - - RestartedFlow { - completed_job_id: Uuid, - step_id: String, - branch_or_iteration_n: Option, - flow_version: Option, - }, - - /// Flow Preview - RawFlow { - value: FlowValue, - path: Option, - restarted_from: Option, - }, - - /// Flow consisting of single script - SingleStepFlow { - path: String, - hash: Option, - flow_version: Option, - args: HashMap>, - retry: Option, - error_handler_path: Option, - error_handler_args: Option>>, - skip_handler: Option, - cache_ttl: Option, - cache_ignore_s3_path: Option, - priority: Option, - tag_override: Option, - trigger_path: Option, - apply_preprocessor: bool, - concurrency_settings: ConcurrencySettings, - debouncing_settings: DebouncingSettings, - }, - DeploymentCallback { - path: String, - debouncing_settings: DebouncingSettings, - }, - Identity, - Noop, - AIAgent { - path: String, - }, -} - -#[derive(Clone, Serialize, Deserialize, Debug)] -pub struct SkipHandler { - pub path: String, - pub args: HashMap>, - pub stop_condition: String, - pub stop_message: String, -} - -#[derive(Clone, Deserialize, Debug, Default)] -pub struct RawCode { - pub content: String, - pub path: Option, - pub hash: Option, - pub language: ScriptLang, - pub lock: Option, - pub cache_ttl: Option, - pub cache_ignore_s3_path: Option, - pub dedicated_worker: Option, - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettingsWithCustom, - #[serde(flatten)] - // NOTE: Since we can only deserialize the struct, - // even though the older versions pass `custom_debounce_key` to RawCode, - // we can still have `debounce_key` in DebouncingSettings - // we just add alias `custom_debounce_key` - // however, serializing this settings will produce `debounce_key` - pub debouncing_settings: DebouncingSettings, -} - -impl JobPayload { - pub fn job_kind(&self) -> JobKind { - match self { - JobPayload::Noop => JobKind::Noop, - JobPayload::Identity => JobKind::Identity, - JobPayload::Code { .. } => JobKind::Preview, - JobPayload::AIAgent { .. } => JobKind::AIAgent, - JobPayload::FlowNode { .. } => JobKind::FlowNode, - JobPayload::ScriptHash { .. } => JobKind::Script, - JobPayload::AppScript { .. } => JobKind::AppScript, - JobPayload::RawFlow { .. } => JobKind::FlowPreview, - JobPayload::ScriptHub { .. } => JobKind::Script_Hub, - JobPayload::FlowScript { .. } => JobKind::FlowScript, - JobPayload::Dependencies { .. } => JobKind::Dependencies, - JobPayload::SingleStepFlow { .. } => JobKind::SingleStepFlow, - JobPayload::AppDependencies { .. } => JobKind::AppDependencies, - JobPayload::FlowDependencies { .. } => JobKind::FlowDependencies, - JobPayload::RawScriptDependencies { .. } => JobKind::Dependencies, - JobPayload::RawFlowDependencies { .. } => JobKind::FlowDependencies, - JobPayload::DeploymentCallback { .. } => JobKind::DeploymentCallback, - JobPayload::Flow { .. } | JobPayload::RestartedFlow { .. } => JobKind::Flow, - } - } -} - -type Tag = String; - -#[derive(Clone, Debug)] -pub struct OnBehalfOf { - pub email: String, - pub permissioned_as: String, -} - pub fn get_has_preprocessor_from_content_and_lang( content: &str, language: &ScriptLang, @@ -564,6 +31,7 @@ pub fn get_has_preprocessor_from_content_and_lang( let args = windmill_parser_ts::parse_deno_signature(&content, true, true, None)?; args.has_preprocessor.unwrap_or(false) } + #[cfg(feature = "python")] ScriptLang::Python3 => { let args = windmill_parser_py::parse_python_signature(&content, None, true)?; args.has_preprocessor.unwrap_or(false) @@ -670,11 +138,6 @@ pub async fn script_path_to_payload<'e>( )) } -#[inline(always)] -pub fn generate_dynamic_input_key(workspace_id: &str, path: &str) -> String { - format!("{workspace_id}:{path}") -} - pub async fn get_payload_tag_from_prefixed_path( path: &str, db: &DB, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 3fd4de438a..99fa1c46ca 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -27,15 +27,14 @@ use scripts::ScriptLang; use sqlx::{Acquire, Postgres}; pub mod agent_workers; -pub mod ai_providers; -pub mod ai_types; #[cfg(feature = "bedrock")] pub mod ai_bedrock; +pub mod ai_providers; +pub mod ai_types; pub mod apps; -pub mod audit; pub mod assets; +pub mod audit; pub mod auth; -pub mod runtime_assets; #[cfg(feature = "benchmark")] pub mod bench; pub mod cache; @@ -59,10 +58,11 @@ pub mod indexer; pub mod job_metrics; #[cfg(all(feature = "parquet", feature = "private"))] pub mod job_s3_helpers_ee; -pub mod min_version; -pub mod notify_events; #[cfg(feature = "parquet")] pub mod job_s3_helpers_oss; +pub mod min_version; +pub mod notify_events; +pub mod runtime_assets; pub mod workspace_dependencies; #[cfg(feature = "private")] @@ -84,9 +84,9 @@ pub mod result_stream; pub mod runnable_settings; pub mod s3_helpers; pub mod schedule; -pub mod secret_backend; pub mod schema; pub mod scripts; +pub mod secret_backend; pub mod server; #[cfg(feature = "private")] pub mod stats_ee; @@ -97,9 +97,11 @@ pub mod teams_ee; pub mod teams_oss; pub mod tracing_init; pub mod triggers; +pub mod usernames; pub mod users; pub mod utils; pub mod variables; +pub mod webhook; pub mod worker; pub mod worker_group_job_stats; pub mod workspaces; @@ -360,16 +362,22 @@ pub struct PgDatabase { // Wrapper enum to hold either Tls or NoTls connection pub enum TokioPgConnection { - Tls(tokio_postgres::Connection>), + Tls( + tokio_postgres::Connection< + tokio_postgres::Socket, + postgres_native_tls::TlsStream, + >, + ), NoTls(tokio_postgres::Connection), } - -impl Future for TokioPgConnection -{ +impl Future for TokioPgConnection { type Output = Result<(), tokio_postgres::Error>; - fn poll(self: std::pin::Pin<&mut Self>, cx: &mut core::task::Context<'_>) -> core::task::Poll { + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll { // SAFETY: We're simply projecting the Pin from the outer enum to the inner connection field. // The inner connection is never moved out, so this is safe. unsafe { @@ -381,7 +389,6 @@ impl Future for TokioPgConnection } } - impl PgDatabase { pub fn to_uri(&self) -> String { let sslmode = match self.sslmode.as_deref() { @@ -405,32 +412,46 @@ impl PgDatabase { format!( "dbname={dbname} {user} host={host} {password} {port} {sslmode}", dbname = self.dbname, - user = self.user.as_ref().map(|u| format!("user={}", urlencoding::encode(u))).unwrap_or_default(), + user = self + .user + .as_ref() + .map(|u| format!("user={}", urlencoding::encode(u))) + .unwrap_or_default(), host = self.host, - password = self.password - .as_ref().map(|p| format!("password={}", urlencoding::encode(p))) + password = self + .password + .as_ref() + .map(|p| format!("password={}", urlencoding::encode(p))) .unwrap_or_default(), port = self.port.map(|p| format!("port={}", p)).unwrap_or_default(), - sslmode = self.sslmode - .as_ref().map(|s| format!("sslmode={}", s.clone())) + sslmode = self + .sslmode + .as_ref() + .map(|s| format!("sslmode={}", s.clone())) .unwrap_or_default(), ) } - pub async fn connect(&self) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { - use tokio_postgres::tls::{ NoTls }; - use postgres_native_tls::MakeTlsConnector; + pub async fn connect( + &self, + ) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { use native_tls::{Certificate, TlsConnector}; - let ssl_mode_is_require = matches!(self.sslmode.as_deref(), Some("require") | Some("verify-ca") | Some("verify-full")); - + use postgres_native_tls::MakeTlsConnector; + use tokio_postgres::tls::NoTls; + let ssl_mode_is_require = matches!( + self.sslmode.as_deref(), + Some("require") | Some("verify-ca") | Some("verify-full") + ); + if ssl_mode_is_require { tracing::info!("Creating new connection"); let mut connector = TlsConnector::builder(); if let Some(root_certificate_pem) = &self.root_certificate_pem { if !root_certificate_pem.is_empty() { connector.add_root_certificate( - Certificate::from_pem(root_certificate_pem.as_bytes()) - .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, + Certificate::from_pem(root_certificate_pem.as_bytes()).map_err(|e| { + error::Error::BadConfig(format!("Invalid Certs: {e:#}")) + })?, ); } else { connector.danger_accept_invalid_certs(true); @@ -441,7 +462,7 @@ impl PgDatabase { .danger_accept_invalid_certs(true) .danger_accept_invalid_hostnames(true); } - + let (client, connection) = tokio::time::timeout( std::time::Duration::from_secs(20), tokio_postgres::connect( @@ -452,7 +473,7 @@ impl PgDatabase { .await .map_err(to_anyhow)? .map_err(to_anyhow)?; - + Ok((client, TokioPgConnection::Tls(connection))) } else { tracing::info!("Creating new connection"); @@ -463,17 +484,19 @@ impl PgDatabase { .await .map_err(to_anyhow)? .map_err(to_anyhow)?; - + Ok((client, TokioPgConnection::NoTls(connection))) } } pub fn parse_uri(url: &str) -> Result { - let parsed_url = - url::Url::parse(url).map_err(|_| Error::BadConfig("Invalid PostgreSQL URL".to_string()))?; + let parsed_url = url::Url::parse(url) + .map_err(|_| Error::BadConfig("Invalid PostgreSQL URL".to_string()))?; let username = parsed_url.username().to_string(); - let username = urlencoding::decode(&username).map_err(to_anyhow)?.to_string(); + let username = urlencoding::decode(&username) + .map_err(to_anyhow)? + .to_string(); let password = parsed_url.password().map(|p| p.to_string()); let password = match password { Some(p) => Some(urlencoding::decode(&p).map_err(to_anyhow)?.to_string()), @@ -541,10 +564,8 @@ impl DatabaseUrl { let guard = rds_url.read().await; Ok(guard.connect_options()) } - DatabaseUrl::Static(url) => { - sqlx::postgres::PgConnectOptions::from_str(url) - .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))) - } + DatabaseUrl::Static(url) => sqlx::postgres::PgConnectOptions::from_str(url) + .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))), } } @@ -781,7 +802,9 @@ pub async fn connect( } }) .connect_with( - database_url.connect_options().await? + database_url + .connect_options() + .await? .statement_cache_capacity(400), ) .await @@ -793,7 +816,10 @@ type Tag = String; pub use db::DB; use crate::{ - auth::{PermsCache, FLOW_PERMS_CACHE, HASH_PERMS_CACHE}, db::{AuthedRef, UserDbWithAuthed}, error::to_anyhow, runnable_settings::RunnableSettings, scripts::{ScriptHash, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline} + auth::{PermsCache, FLOW_PERMS_CACHE, HASH_PERMS_CACHE}, + db::{AuthedRef, UserDbWithAuthed}, + error::to_anyhow, + scripts::{ScriptHash, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, }; #[derive(Clone)] @@ -822,19 +848,19 @@ pub struct ScriptHashInfo { } impl ScriptHashInfo { - pub async fn prefetch_cached<'a>(self, db: &DB) -> error::Result> { - - let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( + pub async fn prefetch_cached<'a>( + self, + db: &DB, + ) -> error::Result> { + let rs = runnable_settings::from_handle( self.runnable_settings.runnable_settings_handle, db, ) - .await? - .prefetch_cached(db) .await?; + let (debouncing_settings, concurrency_settings) = + runnable_settings::prefetch_cached(&rs, db).await?; - Ok( - ScriptHashInfo { + Ok(ScriptHashInfo { path: self.path, hash: self.hash, tag: self.tag, @@ -848,17 +874,17 @@ impl ScriptHashInfo { has_preprocessor: self.has_preprocessor, on_behalf_of_email: self.on_behalf_of_email, created_by: self.created_by, - runnable_settings: ScriptRunnableSettingsInline{ - concurrency_settings: concurrency_settings - .maybe_fallback( + runnable_settings: ScriptRunnableSettingsInline { + concurrency_settings: concurrency_settings.maybe_fallback( self.runnable_settings.concurrency_key, self.runnable_settings.concurrent_limit, self.runnable_settings.concurrency_time_window_s, ), - debouncing_settings: debouncing_settings - .maybe_fallback(self.runnable_settings.debounce_key, self.runnable_settings.debounce_delay_s), + debouncing_settings: debouncing_settings.maybe_fallback( + self.runnable_settings.debounce_key, + self.runnable_settings.debounce_delay_s, + ), }, - }) } } @@ -1329,4 +1355,3 @@ impl KillpillSender { // self.already_sent.load(Ordering::SeqCst) // } } - diff --git a/backend/windmill-common/src/more_serde.rs b/backend/windmill-common/src/more_serde.rs index 97df70219c..8288d79887 100644 --- a/backend/windmill-common/src/more_serde.rs +++ b/backend/windmill-common/src/more_serde.rs @@ -8,60 +8,4 @@ //! helpers for serde + serde derive attributes -use crate::utils::rd_string; -use serde::{Deserialize, Deserializer}; -use serde_json::value::RawValue; -use std::{fmt::Display, str::FromStr}; - -pub fn default_true() -> bool { - true -} - -pub fn default_false() -> bool { - false -} - -pub fn default_null() -> Box { - RawValue::from_string("null".to_string()).unwrap() -} - -pub fn default_empty_string() -> String { - String::new() -} - -pub fn default_id() -> String { - rd_string(6) -} - -pub fn is_default(t: &T) -> bool { - &T::default() == t -} - -pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, - T: FromStr + serde::Deserialize<'de>, - ::Err: Display, -{ - #[derive(Deserialize)] - #[serde(untagged)] - enum NumericOrNull<'a, T> { - String(String), - Str(&'a str), - RawT(T), - Null, - } - - match NumericOrNull::::deserialize(deserializer)? { - NumericOrNull::String(s) => match s.as_str() { - "" => Ok(None), - _ => T::from_str(&s).map(Some).map_err(serde::de::Error::custom), - }, - NumericOrNull::Str(s) => match s { - "" => Ok(None), - _ => T::from_str(s).map(Some).map_err(serde::de::Error::custom), - }, - NumericOrNull::RawT(i) => Ok(Some(i)), - NumericOrNull::Null => Ok(None), - } -} +pub use windmill_types::more_serde::*; diff --git a/backend/windmill-common/src/notify_events.rs b/backend/windmill-common/src/notify_events.rs index e00b6d7d49..84e3c8a4ec 100644 --- a/backend/windmill-common/src/notify_events.rs +++ b/backend/windmill-common/src/notify_events.rs @@ -51,7 +51,10 @@ pub async fn get_latest_event_id(db: &Pool) -> Result { /// Delete events older than the specified number of minutes. /// Returns the number of deleted rows. -pub async fn cleanup_old_events(db: &Pool, older_than_minutes: i32) -> Result { +pub async fn cleanup_old_events( + db: &Pool, + older_than_minutes: i32, +) -> Result { let result = sqlx::query( "DELETE FROM notify_event WHERE created_at < now() - make_interval(mins => $1)", ) diff --git a/backend/windmill-common/src/runnable_settings/settings.rs b/backend/windmill-common/src/runnable_settings/settings.rs index 6dc4eb38ca..04b62fbaab 100644 --- a/backend/windmill-common/src/runnable_settings/settings.rs +++ b/backend/windmill-common/src/runnable_settings/settings.rs @@ -1,9 +1,5 @@ -use std::{ - future::Future, - hash::{Hash, Hasher}, -}; +use std::future::Future; -use serde::{Deserialize, Serialize}; use sqlx::{Pool, Postgres}; use crate::{ @@ -14,49 +10,43 @@ use crate::{ DB, }; -#[derive(Deserialize, Clone, Copy, Serialize, Default, Hash)] -pub struct RunnableSettings { - pub debouncing_settings: Option, - pub concurrency_settings: Option, +pub use windmill_types::runnable_settings::*; + +pub async fn prefetch_cached( + rs: &RunnableSettings, + db: &DB, +) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { + Ok(( + if let Some(hash) = rs.debouncing_settings { + DebouncingSettings::get(hash, db).await? + } else { + Default::default() + }, + if let Some(hash) = rs.concurrency_settings { + ConcurrencySettings::get(hash, db).await? + } else { + Default::default() + }, + )) } -impl RunnableSettings { - pub async fn prefetch_cached<'a>( - &self, - db: &DB, - ) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { - Ok(( - if let Some(hash) = self.debouncing_settings { - DebouncingSettings::get(hash, db).await? - } else { - Default::default() - }, - if let Some(hash) = self.concurrency_settings { - ConcurrencySettings::get(hash, db).await? - } else { - Default::default() - }, - )) - } +pub async fn prefetch_cached_from_handle( + hash: Option, + db: &DB, +) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { + let rs = from_handle(hash, db).await?; + prefetch_cached(&rs, db).await +} - pub async fn prefetch_cached_from_handle<'a>( - hash: Option, - db: &'a DB, - ) -> error::Result<(DebouncingSettings, ConcurrencySettings)> { - Self::from_runnable_settings_handle(hash, db) - .await? - .prefetch_cached(db) - .await - } - /// Returns error if provided `hash` has no corresponding entry in db - /// If `hash` is None, returnes Default - pub fn from_runnable_settings_handle<'a>( - hash: Option, - db: &'a DB, - ) -> impl Future> + 'a { - async move { - if let Some(hash) = hash { - super::RUNNABLE_SETTINGS_REFERENCES +/// Returns error if provided `hash` has no corresponding entry in db +/// If `hash` is None, returns Default +pub fn from_handle<'a>( + hash: Option, + db: &'a DB, +) -> impl Future> + 'a { + async move { + if let Some(hash) = hash { + super::RUNNABLE_SETTINGS_REFERENCES .get_or_insert_async(hash, async { sqlx::query_as!( RunnableSettings, @@ -68,138 +58,45 @@ impl RunnableSettings { .map_err(error::Error::from) }) .await - } else { - Ok(Self::default()) - } + } else { + Ok(RunnableSettings::default()) } } +} - pub async fn insert_cached(self, db: &Pool) -> error::Result> { - if !min_version_supports_runnable_settings_v0().await - || (self.debouncing_settings.is_none() && self.concurrency_settings.is_none()) - { - return Ok(None); - } +pub async fn insert_rs(rs: RunnableSettings, db: &Pool) -> error::Result> { + use std::hash::{Hash, Hasher}; - let hash = { - let mut h = std::hash::DefaultHasher::new(); - self.hash(&mut h); - h.finish() as i64 - }; + if !min_version_supports_runnable_settings_v0().await + || (rs.debouncing_settings.is_none() && rs.concurrency_settings.is_none()) + { + return Ok(None); + } - super::RUNNABLE_SETTINGS_REFERENCES - .get_or_insert_async(hash, async { - sqlx::query!( - "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings) + let hash = { + let mut h = std::hash::DefaultHasher::new(); + rs.hash(&mut h); + h.finish() as i64 + }; + + super::RUNNABLE_SETTINGS_REFERENCES + .get_or_insert_async(hash, async { + sqlx::query!( + "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings) VALUES ($1, $2, $3) ON CONFLICT (hash) DO NOTHING", - hash, - self.debouncing_settings, - self.concurrency_settings - ) - .execute(db) - .await?; - // .map_err(error::Error::from) - Ok(self) - }) + hash, + rs.debouncing_settings, + rs.concurrency_settings + ) + .execute(db) .await?; + Ok(rs) + }) + .await?; - Ok(Some(hash)) - } -} - -// TODO: Add validation logic. -#[derive( - Debug, Clone, Serialize, Deserialize, Default, Hash, PartialEq, sqlx::FromRow, sqlx::Type, -)] -pub struct DebouncingSettings { - #[serde(skip_serializing_if = "Option::is_none", alias = "custom_debounce_key")] - /// debounce key is usually stored in the db - /// including when: - /// - /// 1. User have created custom debounce key from ui or cli - /// 2. User used default one - /// - /// Default: hash(path + step_id + inputs) - pub debounce_key: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - /// Debouncing delay will be determined by the first job with the key. - /// All subsequent jobs with Some will get debounced. - /// If the job has no delay, it will execute immediately, fully ignoring pending delays. - pub debounce_delay_s: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub max_total_debouncing_time: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub max_total_debounces_amount: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - /// top level arguments to preserve - /// For every debounce selected arguments will be saved - /// in the end (when job finally starts) arguments will be appended and passed to runnable - /// - /// NOTE: selected args should be the lists. - pub debounce_args_to_accumulate: Option>, -} - -#[derive( - Debug, Default, Clone, Serialize, Deserialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode, -)] -pub struct ConcurrencySettings { - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] -pub struct ConcurrencySettingsWithCustom { - #[serde(skip_serializing_if = "Option::is_none")] - pub custom_concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, -} - -impl DebouncingSettings { - pub fn maybe_fallback( - self, - debounce_key: Option, - debounce_delay_s: Option, - ) -> Self { - Self { - debounce_key: self.debounce_key.or(debounce_key), - debounce_delay_s: self.debounce_delay_s.or(debounce_delay_s), - ..self - } - } - - pub fn is_legacy_compatible(&self) -> bool { - self.max_total_debouncing_time.is_none() - && self.max_total_debounces_amount.is_none() - && self.debounce_args_to_accumulate.is_none() - } -} - -impl ConcurrencySettings { - pub fn maybe_fallback( - self, - concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, - ) -> Self { - Self { - concurrency_key: self.concurrency_key.or(concurrency_key), - concurrent_limit: self.concurrent_limit.or(concurrent_limit), - concurrency_time_window_s: self.concurrency_time_window_s.or(concurrency_time_window_s), - } - } + Ok(Some(hash)) } impl super::private_mod::RunnableSettingsTraitInternal for DebouncingSettings { @@ -235,31 +132,3 @@ impl super::private_mod::RunnableSettingsTraitInternal for ConcurrencySettings { } } impl super::RunnableSettingsTrait for ConcurrencySettings {} - -impl From for ConcurrencySettingsWithCustom { - fn from( - ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }: ConcurrencySettings, - ) -> Self { - ConcurrencySettingsWithCustom { - custom_concurrency_key: concurrency_key, - concurrency_time_window_s, - concurrent_limit, - } - } -} - -impl From for ConcurrencySettings { - fn from( - ConcurrencySettingsWithCustom { - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - }: ConcurrencySettingsWithCustom, - ) -> Self { - ConcurrencySettings { - concurrency_key: custom_concurrency_key, - concurrency_time_window_s, - concurrent_limit, - } - } -} diff --git a/backend/windmill-common/src/runtime_assets.rs b/backend/windmill-common/src/runtime_assets.rs index 811e62e9d8..817ff99bf4 100644 --- a/backend/windmill-common/src/runtime_assets.rs +++ b/backend/windmill-common/src/runtime_assets.rs @@ -52,7 +52,7 @@ fn extract_assets_from_raw_value( if prefix { let s = serde_json::from_str::(value.get()).ok()?; let (kind, path) = parse_asset_syntax(&s, false)?; - assets.push(RuntimeAsset { path: path.to_string(), kind: kind.into() }); + assets.push(RuntimeAsset { path: path.to_string(), kind: crate::assets::asset_kind_from_parser(kind) }); } } None @@ -178,7 +178,7 @@ pub fn init_runtime_asset_loop( match RUNTIME_ASSET_SENDER.set(tx) { Ok(_) => {} Err(_) => { - tracing::error!("RUNTIME_ASSET_SENDER was already set, skipping"); + tracing::debug!("RUNTIME_ASSET_SENDER was already set, skipping"); return; } } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index b3980aa596..18eeecd2ed 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -792,10 +792,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result error::Result, - pub schedule: String, - pub timezone: String, - pub enabled: bool, - pub script_path: String, - pub is_flow: bool, - pub args: Option>>, - pub extra_perms: serde_json::Value, - pub email: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure_times: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure_exact: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_failure_extra_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_recovery: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_recovery_times: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_recovery_extra_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_success: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_success_extra_args: Option>>, - pub ws_error_handler_muted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, - pub no_flow_overlap: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub summary: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub paused_until: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub cron_version: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub dynamic_skip: Option, -} - -impl Schedule { - pub fn parse_retry(self) -> Option { - self.retry.map(|r| serde_json::from_value(r).ok()).flatten() - } -} - -pub fn schedule_to_user(path: &str) -> String { - format!("schedule-{}", path.replace('/', "-")) -} +pub use windmill_types::schedule::*; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 28572894d2..2f87cb14b0 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -6,17 +6,11 @@ * LICENSE-AGPL for a copy of the license. */ -use std::{ - fmt::{self, Display}, - hash::{Hash, Hasher}, - ops::Deref, - str::FromStr, -}; +pub use windmill_types::scripts::*; use crate::{ - assets::AssetWithAltAccessType, error::{to_anyhow, Error}, - runnable_settings::{ConcurrencySettings, DebouncingSettings, RunnableSettings}, + runnable_settings::{self}, utils::http_get_from_hub, workspace_dependencies::WorkspaceDependenciesAnnotatedRefs, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, PRIVATE_HUB_MIN_VERSION, @@ -26,708 +20,119 @@ use crate::worker::HUB_CACHE_DIR; use anyhow::Context; use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; -use itertools::Itertools; use regex::Regex; -use serde::de::Error as _; -use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; use crate::utils::StripPath; -#[derive( - Serialize, - Deserialize, - Debug, - PartialEq, - Copy, - Clone, - Hash, - Eq, - sqlx::Type, - Default, - Ord, - PartialOrd, -)] -#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum ScriptLang { - Nativets, - #[default] - Deno, - Python3, - Go, - Bash, - Powershell, - Postgresql, - Bun, - Bunnative, - Mysql, - Bigquery, - Snowflake, - Graphql, - Mssql, - OracleDB, - DuckDb, - Php, - Rust, - Ansible, - CSharp, - Nu, - Java, - Ruby, - // for related places search: ADD_NEW_LANG +pub fn extract_workspace_dependencies_annotated_refs( + lang: &ScriptLang, + code: &str, + runnable_path: &str, +) -> Option> { + use ScriptLang::*; + lazy_static::lazy_static! { + static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); + } + match lang { + // TODO: Maybe use regex + Bun | Bunnative | Nativets => WorkspaceDependenciesAnnotatedRefs::parse( + "//", + "package_json", + code, + None, + runnable_path, + ), + Python3 => WorkspaceDependenciesAnnotatedRefs::parse( + "#", + "requirements", + code, + Some(&RE_PYTHON), + runnable_path, + ), + Go => { + WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path) + } + Php => WorkspaceDependenciesAnnotatedRefs::parse( + "//", + "composer_json", + code, + None, + runnable_path, + ), + _ => return None, + } } -impl ScriptLang { - pub fn as_str(&self) -> &'static str { - match self { - ScriptLang::Bun => "bun", - ScriptLang::Bunnative => "bunnative", - ScriptLang::Nativets => "nativets", - ScriptLang::Deno => "deno", - ScriptLang::Python3 => "python3", - ScriptLang::Go => "go", - ScriptLang::Bash => "bash", - ScriptLang::Powershell => "powershell", - ScriptLang::Postgresql => "postgresql", - ScriptLang::Mysql => "mysql", - ScriptLang::Bigquery => "bigquery", - ScriptLang::Snowflake => "snowflake", - ScriptLang::Mssql => "mssql", - ScriptLang::Graphql => "graphql", - ScriptLang::OracleDB => "oracledb", - ScriptLang::DuckDb => "duckdb", - ScriptLang::Php => "php", - ScriptLang::Rust => "rust", - ScriptLang::Ansible => "ansible", - ScriptLang::CSharp => "csharp", - ScriptLang::Nu => "nu", - ScriptLang::Java => "java", - ScriptLang::Ruby => "ruby", - // for related places search: ADD_NEW_LANG - } - } +pub async fn prefetch_cached_script( + script: Script, + db: &DB, +) -> crate::error::Result> { + let rs = runnable_settings::from_handle( + script.runnable_settings.runnable_settings_handle, + db, + ) + .await?; + let (debouncing_settings, concurrency_settings) = + runnable_settings::prefetch_cached(&rs, db).await?; - pub fn as_dependencies_filename(&self) -> Option { - use ScriptLang::*; - Some( - match self { - Bun | Bunnative => "package.json", - Python3 => "requirements.in", - // Go => "go.mod", - Php => "composer.json", - _ => return None, - } - .to_owned(), - ) - } - - pub fn as_comment_lit(&self) -> String { - use ScriptLang::*; - match self { - Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//", - Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", - Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", - Rust => "//!", - // for related places search: ADD_NEW_LANG - } - .to_owned() - } - - pub fn extract_workspace_dependencies_annotated_refs( - &self, - code: &str, - runnable_path: &str, - ) -> Option> { - use ScriptLang::*; - lazy_static::lazy_static! { - static ref RE_PYTHON: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); - } - match self { - // TODO: Maybe use regex - Bun | Bunnative => WorkspaceDependenciesAnnotatedRefs::parse( - "//", - "package_json", - code, - None, - runnable_path, + Ok(Script { + workspace_id: script.workspace_id, + hash: script.hash, + path: script.path, + parent_hashes: script.parent_hashes, + summary: script.summary, + description: script.description, + content: script.content, + created_by: script.created_by, + created_at: script.created_at, + archived: script.archived, + schema: script.schema, + deleted: script.deleted, + is_template: script.is_template, + extra_perms: script.extra_perms, + lock: script.lock, + lock_error_logs: script.lock_error_logs, + language: script.language, + kind: script.kind, + tag: script.tag, + draft_only: script.draft_only, + envs: script.envs, + dedicated_worker: script.dedicated_worker, + ws_error_handler_muted: script.ws_error_handler_muted, + priority: script.priority, + cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, + timeout: script.timeout, + delete_after_use: script.delete_after_use, + restart_unless_cancelled: script.restart_unless_cancelled, + visible_to_runner_only: script.visible_to_runner_only, + no_main_func: script.no_main_func, + codebase: script.codebase, + has_preprocessor: script.has_preprocessor, + on_behalf_of_email: script.on_behalf_of_email, + assets: script.assets, + runnable_settings: ScriptRunnableSettingsInline { + concurrency_settings: concurrency_settings.maybe_fallback( + script.runnable_settings.concurrency_key, + script.runnable_settings.concurrent_limit, + script.runnable_settings.concurrency_time_window_s, ), - Python3 => WorkspaceDependenciesAnnotatedRefs::parse( - "#", - "requirements", - code, - Some(&RE_PYTHON), - runnable_path, + debouncing_settings: debouncing_settings.maybe_fallback( + script.runnable_settings.debounce_key, + script.runnable_settings.debounce_delay_s, ), - Go => { - WorkspaceDependenciesAnnotatedRefs::parse("//", "go_mod", code, None, runnable_path) - } - Php => WorkspaceDependenciesAnnotatedRefs::parse( - "//", - "composer_json", - code, - None, - runnable_path, - ), - _ => return None, - } - } + }, + }) } -impl FromStr for ScriptLang { - type Err = Error; - fn from_str(s: &str) -> Result { - let language = match s.to_lowercase().as_str() { - "bun" => ScriptLang::Bun, - "bunnative" => ScriptLang::Bunnative, - "nativets" => ScriptLang::Nativets, - "deno" => ScriptLang::Deno, - "python3" => ScriptLang::Python3, - "go" => ScriptLang::Go, - "bash" => ScriptLang::Bash, - "powershell" => ScriptLang::Powershell, - "postgresql" => ScriptLang::Postgresql, - "mysql" => ScriptLang::Mysql, - "bigquery" => ScriptLang::Bigquery, - "snowflake" => ScriptLang::Snowflake, - "mssql" => ScriptLang::Mssql, - "graphql" => ScriptLang::Graphql, - "oracledb" => ScriptLang::OracleDB, - "php" => ScriptLang::Php, - "rust" => ScriptLang::Rust, - "ansible" => ScriptLang::Ansible, - "csharp" => ScriptLang::CSharp, - "nu" => ScriptLang::Nu, - "java" => ScriptLang::Java, - "ruby" => ScriptLang::Ruby, - // for related places search: ADD_NEW_LANG - language => { - return Err(anyhow::anyhow!("{} is currently not supported", language).into()) - } - }; - - Ok(language) - } -} - -#[derive(Eq, PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)] -#[sqlx(transparent)] -pub struct ScriptHash(pub i64); - -impl Deref for ScriptHash { - type Target = i64; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Into for ScriptHash { - fn into(self) -> u64 { - self.0 as u64 - } -} - -impl From for ScriptHash { - fn from(value: i64) -> Self { - Self(value) - } -} - -#[derive(PartialEq, sqlx::Type, Debug)] -#[sqlx(transparent, no_pg_array)] -pub struct ScriptHashes(pub Vec); - -impl Display for ScriptHash { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", to_hex_string(&self.0)) - } -} -impl Serialize for ScriptHash { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - serializer.serialize_str(to_hex_string(&self.0).as_str()) - } -} -impl<'de> Deserialize<'de> for ScriptHash { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - let i = to_i64(&s).map_err(|e| { - tracing::error!("Could not deserialize ScriptHash. Note, input should be in Hex and digit amount should be divisible by 16 (can be padded). err: {}", &e); - D::Error::custom(format!("{}", e)) - })?; - Ok(ScriptHash(i)) - } -} - -impl Serialize for ScriptHashes { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.0.len()))?; - for element in &self.0 { - seq.serialize_element(&ScriptHash(*element))?; - } - seq.end() - } -} - -#[derive(Serialize, Deserialize, Debug, Hash, sqlx::Type)] -#[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum ScriptKind { - Trigger, - Failure, - Script, - Approval, - Preprocessor, -} - -impl Display for ScriptKind { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_str(match self { - ScriptKind::Trigger => "trigger", - ScriptKind::Failure => "failure", - ScriptKind::Script => "script", - ScriptKind::Approval => "approval", - ScriptKind::Preprocessor => "preprocessor", - })?; - Ok(()) - } -} - -const PREVIEW_IS_CODEBASE_HASH: i64 = -42; -const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; -const PREVIEW_IS_ESM_CODEBASE_HASH: i64 = -44; -const PREVIEW_IS_TAR_ESM_CODEBASE_HASH: i64 = -45; - -pub fn is_special_codebase_hash(hash: i64) -> bool { - hash == PREVIEW_IS_CODEBASE_HASH - || hash == PREVIEW_IS_TAR_CODEBASE_HASH - || hash == PREVIEW_IS_ESM_CODEBASE_HASH - || hash == PREVIEW_IS_TAR_ESM_CODEBASE_HASH -} - -pub fn codebase_to_hash(is_tar: bool, is_esm: bool) -> i64 { - if is_tar { - if is_esm { - PREVIEW_IS_TAR_ESM_CODEBASE_HASH - } else { - PREVIEW_IS_TAR_CODEBASE_HASH - } - } else { - if is_esm { - PREVIEW_IS_ESM_CODEBASE_HASH - } else { - PREVIEW_IS_CODEBASE_HASH - } - } -} - -pub fn hash_to_codebase_id(job_id: &str, hash: i64) -> Option { - match hash { - PREVIEW_IS_CODEBASE_HASH => Some(job_id.to_string()), - PREVIEW_IS_TAR_CODEBASE_HASH => Some(format!("{}.tar", job_id)), - PREVIEW_IS_ESM_CODEBASE_HASH => Some(format!("{}.esm", job_id)), - PREVIEW_IS_TAR_ESM_CODEBASE_HASH => Some(format!("{}.esm.tar", job_id)), - _ => None, - } -} - -pub struct CodebaseInfo { - pub is_tar: bool, - pub is_esm: bool, -} - -pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { - let is_tar = id.ends_with(".tar"); - let is_esm = id.contains(".esm"); - CodebaseInfo { is_tar, is_esm } -} -#[derive(Serialize, sqlx::FromRow, Debug)] -pub struct Script { - pub workspace_id: String, - pub hash: ScriptHash, - pub path: String, - pub parent_hashes: Option, - pub summary: String, - pub description: String, - pub content: String, - pub created_by: String, - pub created_at: chrono::DateTime, - pub archived: bool, - pub schema: Option, - pub deleted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub is_template: Option, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub lock: Option, - pub lock_error_logs: Option, - pub language: ScriptLang, - pub kind: ScriptKind, - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub envs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub dedicated_worker: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ttl: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub cache_ignore_s3_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub restart_unless_cancelled: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub visible_to_runner_only: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub no_main_func: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub codebase: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub has_preprocessor: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub on_behalf_of_email: Option, - #[serde(skip_serializing_if = "Option::is_none")] - #[sqlx(json(nullable))] - pub assets: Option>, - #[serde(flatten)] - #[sqlx(flatten)] - pub runnable_settings: SR, -} - -// Not serializable -#[derive(sqlx::FromRow, Debug, Clone)] -pub struct ScriptRunnableSettingsHandle { - // legacy - for backwards compatibility - // don't add new values. - pub concurrency_key: Option, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub debounce_key: Option, - pub debounce_delay_s: Option, - - // add here as well. - pub runnable_settings_handle: Option, -} - -// Not sqlx queriable -#[derive(Serialize, Debug, Clone, Default)] -pub struct ScriptRunnableSettingsInline { - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettings, - #[serde(flatten)] - pub debouncing_settings: DebouncingSettings, -} - -impl Script { - pub async fn prefetch_cached<'a>( - self, - db: &DB, - ) -> crate::error::Result> { - let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - self.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; - - Ok(Script { - workspace_id: self.workspace_id, - hash: self.hash, - path: self.path, - parent_hashes: self.parent_hashes, - summary: self.summary, - description: self.description, - content: self.content, - created_by: self.created_by, - created_at: self.created_at, - archived: self.archived, - schema: self.schema, - deleted: self.deleted, - is_template: self.is_template, - extra_perms: self.extra_perms, - lock: self.lock, - lock_error_logs: self.lock_error_logs, - language: self.language, - kind: self.kind, - tag: self.tag, - draft_only: self.draft_only, - envs: self.envs, - dedicated_worker: self.dedicated_worker, - ws_error_handler_muted: self.ws_error_handler_muted, - priority: self.priority, - cache_ttl: self.cache_ttl, - cache_ignore_s3_path: self.cache_ignore_s3_path, - timeout: self.timeout, - delete_after_use: self.delete_after_use, - restart_unless_cancelled: self.restart_unless_cancelled, - visible_to_runner_only: self.visible_to_runner_only, - no_main_func: self.no_main_func, - codebase: self.codebase, - has_preprocessor: self.has_preprocessor, - on_behalf_of_email: self.on_behalf_of_email, - assets: self.assets, - runnable_settings: ScriptRunnableSettingsInline { - concurrency_settings: concurrency_settings.maybe_fallback( - self.runnable_settings.concurrency_key, - self.runnable_settings.concurrent_limit, - self.runnable_settings.concurrency_time_window_s, - ), - debouncing_settings: debouncing_settings.maybe_fallback( - self.runnable_settings.debounce_key, - self.runnable_settings.debounce_delay_s, - ), - }, - }) - } -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct ScriptWithStarred { - #[sqlx(flatten)] - #[serde(flatten)] - pub script: Script, - #[serde(skip_serializing_if = "Option::is_none")] - pub starred: Option, -} -impl ScriptWithStarred { - pub async fn prefetch_cached<'a>( - self, - db: &DB, - ) -> crate::error::Result> { - Ok(ScriptWithStarred { - script: self.script.prefetch_cached(db).await?, - starred: self.starred, - }) - } -} - -#[derive(Serialize, sqlx::FromRow)] -pub struct ListableScript { - pub hash: ScriptHash, - pub path: String, - pub summary: String, - pub created_at: chrono::DateTime, - pub archived: bool, - pub extra_perms: serde_json::Value, - pub language: ScriptLang, - pub starred: bool, - pub tag: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub has_draft: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub draft_only: Option, - pub has_deploy_errors: bool, - pub ws_error_handler_muted: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub no_main_func: Option, - #[serde(skip_serializing_if = "is_false")] - pub use_codebase: bool, - #[sqlx(default)] - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, - pub kind: ScriptKind, -} - -fn is_false(x: &bool) -> bool { - return !x; -} - -#[derive(Serialize)] -pub struct ScriptHistory { - pub script_hash: ScriptHash, - #[serde(skip_serializing_if = "Option::is_none")] - pub deployment_msg: Option, -} - -#[derive(Deserialize)] -pub struct ScriptHistoryUpdate { - pub deployment_msg: Option, -} - -#[derive(Serialize, Deserialize, Debug, sqlx::Type, Clone)] -#[sqlx(transparent)] -#[serde(transparent)] -pub struct Schema(pub sqlx::types::Json>); - -impl Hash for Schema { - fn hash(&self, state: &mut H) { - self.0.get().hash(state); - } -} - -#[derive(Serialize, Deserialize, Hash, Debug)] -pub struct NewScript { - pub path: String, - pub parent_hash: Option, - pub summary: String, - pub description: String, - pub content: String, - pub schema: Option, - pub is_template: Option, - #[serde(default = "Option::default")] - #[serde(deserialize_with = "lock_deserialize")] - pub lock: Option, - pub language: ScriptLang, - pub kind: Option, - pub tag: Option, - pub draft_only: Option, - pub envs: Option>, - #[serde(flatten)] - pub concurrency_settings: ConcurrencySettings, - #[serde(flatten)] - pub debouncing_settings: DebouncingSettings, - pub cache_ttl: Option, - pub cache_ignore_s3_path: Option, - pub dedicated_worker: Option, - pub ws_error_handler_muted: Option, - pub priority: Option, - pub timeout: Option, - pub delete_after_use: Option, - pub restart_unless_cancelled: Option, - pub deployment_message: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub visible_to_runner_only: Option, - pub no_main_func: Option, - pub codebase: Option, - pub has_preprocessor: Option, - pub on_behalf_of_email: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub assets: Option>, -} - -fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> -where - D: serde::de::Deserializer<'de>, -{ - struct StringOrArrayVisitor; - - impl<'de> serde::de::Visitor<'de> for StringOrArrayVisitor { - type Value = Option; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("either a string or an array of strings") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Ok(Some(v.to_string())) - } - - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - - fn visit_unit(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut split_lock: Vec = vec![]; - loop { - if let Ok(Some(elem)) = seq.next_element::() { - split_lock.push(elem); - } else { - break; - } - } - let lock = split_lock.join("\n"); - return Ok(Some(lock)); - } - } - deserializer.deserialize_any(StringOrArrayVisitor) -} - -#[derive(Debug, Deserialize)] -pub struct ListScriptQuery { - pub without_description: Option, - pub path_start: Option, - pub path_exact: Option, - pub created_by: Option, - pub first_parent_hash: Option, - pub last_parent_hash: Option, - pub parent_hash: Option, - pub show_archived: Option, - pub order_by: Option, - pub order_desc: Option, - pub is_template: Option, - pub kinds: Option, - pub starred_only: Option, - pub include_without_main: Option, - pub include_draft_only: Option, - pub with_deployment_msg: Option, - #[serde(default, deserialize_with = "from_seq")] - pub languages: Option>, - pub dedicated_worker: Option, -} - -fn from_seq<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - let s = ::deserialize(deserializer)?; - - let languages: Vec = s - .split(",") - .map(ScriptLang::from_str) - .try_collect() - .map_err(|e| serde::de::Error::custom(e.to_string()))?; - - let languages = if languages.is_empty() { - None - } else { - Some(languages) - }; - - Ok(languages) -} - -pub fn to_i64(s: &str) -> crate::error::Result { - let v = hex::decode(s)?; - if v.len() < 8 { - return Err(crate::error::Error::BadRequest(format!( - "hex string did not decode to an u64: {s}", - ))); - } - let nb: u64 = u64::from_be_bytes( - v[0..8] - .try_into() - .map_err(|_| hex::FromHexError::InvalidStringLength)?, - ); - Ok(nb as i64) -} - -pub fn to_hex_string(i: &i64) -> String { - hex::encode(i.to_be_bytes()) +pub async fn prefetch_cached_script_with_starred( + sws: ScriptWithStarred, + db: &DB, +) -> crate::error::Result> { + Ok(ScriptWithStarred { + script: prefetch_cached_script(sws.script, db).await?, + starred: sws.starred, + }) } pub async fn get_hub_script_by_path( @@ -896,21 +301,6 @@ async fn get_full_hub_script_by_path_inner( Ok(script) } -#[derive(Deserialize, Serialize)] -pub struct HubScript { - pub content: String, - pub lockfile: Option, - pub language: ScriptLang, - pub schema: Box, - pub summary: Option, -} - -pub fn hash_script(ns: &NewScript) -> i64 { - let mut dh = std::hash::DefaultHasher::new(); - ns.hash(&mut dh); - dh.finish() as i64 -} - pub async fn fetch_script_for_update<'a>( path: &str, w_id: &str, @@ -984,18 +374,18 @@ pub async fn clone_script<'c>( s } else { return Err(crate::error::Error::NotFound(format!( - "Non-archived script with path '{}' not found", path + "Non-archived script with path '{}' not found", + path ))); }; + let rs = runnable_settings::from_handle( + s.runnable_settings.runnable_settings_handle, + db, + ) + .await?; let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - s.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; + runnable_settings::prefetch_cached(&rs, db).await?; let ns = NewScript { path: s.path.clone(), diff --git a/backend/windmill-common/src/secret_backend/database.rs b/backend/windmill-common/src/secret_backend/database.rs index 3dee114481..b450cc7e15 100644 --- a/backend/windmill-common/src/secret_backend/database.rs +++ b/backend/windmill-common/src/secret_backend/database.rs @@ -59,9 +59,8 @@ impl SecretBackend for DatabaseBackend { } let mc = build_crypt(&self.db, workspace_id).await?; - decrypt(&mc, value).map_err(|e| { - Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) - }) + decrypt(&mc, value) + .map_err(|e| Error::internal_err(format!("Error decrypting variable {}: {}", path, e))) } async fn set_secret(&self, workspace_id: &str, path: &str, value: &str) -> Result<()> { diff --git a/backend/windmill-common/src/secret_backend/vault_oss.rs b/backend/windmill-common/src/secret_backend/vault_oss.rs index b338388312..eba15bff67 100644 --- a/backend/windmill-common/src/secret_backend/vault_oss.rs +++ b/backend/windmill-common/src/secret_backend/vault_oss.rs @@ -16,7 +16,10 @@ use std::sync::Arc; use crate::db::DB; use crate::error::{Error, Result}; -use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings}; +use super::{ + database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, + VaultSettings, +}; /// Stub VaultBackend for OSS - all operations return EE required error pub struct VaultBackend; diff --git a/backend/windmill-common/src/stats_oss.rs b/backend/windmill-common/src/stats_oss.rs index 4e388daf5c..161c78f81b 100644 --- a/backend/windmill-common/src/stats_oss.rs +++ b/backend/windmill-common/src/stats_oss.rs @@ -50,3 +50,19 @@ pub async fn get_user_usage<'c, E: sqlx::Executor<'c, Database = Postgres>>( let usage = ActiveUserUsage { author_count: None, operator_count: None }; Ok(usage) } + +#[cfg(not(feature = "private"))] +#[derive(serde::Serialize)] +pub struct Stats {} + +#[cfg(not(feature = "private"))] +pub async fn get_stats_payload(_db: &DB, _reason: &SendStatsReason) -> Result { + // stats details are closed source + Ok(Stats {}) +} + +#[cfg(not(feature = "private"))] +pub fn encrypt_stats(_stats: &Stats) -> Result { + // stats details are closed source + Ok(String::new()) +} diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 950f13c185..701a0887d0 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -47,8 +47,8 @@ pub const VERBOSE_TARGET: &str = "windmill_verbose"; /// Creates a Targets filter that optionally filters out verbose logs when quiet mode is enabled. fn create_targets_filter(default_env_filter: LevelFilter) -> Targets { - let targets = Targets::new() - .with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF); + let targets = + Targets::new().with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF); if *QUIET_MODE { targets diff --git a/backend/windmill-common/src/triggers.rs b/backend/windmill-common/src/triggers.rs index 6a5d2e4f45..59b827a8cb 100644 --- a/backend/windmill-common/src/triggers.rs +++ b/backend/windmill-common/src/triggers.rs @@ -1,101 +1,9 @@ use lazy_static::lazy_static; use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use std::fmt; -use strum_macros::EnumIter; -use crate::jobs::JobTriggerKind; - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, EnumIter)] -#[sqlx(type_name = "TRIGGER_KIND", rename_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum TriggerKind { - Webhook, - Http, - Websocket, - Kafka, - DefaultEmail, - Email, - Nats, - Mqtt, - Sqs, - Postgres, - Gcp, - Nextcloud, -} - -impl TriggerKind { - pub fn to_key(&self) -> String { - match self { - TriggerKind::Webhook => "webhook".to_string(), - TriggerKind::Http => "http".to_string(), - TriggerKind::Websocket => "websocket".to_string(), - TriggerKind::Kafka => "kafka".to_string(), - TriggerKind::Email => "email".to_string(), - TriggerKind::DefaultEmail => "email".to_string(), // to the user we also show kind email for default email - TriggerKind::Nats => "nats".to_string(), - TriggerKind::Mqtt => "mqtt".to_string(), - TriggerKind::Sqs => "sqs".to_string(), - TriggerKind::Postgres => "postgres".to_string(), - TriggerKind::Gcp => "gcp".to_string(), - TriggerKind::Nextcloud => "nextcloud".to_string(), - } - } -} - -impl fmt::Display for TriggerKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - TriggerKind::Webhook => "webhook", - TriggerKind::Http => "http", - TriggerKind::Websocket => "websocket", - TriggerKind::Kafka => "kafka", - TriggerKind::Email => "email", - TriggerKind::DefaultEmail => "default_email", - TriggerKind::Nats => "nats", - TriggerKind::Mqtt => "mqtt", - TriggerKind::Sqs => "sqs", - TriggerKind::Postgres => "postgres", - TriggerKind::Gcp => "gcp", - TriggerKind::Nextcloud => "nextcloud", - }; - write!(f, "{}", s) - } -} - -#[derive(Eq, PartialEq, Hash)] -pub enum HubOrWorkspaceId { - Hub, - WorkspaceId(String), -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] -pub struct RunnableFormat { - pub version: RunnableFormatVersion, - pub has_preprocessor: bool, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] -pub enum RunnableFormatVersion { - V1, - V2, -} - -pub type RunnableFormatCacheKey = (HubOrWorkspaceId, i64, TriggerKind); +pub use windmill_types::triggers::*; lazy_static! { pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache = Cache::new(1000); } - -#[derive(Debug, Clone)] -pub struct TriggerMetadata { - pub trigger_path: Option, - pub trigger_kind: JobTriggerKind, -} - -impl TriggerMetadata { - pub fn new(trigger_path: Option, trigger_kind: JobTriggerKind) -> TriggerMetadata { - TriggerMetadata { trigger_path, trigger_kind } - } -} diff --git a/backend/windmill-common/src/usernames.rs b/backend/windmill-common/src/usernames.rs new file mode 100644 index 0000000000..645ee90e51 --- /dev/null +++ b/backend/windmill-common/src/usernames.rs @@ -0,0 +1,97 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use regex::Regex; +use sqlx::{Postgres, Transaction}; + +use crate::error::{self, Error}; + +lazy_static::lazy_static! { + pub static ref INVALID_USERNAME_CHARS: Regex = Regex::new(r"[^A-Za-z0-9_]").unwrap(); + pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); +} + +pub async fn generate_instance_wide_unique_username<'c>( + tx: &mut Transaction<'c, Postgres>, + email: &str, +) -> error::Result { + let mut username = email.split('@').next().unwrap().to_string(); + + username = INVALID_USERNAME_CHARS + .replace_all(&mut username, "") + .to_string(); + + if username.is_empty() { + username = "user".to_string() + } + + let base_username = username.clone(); + let mut username_conflict = true; + let mut i = 1; + while username_conflict { + if i > 1000 { + return Err(Error::internal_err(format!( + "too many username conflicts for {}", + email + ))); + } + if i > 1 { + username = format!("{}{}", base_username, i) + } + username_conflict = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", + &username, + &email + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + i += 1; + } + + Ok(username) +} + +pub async fn get_instance_username_or_create_pending<'c>( + tx: &mut Transaction<'c, Postgres>, + email: &str, +) -> error::Result { + let user = sqlx::query_scalar!("SELECT username FROM password WHERE email = $1", email) + .fetch_optional(&mut **tx) + .await?; + + if let Some(opt_username) = user { + if let Some(username) = opt_username { + Ok(username) + } else { + Err(Error::BadRequest(format!("No instance-wide username found for {email}. The user has different usernames for different workspaces. Ask the instance administrator to solve the conflict in the instance settings."))) + } + } else { + let pending_username = + sqlx::query_scalar!("SELECT username FROM pending_user WHERE email = $1", email) + .fetch_optional(&mut **tx) + .await?; + + if let Some(username) = pending_username { + Ok(username) + } else { + let username = generate_instance_wide_unique_username(&mut *tx, email).await?; + + sqlx::query!( + "INSERT INTO pending_user (email, username) VALUES ($1, $2)", + email, + username + ) + .execute(&mut **tx) + .await + .map_err(|e| Error::internal_err(format!("creating pending user: {e:#}")))?; + + Ok(username) + } + } +} diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 4b2794c399..4a6f4604a1 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -51,6 +51,9 @@ use std::sync::atomic::Ordering; use crate::worker::CLOUD_HOSTED; lazy_static::lazy_static! { + pub static ref COOKIE_DOMAIN: Option = std::env::var("COOKIE_DOMAIN").ok(); + pub static ref IS_SECURE: Arc> = Arc::new(RwLock::new(false)); + pub static ref FORCE_IPV4: bool = std::env::var("FORCE_IPV4") .map(|v| v.to_lowercase() == "true" || v == "1") .unwrap_or(false); @@ -163,6 +166,16 @@ pub struct Pagination { pub per_page: Option, } +#[derive(Deserialize)] +pub struct WithStarredInfoQuery { + pub with_starred_info: Option, +} + +#[derive(Deserialize)] +pub struct BulkDeleteRequest { + pub paths: Vec, +} + #[derive(Debug, Serialize, Deserialize)] pub struct StripPath(pub String); @@ -964,6 +977,14 @@ pub async fn get_custom_pg_instance_password(db: &DB) -> Result { ) } +/// Convert a JSON string to a `Box` without validation. +/// +/// # Safety +/// The caller must ensure the string is valid JSON. +pub fn unsafe_raw(json: String) -> Box { + unsafe { std::mem::transmute::, Box>(json.into()) } +} + // Avoid JSON parsing for merging raw JSON values into an object pub fn merge_raw_values_to_object( pairs: &[(String, Box)], diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs new file mode 100644 index 0000000000..f1f8508f74 --- /dev/null +++ b/backend/windmill-common/src/webhook.rs @@ -0,0 +1,292 @@ +use std::time::Duration; + +use quick_cache::sync::Cache; +use serde::Serialize; +use tokio::{select, sync::mpsc}; + +#[cfg(feature = "prometheus")] +use crate::METRICS_ENABLED; + +use crate::db::DB; +use crate::oauth2::InstanceEvent; +use crate::utils::configure_client; + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens. + static ref WEBHOOK_REQUEST_COUNT: prometheus::Histogram = prometheus::register_histogram!( + "webhook_request", + "Histogram of webhook requests made" + ) + .unwrap(); + +} + +lazy_static::lazy_static! { + + pub static ref INSTANCE_EVENTS_WEBHOOK: Option = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok(); + + pub static ref WEBHOOK_CACHE: Cache> = Cache::new(100); + +} + +pub enum WebhookPayload { + WorkspaceEvent(String, WebhookMessage), + InstanceEvent(InstanceEvent), +} + +#[derive(Serialize)] +#[serde(tag = "type")] +pub enum WebhookMessage { + // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON + CreateApp { workspace: String, path: String }, + DeleteApp { workspace: String, path: String }, + UpdateApp { workspace: String, old_path: String, new_path: String }, + CreateFlow { workspace: String, path: String }, + UpdateFlow { workspace: String, old_path: String, new_path: String }, + ArchiveFlow { workspace: String, path: String }, + DeleteFlow { workspace: String, path: String }, + CreateFolder { workspace: String, name: String }, + UpdateFolder { workspace: String, name: String }, + DeleteFolder { workspace: String, name: String }, + DeleteResource { workspace: String, path: String }, + CreateResource { workspace: String, path: String }, + UpdateResource { workspace: String, old_path: String, new_path: String }, + CreateResourceType { name: String }, + DeleteResourceType { name: String }, + UpdateResourceType { name: String }, + CreateScript { workspace: String, path: String, hash: String }, + UpdateScript { workspace: String, path: String, hash: String }, + DeleteScript { workspace: String, hash: String }, + DeleteScriptPath { workspace: String, path: String }, + CreateVariable { workspace: String, path: String }, + UpdateVariable { workspace: String, old_path: String, new_path: String }, + DeleteVariable { workspace: String, path: String }, +} + +#[derive(Clone)] +pub struct WebhookShared { + pub channel: mpsc::UnboundedSender, +} + +impl WebhookShared { + pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self { + let (tx, mut rx) = mpsc::unbounded_channel::(); + let _process = tokio::spawn(async move { + let client = configure_client( + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + // TODO: investigate pool timeouts and such if TCP load is high + .timeout(Duration::from_secs(5)), + ) + .build() + .unwrap(); + + loop { + select! { + biased; + _ = shutdown_rx.recv() => break, + r = rx.recv() => match r { + Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => { + let webhook_opt = match WEBHOOK_CACHE.get(&workspace_id) { + Some(guard) => { + guard + }, + None => { + let Ok(mut webhook_opt) = + sqlx::query_scalar!( + "SELECT webhook FROM workspace_settings WHERE workspace_id = $1", + workspace_id + ) + .fetch_one( + &db, + ) + .await else { + tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}"); + continue; + }; + if webhook_opt.as_ref().is_some_and(|x| x.is_empty()) { + webhook_opt = None; + } + WEBHOOK_CACHE.insert(workspace_id, webhook_opt.clone()); + webhook_opt + } + }; + if let Some(url) = webhook_opt { + #[cfg(feature = "prometheus")] + let timer = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; + tracing::info!("Sending webhook message to {}", url); + let _ = client.post(url).json(&message).send().await; + #[cfg(feature = "prometheus")] + timer.map(|x| x.stop_and_record()); + } + }, + Some(WebhookPayload::InstanceEvent(event)) => { + #[cfg(feature = "prometheus")] + if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; + let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await; + if let Err(e) = r { + tracing::error!("Error sending instance event: {}", e); + } + }, + None => break, + }, + } + } + }); + + Self { channel: tx } + } + + pub fn send_message(&self, workspace_id: String, message: WebhookMessage) { + let _ = self.channel.send(WebhookPayload::WorkspaceEvent( + workspace_id.clone(), + message, + )); + } + + pub fn send_instance_event(&self, event: InstanceEvent) { + if INSTANCE_EVENTS_WEBHOOK.is_none() { + return; + } + let _ = self.channel.send(WebhookPayload::InstanceEvent(event)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_webhook_message_create_script() { + let msg = WebhookMessage::CreateScript { + workspace: "demo".to_string(), + path: "f/test/script".to_string(), + hash: "abc123".to_string(), + }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "CreateScript"); + assert_eq!(json["workspace"], "demo"); + assert_eq!(json["path"], "f/test/script"); + assert_eq!(json["hash"], "abc123"); + } + + #[test] + fn test_webhook_message_update_flow() { + let msg = WebhookMessage::UpdateFlow { + workspace: "staging".to_string(), + old_path: "f/old/flow".to_string(), + new_path: "f/new/flow".to_string(), + }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "UpdateFlow"); + assert_eq!(json["old_path"], "f/old/flow"); + assert_eq!(json["new_path"], "f/new/flow"); + } + + #[test] + fn test_webhook_message_delete_resource() { + let msg = WebhookMessage::DeleteResource { + workspace: "prod".to_string(), + path: "u/admin/db".to_string(), + }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "DeleteResource"); + assert_eq!(json["workspace"], "prod"); + assert_eq!(json["path"], "u/admin/db"); + } + + #[test] + fn test_webhook_message_create_folder() { + let msg = WebhookMessage::CreateFolder { + workspace: "demo".to_string(), + name: "shared".to_string(), + }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "CreateFolder"); + assert_eq!(json["name"], "shared"); + } + + #[test] + fn test_webhook_message_resource_type() { + let msg = WebhookMessage::CreateResourceType { name: "postgresql".to_string() }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "CreateResourceType"); + assert_eq!(json["name"], "postgresql"); + // Should NOT have workspace field + assert!(json.get("workspace").is_none()); + } + + #[test] + fn test_webhook_message_all_variants_have_type_tag() { + let messages: Vec = vec![ + WebhookMessage::CreateApp { workspace: "w".into(), path: "p".into() }, + WebhookMessage::DeleteApp { workspace: "w".into(), path: "p".into() }, + WebhookMessage::UpdateApp { + workspace: "w".into(), + old_path: "o".into(), + new_path: "n".into(), + }, + WebhookMessage::CreateFlow { workspace: "w".into(), path: "p".into() }, + WebhookMessage::UpdateFlow { + workspace: "w".into(), + old_path: "o".into(), + new_path: "n".into(), + }, + WebhookMessage::ArchiveFlow { workspace: "w".into(), path: "p".into() }, + WebhookMessage::DeleteFlow { workspace: "w".into(), path: "p".into() }, + WebhookMessage::CreateFolder { workspace: "w".into(), name: "n".into() }, + WebhookMessage::UpdateFolder { workspace: "w".into(), name: "n".into() }, + WebhookMessage::DeleteFolder { workspace: "w".into(), name: "n".into() }, + WebhookMessage::DeleteResource { workspace: "w".into(), path: "p".into() }, + WebhookMessage::CreateResource { workspace: "w".into(), path: "p".into() }, + WebhookMessage::UpdateResource { + workspace: "w".into(), + old_path: "o".into(), + new_path: "n".into(), + }, + WebhookMessage::CreateResourceType { name: "n".into() }, + WebhookMessage::DeleteResourceType { name: "n".into() }, + WebhookMessage::UpdateResourceType { name: "n".into() }, + WebhookMessage::CreateScript { + workspace: "w".into(), + path: "p".into(), + hash: "h".into(), + }, + WebhookMessage::UpdateScript { + workspace: "w".into(), + path: "p".into(), + hash: "h".into(), + }, + WebhookMessage::DeleteScript { workspace: "w".into(), hash: "h".into() }, + WebhookMessage::DeleteScriptPath { workspace: "w".into(), path: "p".into() }, + WebhookMessage::CreateVariable { workspace: "w".into(), path: "p".into() }, + WebhookMessage::UpdateVariable { + workspace: "w".into(), + old_path: "o".into(), + new_path: "n".into(), + }, + WebhookMessage::DeleteVariable { workspace: "w".into(), path: "p".into() }, + ]; + + for msg in &messages { + let json = serde_json::to_value(msg).unwrap(); + assert!( + json.get("type").is_some(), + "Missing 'type' tag in: {}", + serde_json::to_string(msg).unwrap() + ); + } + } + + #[test] + fn test_webhook_message_type_tags_are_variant_names() { + let msg = WebhookMessage::CreateApp { workspace: "w".into(), path: "p".into() }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "CreateApp"); + + let msg = WebhookMessage::DeleteVariable { workspace: "w".into(), path: "p".into() }; + let json = serde_json::to_value(&msg).unwrap(); + assert_eq!(json["type"], "DeleteVariable"); + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 0b4b25472c..4a12ad00a4 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1045,16 +1045,17 @@ pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { } #[cfg(all(feature = "enterprise", feature = "parquet"))] fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { - use std::fs::{File, Permissions}; + use std::fs::File; use std::io::Write; - #[cfg(unix)] - use std::os::unix::fs::PermissionsExt; - let mut file = File::create(main_path)?; file.write_all(byts)?; #[cfg(unix)] - file.set_permissions(Permissions::from_mode(0o755))?; + { + use std::fs::Permissions; + use std::os::unix::fs::PermissionsExt; + file.set_permissions(Permissions::from_mode(0o755))?; + } file.flush()?; Ok(()) } diff --git a/backend/windmill-common/src/workspace_dependencies.rs b/backend/windmill-common/src/workspace_dependencies.rs index 12be155668..68f20967f4 100644 --- a/backend/windmill-common/src/workspace_dependencies.rs +++ b/backend/windmill-common/src/workspace_dependencies.rs @@ -178,6 +178,13 @@ impl WorkspaceDependencies { ?name, "fetching latest workspace dependencies id" ); + + // Bunnative and Nativets workspace dependencies go under Bun language + let language = match language { + ScriptLang::Nativets | ScriptLang::Bunnative => ScriptLang::Bun, + l => l, + }; + let result = sqlx::query_scalar!( r#" SELECT id FROM workspace_dependencies @@ -209,13 +216,13 @@ impl WorkspaceDependencies { workspace_id: &str, conn: Connection, ) -> error::Result> { - if language.as_dependencies_filename().is_none() { + let Some(dependencies_filename) = language.as_dependencies_filename() else { return Ok(None); - } + }; if name.is_none() && get_cached_is_unnamed_workspace_dependencies_exists( - language, + dependencies_filename.clone(), workspace_id.to_owned(), ) .map(|exists| exists == false) @@ -237,7 +244,7 @@ impl WorkspaceDependencies { else { if name.is_none() { set_cached_is_unnamed_workspace_dependencies_exists( - language, + dependencies_filename.clone(), workspace_id.to_owned(), false, ); @@ -277,7 +284,7 @@ impl WorkspaceDependencies { if name.is_none() { set_cached_is_unnamed_workspace_dependencies_exists( - language, + dependencies_filename, workspace_id.to_owned(), wd.is_some(), ); @@ -379,7 +386,7 @@ impl WorkspaceDependenciesPrefetched { Box::pin(async { let r = if let Some(wdar) = - language.extract_workspace_dependencies_annotated_refs(code, runnable_path) + crate::scripts::extract_workspace_dependencies_annotated_refs(&language, code, runnable_path) { tracing::debug!(workspace_id, ?language, "found explicit annotations"); diff --git a/backend/windmill-dep-map/Cargo.toml b/backend/windmill-dep-map/Cargo.toml new file mode 100644 index 0000000000..ae81f5d1a7 --- /dev/null +++ b/backend/windmill-dep-map/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "windmill-dep-map" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_dep_map" +path = "src/lib.rs" + +[features] +default = [] +python = ["dep:windmill-parser-py-imports"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-py-imports = { workspace = true, optional = true } +sqlx.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +lazy_static.workspace = true +chrono.workspace = true +itertools.workspace = true +uuid.workspace = true diff --git a/backend/windmill-dep-map/src/lib.rs b/backend/windmill-dep-map/src/lib.rs new file mode 100644 index 0000000000..39f2413bf7 --- /dev/null +++ b/backend/windmill-dep-map/src/lib.rs @@ -0,0 +1,203 @@ +pub mod scoped_dependency_map; +pub mod trigger_dependents; +pub mod workspace_dependencies; + +use std::collections::HashMap; +use std::path::{Component, Path, PathBuf}; + +use serde_json::value::RawValue; +use sqlx::types::Json; +use uuid::Uuid; +use windmill_common::error; +use windmill_common::scripts::ScriptLang; +use windmill_common::utils::WarnAfterExt; +use windmill_common::workspace_dependencies::{ + WorkspaceDependencies, WorkspaceDependenciesPrefetched, +}; +use windmill_parser_ts::parse_expr_for_imports; + +fn try_normalize(path: &Path) -> Option { + let mut ret = PathBuf::new(); + + for component in path.components() { + match component { + Component::Prefix(..) | Component::RootDir => return None, + Component::CurDir => {} + Component::ParentDir => { + if !ret.pop() { + return None; + } + } + Component::Normal(c) => { + ret.push(c); + } + } + } + + Some(ret) +} + +fn parse_ts_relative_imports( + raw_code: &str, + script_path: &str, +) -> windmill_common::error::Result> { + let mut relative_imports = vec![]; + let r = parse_expr_for_imports(raw_code, true)?; + for import in r { + let import = import.trim_end_matches(".ts"); + if import.starts_with("/") { + relative_imports.push(import.trim_start_matches("/").to_string()); + } else if import.starts_with(".") { + let normalized = try_normalize(std::path::Path::new(&format!( + "{}/../{}", + script_path, import + ))); + if let Some(normalized) = normalized { + let normalized = normalized.to_str().unwrap().to_string(); + relative_imports.push(normalized); + } else { + tracing::error!("error canonicalizing path: {script_path} with import {import}"); + } + } + } + + Ok(relative_imports) +} + +pub fn extract_relative_imports( + raw_code: &str, + script_path: &str, + language: &Option, +) -> Option> { + match language { + #[cfg(feature = "python")] + Some(ScriptLang::Python3) => { + windmill_parser_py_imports::parse_relative_imports(&raw_code, script_path).ok() + } + Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) | Some(ScriptLang::Deno) => { + parse_ts_relative_imports(&raw_code, script_path).ok() + } + _ => None, + } +} + +pub fn extract_referenced_paths( + raw_code: &str, + script_path: &str, + language: Option, +) -> Option> { + let mut referenced_paths = vec![]; + if let Some(wk_deps_refs) = language + .and_then(|l| windmill_common::scripts::extract_workspace_dependencies_annotated_refs(&l, raw_code, script_path)) + .map(|r| r.external) + { + let l = language.expect("should be some"); + for wk_deps_ref in wk_deps_refs { + if let Some(path) = WorkspaceDependencies::to_path(&Some(wk_deps_ref), l).ok() { + referenced_paths.push(path); + }; + } + } else if let (Some(l), true /* Only if it is not blacklisted */) = ( + language, + WorkspaceDependenciesPrefetched::is_external_references_permitted(script_path), + ) { + // we assume all runnables without annotated dependencies reference default dependencies file. + WorkspaceDependencies::to_path(&None, l) + .ok() + .inspect(|p| referenced_paths.push(p.to_owned())); + } + + if let Some(relative_imports) = extract_relative_imports(raw_code, script_path, &language) { + referenced_paths.extend(relative_imports); + } + + if referenced_paths.is_empty() { + None + } else { + Some(referenced_paths) + } +} + +pub async fn process_relative_imports( + db: &sqlx::Pool, + _job_id: Option, + args: Option<&Json>>>, + w_id: &str, + script_path: &str, + parent_path: Option, + deployment_message: Option, + code: &str, + script_lang: &Option, + permissioned_as_email: &str, + created_by: &str, + permissioned_as: &str, +) -> error::Result<()> { + use scoped_dependency_map::ScopedDependencyMap; + use trigger_dependents::trigger_dependents_to_recompute_dependencies; + + // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled + { + let mut tx = db.begin().await?; + let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( + &w_id, + script_path, + "script", + &parent_path, + db, + ) + .await?; + + tx = dependency_map + .patch( + extract_referenced_paths(&code, script_path, *script_lang), + // Ideally should be None, but due to current implementation will use empty string to represent None. + "".into(), + tx, + ) + .await?; + + dependency_map.dissolve(tx).await.commit().await?; + } + + { + let mut already_visited = args + .map(|x| { + x.get("already_visited") + .map(|v| serde_json::from_str::>(v.get()).ok()) + .flatten() + }) + .flatten() + .unwrap_or_default(); + + let importers = ScopedDependencyMap::get_dependents(script_path, w_id, db).await?; + + already_visited.push(script_path.to_string()); + match tokio::time::timeout( + core::time::Duration::from_secs(60), + Box::pin(trigger_dependents_to_recompute_dependencies( + w_id, + importers, + deployment_message, + parent_path, + permissioned_as_email, + created_by, + permissioned_as, + db, + already_visited, + )), + ) + .warn_after_seconds(10) + .await + { + Ok(Err(e)) => { + tracing::error!(%e, "error triggering dependents to recompute dependencies") + } + Err(e) => { + tracing::error!(%e, "triggering dependents to recompute dependencies has timed out") + } + _ => {} + } + } + + Ok(()) +} diff --git a/backend/windmill-worker/src/scoped_dependency_map.rs b/backend/windmill-dep-map/src/scoped_dependency_map.rs similarity index 86% rename from backend/windmill-worker/src/scoped_dependency_map.rs rename to backend/windmill-dep-map/src/scoped_dependency_map.rs index 9e9884e538..4770fd516d 100644 --- a/backend/windmill-worker/src/scoped_dependency_map.rs +++ b/backend/windmill-dep-map/src/scoped_dependency_map.rs @@ -11,14 +11,10 @@ use windmill_common::{ use std::collections::HashSet; -use crate::worker_lockfiles::extract_referenced_paths; - -// TODO: To be removed in future versions lazy_static::lazy_static! { pub static ref WMDEBUG_NO_DMAP_DISSOLVE: bool = std::env::var("WMDEBUG_NO_DMAP_DISSOLVE").is_ok(); } -// TODO: Rename to DependencyRelation #[derive(Serialize)] pub struct DependencyMap { pub workspace_id: Option, @@ -48,7 +44,7 @@ impl ScopedDependencyMap { /// Calls DB, however is assumed to be called once per dependency job /// AND is scoped to smaller subset of data /// So it is not too expensive - pub(crate) async fn fetch_maybe_rearranged<'a>( + pub async fn fetch_maybe_rearranged<'a>( w_id: &str, importer_path: &str, importer_kind: &str, @@ -123,7 +119,7 @@ SELECT importer_node_id, imported_path /// Add missing entries to `dependency_map` /// Remove matching entries - pub(crate) async fn patch<'c>( + pub async fn patch<'c>( &mut self, referenced_paths: Option>, node_id: String, // Flow Step/Node ID @@ -134,7 +130,7 @@ SELECT importer_node_id, imported_path Ok(tx) } - pub(crate) async fn patch_tx_ref<'c>( + pub async fn patch_tx_ref<'c>( &mut self, // NOTE: Referenced_paths should include all of the paths. referenced_paths: Option>, @@ -150,26 +146,12 @@ SELECT importer_node_id, imported_path return Ok(()); }; - // This does: - // 1. remove all relative imports from relative_imports that ARE tracked in dependency_map - // 2. remove corresponding trackers from dependency_map - // - // After this operation `relative_imports` variable has only untracked imports. - // We will handle those in the next expression. - // - // After all `reduce`'s called ScopedDependencyMap has only extra/orphan imports - // these are going to be clean up by calling [dissolve] - // NOTE: `retain` iterates over vec and remove the ones whose closures returned false. referenced_paths.retain(|imported_path| { !self .to_delete - // As dmap is HashSet, removing is O(1) operation - // thus making entire process very efficient - // NOTE: `remove` returns true if item was removed and false if wasn't. .remove(&(node_id.to_owned(), imported_path.to_owned())) }); - // As mentioned above, usually this will always be empty. if !referenced_paths.is_empty() { tracing::info!("adding missing entries to dependency_map: importer_node_id - {}, importer_kind - {}, new_imported_paths - {:?}", &node_id, @@ -197,7 +179,7 @@ SELECT importer_node_id, imported_path } /// clean orphan entries from `dependency_map` - pub(crate) async fn dissolve<'a>( + pub async fn dissolve<'a>( self, mut tx: sqlx::Transaction<'a, sqlx::Postgres>, ) -> sqlx::Transaction<'a, sqlx::Postgres> { @@ -210,7 +192,6 @@ SELECT importer_node_id, imported_path tracing::info!("dissolving dependency_map: {:?}", &self); - // We _could_ shove it into single query, but this query is rarely called AND let's keep it simple for redability. for (importer_node_id, imported_path) in self.to_delete.into_iter() { tracing::info!("cleaning orphan entry from dependency_map: importer_kind - {}, imported_path - {}, importer_node_id - {}", &self.importer_kind, @@ -218,7 +199,6 @@ SELECT importer_node_id, imported_path &importer_node_id, ); - // Dissolve MUST succeed. Error in dissolve MUST not block the execution. if let Err(err) = sqlx::query!( " DELETE FROM dependency_map @@ -265,7 +245,6 @@ SELECT importer_node_id, imported_path "discovered orphan entry in `dependency_map`. It will be healed automatically, however please report this issue to Windmill Team. It is also advised to rebuild maps in workspace settings in troubleshooting.", ); - // MUST succeed. Error MUST not block the execution. if let Err(err) = sqlx::query!( "DELETE FROM dependency_map WHERE importer_path = $1 AND importer_kind = $3::text::IMPORTER_KIND @@ -286,7 +265,7 @@ SELECT importer_node_id, imported_path tx } - pub(crate) async fn rebuild_map_unchecked<'c>( + pub async fn rebuild_map_unchecked( w_id: &str, db: &sqlx::Pool, ) -> Result { @@ -305,7 +284,7 @@ SELECT importer_node_id, imported_path tx = dmap .patch( - extract_referenced_paths(&sd.code, &r.path, smd.language), + crate::extract_referenced_paths(&sd.code, &r.path, smd.language), "".into(), tx, ) @@ -318,18 +297,13 @@ SELECT importer_node_id, imported_path } // Fetch only top level versions and paths - // It is not fetching value tracing::info!(workspace_id = w_id, "Rebuilding dependency map for flows"); for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM flow WHERE workspace_id = $1 AND archived = false", w_id).fetch_all(db).await? { if let Some(version) = r.version { - // To reduce stress on db try to fetch from cache - // Since our flow versions are immutable it is safe to assume if we have cache for specific version/id it is up to date. let flow_data = cache::flow::fetch_version(&db.clone().into(), version).await?; - // Create map for specific flow let mut dmap = ScopedDependencyMap::fetch(w_id, &r.path, "flow", db).await?; - // Traverse retrieved flow modules let mut tx = db.begin().await?; let mut to_process = vec![]; let mut modules_to_check = flow_data.flow.modules.iter().collect::>(); @@ -342,10 +316,9 @@ SELECT importer_node_id, imported_path FlowValue::traverse_leafs(modules_to_check, &mut |fmv, id| { match fmv { - // Since we fetched from flow_version it is safe to assume all inline scripts are in form of RawScript. FlowModuleValue::RawScript { content, language, .. } => { to_process.push(( - extract_referenced_paths( + crate::extract_referenced_paths( content, &(r.path.clone() + "/flow"), Some(*language), @@ -353,10 +326,8 @@ SELECT importer_node_id, imported_path id.clone(), )); } - // But just in case we will also handle other cases. FlowModuleValue::FlowScript { .. } => { - // Abort will cancel transaction. - return Err(Error::internal_err("FlowScript is not supposed to be in flow.")); + return Err(Error::internal_err("FlowScript is not supposed to be in flow.").into()); } _ => {} } @@ -382,7 +353,6 @@ SELECT importer_node_id, imported_path tracing::info!(workspace_id = w_id, "Rebuilding dependency map for apps"); for r in sqlx::query!("SELECT path, versions[array_upper(versions, 1)] as version FROM app WHERE workspace_id = $1", w_id).fetch_all(db).await? { if let Some(version) = r.version { - // TODO: Use cache when implemented. let value = sqlx::query_scalar!( "SELECT value FROM app_version WHERE id = $1 LIMIT 1", version @@ -395,7 +365,7 @@ SELECT importer_node_id, imported_path let mut to_process = vec![]; traverse_app_inline_scripts(&value, None, &mut |ais, id| { to_process.push(( - extract_referenced_paths( + crate::extract_referenced_paths( &ais.content, &(r.path.clone() + "/app"), ais.language, @@ -423,6 +393,7 @@ SELECT importer_node_id, imported_path Ok("Success".into()) } + /// Run if you want to rebuild maps on specific workspace. /// Potentially takes much time pub async fn rebuild_map(w_id: &str, db: &sqlx::Pool) -> Result { @@ -459,11 +430,11 @@ SELECT importer_node_id, imported_path sqlx::query_as!( DependencyDependent, r#" - SELECT + SELECT importer_path, - importer_kind::text as "importer_kind!", -- sqlx thinks this is nullable somehow, so enfore with ! + importer_kind::text as "importer_kind!", array_agg(importer_node_id) as importer_node_ids - FROM dependency_map + FROM dependency_map WHERE workspace_id = $1 AND imported_path = $2 GROUP BY importer_path, importer_kind "#, diff --git a/backend/windmill-dep-map/src/trigger_dependents.rs b/backend/windmill-dep-map/src/trigger_dependents.rs new file mode 100644 index 0000000000..d2e3fd56dd --- /dev/null +++ b/backend/windmill-dep-map/src/trigger_dependents.rs @@ -0,0 +1,220 @@ +use std::collections::HashMap; + +use chrono::{Duration, Utc}; +use itertools::Itertools; +use serde_json::value::RawValue; +use windmill_common::error; +use windmill_common::jobs::JobPayload; +use windmill_common::runnable_settings::DebouncingSettings; +use windmill_common::scripts::ScriptHash; +use windmill_common::worker::to_raw_value; +use windmill_queue::PushIsolationLevel; + +use crate::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; + +lazy_static::lazy_static! { + static ref DEPENDENCY_JOB_DEBOUNCE_DELAY: usize = std::env::var("DEPENDENCY_JOB_DEBOUNCE_DELAY").ok().and_then(|flag| flag.parse().ok()).unwrap_or( + if cfg!(test) { 15 } else { 5 } + ); +} + +pub async fn trigger_dependents_to_recompute_dependencies( + w_id: &str, + importers: Vec, + deployment_message: Option, + parent_path: Option, + email: &str, + created_by: &str, + permissioned_as: &str, + db: &sqlx::Pool, + already_visited: Vec, +) -> error::Result<()> { + tracing::debug!( + "Triggering dependents to recompute dependencies: {}", + importers.iter().map(|dd| &dd.importer_path).join(",") + ); + for DependencyDependent { importer_path, importer_kind, importer_node_ids } in importers.iter() + { + tracing::trace!("Processing dependency: {:?}", importer_path); + if already_visited.contains(importer_path) { + tracing::trace!("Skipping already visited dependency"); + continue; + } + + let mut tx = db.clone().begin().await?; + let mut args: HashMap> = HashMap::new(); + if let Some(ref dm) = deployment_message { + args.insert("deployment_message".to_string(), to_raw_value(&dm)); + } + if let Some(ref p_path) = parent_path { + args.insert("common_dependency_path".to_string(), to_raw_value(&p_path)); + } + + args.insert( + "already_visited".to_string(), + to_raw_value(&already_visited), + ); + + args.insert( + "triggered_by_relative_import".to_string(), + to_raw_value(&true), + ); + + let mut debouncing_settings = DebouncingSettings { + debounce_key: Some(format!("{w_id}:{importer_path}:dependency")), + debounce_delay_s: Some(5), + ..Default::default() + }; + + let job_payload = match importer_kind.as_str() { + "script" => match sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(hash) => { + tracing::debug!("newest hash for {} is: {hash}", importer_path); + + let info = + windmill_common::get_script_info_for_hash(None, db, w_id, hash).await?; + + JobPayload::Dependencies { + path: importer_path.clone(), + hash: ScriptHash(hash), + language: info.language, + dedicated_worker: info.dedicated_worker, + debouncing_settings, + } + } + None => { + ScopedDependencyMap::clear_map_for_item( + importer_path, + w_id, + "script", + tx, + &None, + ) + .await + .commit() + .await?; + continue; + } + }, + + "flow" => match sqlx::query_scalar!( + "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => { + tracing::debug!("Handling flow dependency update for: {}", importer_path); + + args.insert( + "nodes_to_relock".to_string(), + to_raw_value(&importer_node_ids), + ); + + debouncing_settings.debounce_args_to_accumulate = Some(vec!["nodes_to_relock".into()]); + + JobPayload::FlowDependencies { + path: importer_path.clone(), + version, + dedicated_worker: None, + debouncing_settings, + } + } + None => { + ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "flow", tx, &None) + .await + .commit() + .await?; + continue; + } + }, + + "app" => match sqlx::query_scalar!( + "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", + importer_path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + { + Some(version) => { + tracing::debug!("Handling app dependency update for: {}", importer_path); + + args.insert( + "components_to_relock".to_string(), + to_raw_value(importer_node_ids), + ); + + debouncing_settings.debounce_args_to_accumulate = Some(vec!["components_to_relock".into()]); + + JobPayload::AppDependencies { path: importer_path.clone(), version, debouncing_settings } + } + None => { + ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "app", tx, &None) + .await + .commit() + .await?; + continue; + } + }, + + _ => { + tracing::error!( + "unexpected importer kind: {kind:?} for path {path}", + kind = importer_kind, + path = importer_path + ); + continue; + } + }; + + tracing::debug!("Pushing dependency job for: {}", importer_path); + let (job_uuid, new_tx) = windmill_queue::push( + db, + PushIsolationLevel::Transaction(tx), + &w_id, + job_payload, + windmill_queue::PushArgs { args: &args, extra: None }, + &created_by, + email, + permissioned_as.to_string(), + Some("trigger.dependents.to.recompute.dependencies"), + Some(Utc::now() + Duration::seconds(*DEPENDENCY_JOB_DEBOUNCE_DELAY as i64)), + None, + None, + None, + None, + None, + false, + false, + None, + true, + Some("dependency".into()), + None, + None, + None, + None, + false, + None, + None, + None, + ) + .await?; + + tracing::info!( + "pushed dependency job due to common python path: {job_uuid} for path {path}", + path = importer_path, + ); + new_tx.commit().await?; + } + Ok(()) +} diff --git a/backend/windmill-worker/src/workspace_dependencies.rs b/backend/windmill-dep-map/src/workspace_dependencies.rs similarity index 54% rename from backend/windmill-worker/src/workspace_dependencies.rs rename to backend/windmill-dep-map/src/workspace_dependencies.rs index 281b08a32c..6186556a25 100644 --- a/backend/windmill-worker/src/workspace_dependencies.rs +++ b/backend/windmill-dep-map/src/workspace_dependencies.rs @@ -5,7 +5,8 @@ use windmill_common::{ }; use crate::{ - scoped_dependency_map::ScopedDependencyMap, trigger_dependents_to_recompute_dependencies, + scoped_dependency_map::ScopedDependencyMap, + trigger_dependents::trigger_dependents_to_recompute_dependencies, }; #[derive(sqlx::FromRow, Clone, Serialize, Deserialize, Hash, Debug)] @@ -16,7 +17,6 @@ pub struct NewWorkspaceDependencies { /// If None, will use description of previous version /// If there is no older versions, will set to default pub description: Option, - // TODO: Make Option, or optimize it in any other way. pub content: String, } @@ -31,16 +31,12 @@ impl NewWorkspaceDependencies { metadata: (String, String, String), db: sqlx::Pool, ) -> error::Result { - // Check if all workers support workspace dependencies feature windmill_common::workspace_dependencies::min_version_supports_v0_workspace_dependencies() .await?; let path = WorkspaceDependencies::to_path(&self.name, self.language)?; - // If it is unnamed then we want to rebuild dependency map. Otherwise trigger dependents to recompute locks will not work - // NOTE: We rebuild first, even before creating new w deps. We want to make sure that if rebuild failed, then no new default workspace dependencies were created. if self.name.is_none() { - // Check if we already rebuilt the map for this workspace by checking if the setting exists let setting_name = format!("workspace_dependencies_map_rebuilt:{}", self.workspace_id); let already_rebuilt = windmill_common::global_settings::load_value_from_global_settings( @@ -57,7 +53,6 @@ impl NewWorkspaceDependencies { ); ScopedDependencyMap::rebuild_map_unchecked(&self.workspace_id, &db).await?; - // Mark as rebuilt by creating the setting windmill_common::global_settings::set_value_in_global_settings( &db, &setting_name, @@ -80,7 +75,7 @@ impl NewWorkspaceDependencies { let prev_description = sqlx::query_scalar!( " UPDATE workspace_dependencies - SET archived = true + SET archived = true WHERE archived = false AND name IS NOT DISTINCT FROM $1 AND workspace_id = $2 @@ -97,7 +92,7 @@ impl NewWorkspaceDependencies { let new_id = sqlx::query_scalar!( " INSERT INTO workspace_dependencies(name, workspace_id, content, language, description) - VALUES ($1, $2, $3, $4, $5) + VALUES ($1, $2, $3, $4, $5) RETURNING id ", self.name.clone(), @@ -141,17 +136,12 @@ pub async fn trigger_dependents_to_recompute_dependencies_in_the_background( language = ?language, "waiting for cache timeout after creating first unnamed workspace dependencies" ); - // Wait for cache timeout. - // For context, workers have cache on whether the unnamed workspace dependencies exists or not. - // when we trigger dependents to recompoute dependencies we want to make sure all workers are having cache timed out. - // otherwise it would result into bug, when workers skip fetch of workspace dependencies because they think they don't exist. tokio::time::sleep(EXISTS_CACHE_TIMEOUT).await; } - // It's ok to fail, it will return an error and user will get notified that they should redeploy workspace dependencies if let Err(e) = trigger_dependents_to_recompute_dependencies( &workspace_id, - match crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( + match ScopedDependencyMap::get_dependents( path.as_str(), &workspace_id, &db, @@ -189,114 +179,5 @@ pub async fn trigger_dependents_to_recompute_dependencies_in_the_background( }); } -// Type aliases for backward compatibility pub type RawRequirements = WorkspaceDependencies; pub type NewRawRequirements = NewWorkspaceDependencies; - -#[cfg(test)] -mod workspace_dependencies_tests { - - // // TODO: test all cases when it should reject. - // #[cfg(feature = "python")] - // mod new_workspace_dependencies { - // use windmill_common::scripts::ScriptLang; - - // use crate::workspace_dependencies::NewWorkspaceDependencies; - - // #[sqlx::test( - // fixtures("../../tests/fixtures/base.sql",), - // migrations = "../migrations" - // )] - // async fn test_create(db: sqlx::Pool) -> anyhow::Result<()> { - // assert_eq!( - // NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::Python3, - // name: None, - // description: None, - // content: "global:rev1".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .unwrap(), - // 1 - // ); - - // assert_eq!( - // NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::Python3, - // name: Some("rrs1".to_owned()), - // description: None, - // content: "rrs1:rev1".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .unwrap(), - // 2 - // ); - - // assert!(NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::DuckDb, - // description: None, - // name: None, - // content: "".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .is_err()); - - // // Will act as redeployment - // assert_eq!( - // NewWorkspaceDependencies { - // workspace_id: "test-workspace".into(), - // language: ScriptLang::Python3, - // description: None, - // name: Some("rrs1".to_owned()), - // content: "rrs1:rev2".to_owned(), - // } - // .create("", "", "", &db) - // .await - // .unwrap(), - // // It will just increment id - // 3 - // ); - // Ok(()) - // } - - // #[sqlx::test( - // fixtures("../../tests/fixtures/base.sql",), - // migrations = "../migrations" - // )] - // async fn violate_constraints(db: sqlx::Pool) -> anyhow::Result<()> { - // let db = &db; - // let create = |name| { - // sqlx::query_scalar!( - // " - // INSERT INTO workspace_dependencies(name, workspace_id, content, language) - // VALUES ($1, 'test-workspace', 'test', 'python3') - // RETURNING id - // ", - // name - // ) - // .fetch_one(db) - // }; - - // assert_eq!(create(Some("test".to_owned())).await.unwrap(), 1); - // assert_eq!(create(None).await.unwrap(), 2); - - // assert!(create(Some("test".to_owned())).await.is_err()); - // assert!(create(None).await.is_err()); - // assert_eq!( - // sqlx::query_scalar!("SELECT COUNT(*) FROM workspace_dependencies",) - // .fetch_one(db) - // .await - // .unwrap() - // .unwrap(), - // 2 - // ); - // Ok(()) - // } - // } -} diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index fa183b18f5..d3a1687b00 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -10,9 +10,9 @@ path = "src/lib.rs" [features] default = [] -parquet = ["dep:object_store"] -private = [] -enterprise = [] +parquet = ["dep:object_store", "windmill-common/parquet"] +private = ["windmill-common/private"] +enterprise = ["windmill-common/enterprise"] [dependencies] windmill-common.workspace = true diff --git a/backend/windmill-jseval/Cargo.toml b/backend/windmill-jseval/Cargo.toml new file mode 100644 index 0000000000..b2f7217404 --- /dev/null +++ b/backend/windmill-jseval/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "windmill-jseval" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_jseval" +path = "src/lib.rs" + +[features] +default = [] +quickjs = ["dep:rquickjs"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +rquickjs = { workspace = true, optional = true } +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +anyhow.workspace = true +regex.workspace = true +lazy_static.workspace = true +mappable-rc.workspace = true +futures.workspace = true +uuid.workspace = true diff --git a/backend/windmill-jseval/src/lib.rs b/backend/windmill-jseval/src/lib.rs new file mode 100644 index 0000000000..4b6bd20be1 --- /dev/null +++ b/backend/windmill-jseval/src/lib.rs @@ -0,0 +1,2649 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! QuickJS-based JavaScript expression evaluation for flow transformations. +//! +//! This crate provides fast, lightweight JS expression evaluation using QuickJS (rquickjs). +//! It is used by both windmill-worker (for flow expression eval) and windmill-api (for batch rerun). +//! +//! ## Performance Characteristics (release mode benchmarks) +//! - **Simple expressions**: ~238μs (QuickJS) vs ~3.05ms (deno_core) = **~13x faster** +//! - **Complex expressions**: ~192μs (QuickJS) vs ~3.09ms (deno_core) = **~16x faster** +//! - **Memory**: ~2.5% of V8's footprint + +use std::collections::HashMap; +use std::sync::Arc; + +use lazy_static::lazy_static; +use regex::Regex; +#[cfg(feature = "quickjs")] +use rquickjs::{ + async_with, + prelude::{Async, Func, MutFn}, + AsyncContext, AsyncRuntime, CatchResultExt, FromJs, IntoJs, Object, Value, +}; +use serde_json::value::RawValue; +use uuid::Uuid; + +use windmill_common::client::AuthedClient; +use windmill_common::flow_status::JobResult; + +// ── Public types ────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +pub struct IdContext { + pub flow_job: Uuid, + #[allow(dead_code)] + pub steps_results: HashMap, + pub previous_id: String, +} + +// ── Constants ───────────────────────────────────────────────────────── + +const FLOW_INPUT_PREFIX: &str = "flow_input"; +const ENV_KEY_PREFIX: &str = "flow_env"; +const DOT_PATTERN: &str = "."; +const START_BRACKET_PATTERN: &str = "[\""; +const END_BRACKET_PATTERN: &str = "\"]"; + +// ── Regex statics ───────────────────────────────────────────────────── + +lazy_static! { + static ref RE: Regex = Regex::new( + r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + ) + .unwrap(); + static ref RE_FULL: Regex = Regex::new( + r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" + ) + .unwrap(); +} + +// ── Shared helper functions (quickjs-only) ────────────────────────── + +#[cfg(feature = "quickjs")] +pub fn replace_with_await(expr: String, fn_name: &str) -> String { + let sep = format!("{}(", fn_name); + let mut split = expr.split(&sep); + let mut s = split.next().unwrap_or("").to_string(); + for x in split { + s.push_str(&format!("(await {}({}", fn_name, add_closing_bracket(x))) + } + s +} + +#[cfg(feature = "quickjs")] +pub fn replace_with_await_result(expr: String) -> String { + RE.replace_all(&expr, "(await $r)").to_string() +} + +#[cfg(feature = "quickjs")] +fn add_closing_bracket(s: &str) -> String { + let mut s = s.to_string(); + let mut level = 1; + let mut idx = 0; + for c in s.chars() { + match c { + '(' => level += 1, + ')' => level -= 1, + _ => (), + }; + if level == 0 { + break; + } + idx += 1; + } + s.insert_str(idx, ")"); + s +} + +pub fn try_exact_property_access( + expr: &str, + flow_input: Option<&mappable_rc::Marc>>>, + flow_env: Option<&HashMap>>, +) -> Option> { + let obj = if expr.starts_with(FLOW_INPUT_PREFIX) { + Some(( + FLOW_INPUT_PREFIX, + flow_input.as_ref().map(|obj| obj.as_ref()), + )) + } else if expr.starts_with(ENV_KEY_PREFIX) { + Some((ENV_KEY_PREFIX, flow_env)) + } else { + None + }; + + if let Some((prefix, obj)) = obj { + let access_pattern_pos = prefix.len(); + let suffix = &expr[access_pattern_pos..]; + let maybe_key_name = if suffix.starts_with(DOT_PATTERN) { + let key_name_pos = DOT_PATTERN.len(); + Some(&suffix[key_name_pos..]) + } else if suffix.starts_with(START_BRACKET_PATTERN) { + let key_name_pos = START_BRACKET_PATTERN.len(); + let suffix = &suffix[key_name_pos..]; + + let flow_arg_name = suffix + .ends_with(END_BRACKET_PATTERN) + .then(|| { + let start_key_name_pos = access_pattern_pos + key_name_pos; + let end_key_name_pos = expr.len() - END_BRACKET_PATTERN.len(); + &expr[start_key_name_pos..end_key_name_pos] + }) + .filter(|s| s.len() > 0); + flow_arg_name + } else { + None + }; + + if let Some(key_name) = maybe_key_name { + if let Some(key_value) = obj.and_then(|obj| obj.get(key_name)) { + return Some(key_value.clone()); + } + } + } + None +} + +pub async fn handle_full_regex( + expr: &str, + authed_client: &AuthedClient, + by_id: &IdContext, +) -> Option>> { + if let Some(captures) = RE_FULL.captures(&expr) { + let obj_name = captures.get(1).unwrap().as_str(); + let obj_key = captures.get(2).unwrap().as_str(); + let idx_o = captures.get(3).map(|y| y.as_str()); + let rest = captures.get(4).map(|y| y.as_str()); + let query = if let Some(idx) = idx_o { + match rest { + Some(rest) => Some(format!("{}{}", idx, rest)), + None => Some(idx.to_string()), + } + } else { + rest.map(|x| x.trim_start_matches('.').to_string()) + }; + + let result = if obj_name == "results" { + let res = authed_client + .get_result_by_id::>>( + &by_id.flow_job.to_string(), + obj_key, + query, + ) + .await + .ok() + .flatten(); + match res { + Some(v) => Ok(v), + None => serde_json::value::to_raw_value(&serde_json::Value::Null) + .map_err(|e| anyhow::anyhow!("Failed to serialize null: {}", e)), + } + } else if obj_name == "flow_env" { + authed_client + .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) + .await + } else { + unreachable!(); + }; + + return Some(result); + } + + return None; +} + +#[cfg(feature = "quickjs")] +use windmill_common::utils::unsafe_raw; + +// ── QuickJS evaluation ─────────────────────────────────────────────── + +#[cfg(feature = "quickjs")] + +/// Shared state for async operations within QuickJS +#[derive(Clone)] +struct AsyncOpState { + client: AuthedClient, +} + +#[cfg(feature = "quickjs")] +pub async fn eval_timeout_quickjs( + expr: String, + transform_context: HashMap>>, + flow_input: Option>>>, + flow_env: Option<&HashMap>>, + authed_client: Option<&AuthedClient>, + by_id: Option<&IdContext>, + ctx: Option>, +) -> anyhow::Result> { + let expr = expr.trim().to_string(); + + tracing::debug!( + "evaluating js eval (quickjs): {} with context {:?}", + expr, + transform_context + ); + + // Clone data for the blocking task + let by_id_clone = by_id.cloned(); + let flow_input_clone = flow_input.clone(); + let flow_env_clone = flow_env.cloned(); + let authed_client_clone = authed_client.cloned(); + + // Determine which context keys are actually used in the expression + let p_ids = by_id.map(|x| { + [ + format!("results.{}", x.previous_id), + format!("results?.{}", x.previous_id), + format!("results[\"{}\"]", x.previous_id), + format!("results?.[\"{}\"]", x.previous_id), + ] + }); + + let mut context_keys: Vec = transform_context + .keys() + .filter(|x| expr.contains(&x.to_string())) + .cloned() + .collect(); + + if (!context_keys.contains(&"previous_result".to_string()) + && p_ids.is_some() + && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) + || expr.contains("error") + { + context_keys.push("previous_result".to_string()); + } + + let has_flow_input = expr.contains("flow_input"); + if has_flow_input { + context_keys.push("flow_input".to_string()) + } + + // Filter transform_context to only include used keys + let filtered_context: HashMap>> = transform_context + .into_iter() + .filter(|(k, _)| context_keys.contains(k)) + .collect(); + + let expr_clone = expr.clone(); + + // Run the QuickJS evaluation with a timeout + tokio::time::timeout( + std::time::Duration::from_millis(10000), + tokio::task::spawn_blocking(move || { + // Create a new tokio runtime for async operations within the blocking context + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + rt.block_on(async move { + eval_quickjs_inner( + &expr_clone, + filtered_context, + flow_input_clone, + flow_env_clone, + authed_client_clone, + by_id_clone, + ctx, + context_keys, + ) + .await + }) + }), + ) + .await + .map_err(|_| { + anyhow::anyhow!("The expression evaluation `{expr}` took too long to execute (>10000ms)") + })?? +} + +#[cfg(feature = "quickjs")] +const QUICKJS_MEMORY_LIMIT: usize = 32 * 1024 * 1024; + +#[cfg(feature = "quickjs")] +async fn eval_quickjs_inner( + expr: &str, + transform_context: HashMap>>, + flow_input: Option>>>, + flow_env: Option>>, + authed_client: Option, + by_id: Option, + extra_ctx: Option>, + context_keys: Vec, +) -> anyhow::Result> { + let runtime = AsyncRuntime::new()?; + runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await; + let context = AsyncContext::full(&runtime).await?; + + // Create shared state for async ops if we have a client + let op_state = authed_client.map(|client| Arc::new(AsyncOpState { client })); + + let op_state_clone = op_state.clone(); + let by_id_clone = by_id.clone(); + + // Transform expression to add await for variable/resource/results access + let expr_with_funcs = ["variable", "resource"] + .into_iter() + .fold(expr.to_string(), replace_with_await); + let transformed_expr = replace_with_await_result(expr_with_funcs); + + async_with!(context => |ctx| { + let globals = ctx.globals(); + + // Set up context variables + for key in &context_keys { + if key == "flow_input" { + if let Some(ref fi) = flow_input { + let json_str = serde_json::to_string(fi.as_ref())?; + let val: serde_json::Value = serde_json::from_str(&json_str)?; + let js_val = json_to_js(&ctx, &val)?; + globals.set(key.as_str(), js_val)?; + } else { + globals.set(key.as_str(), Value::new_null(ctx.clone()))?; + } + } else if let Some(raw_val) = transform_context.get(key) { + let val: serde_json::Value = serde_json::from_str(raw_val.get())?; + let js_val = json_to_js(&ctx, &val)?; + globals.set(key.as_str(), js_val)?; + } + } + + // Set up flow_env if referenced + if expr.contains("flow_env") { + if let Some(ref fe) = flow_env { + let obj = Object::new(ctx.clone())?; + for (k, v) in fe { + let val: serde_json::Value = serde_json::from_str(v.get())?; + let js_val = json_to_js(&ctx, &val)?; + obj.set(k.as_str(), js_val)?; + } + globals.set("flow_env", obj)?; + } else { + globals.set("flow_env", Object::new(ctx.clone())?)?; + } + } + + // Set up additional context variables + if let Some(ctx_vars) = extra_ctx { + for (k, v) in ctx_vars { + globals.set(k.as_str(), v.as_str())?; + } + } + + // Set up error extraction if needed + if expr.contains("error") && context_keys.contains(&"previous_result".to_string()) { + let error_setup = r#" + let error = previous_result?.error; + if (!error) { + if (Array.isArray(previous_result)) { + const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item); + if (errors.length === 1) { + error = errors[0].error; + } else if (errors.length > 1) { + error = { + name: 'MultipleErrors', + message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '), + errors: previous_result + }; + } else { + error = { + name: 'MultipleErrors', + message: "Could not parse errors", + errors: previous_result + }; + } + } else { + if (previous_result) { + error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result }; + } else { + error = { name: 'UnknownError', message: 'No error found' }; + } + } + } + "#; + ctx.eval::<(), _>(error_setup).catch(&ctx).map_err(quickjs_error_to_anyhow)?; + } + + // Set up async functions if we have a client + if let Some(ref state) = op_state_clone { + setup_async_ops(&ctx, &globals, state.clone())?; + } else { + // Set up stub functions that throw errors + setup_stub_functions(&ctx, &globals)?; + } + + // Set up results proxy if we have by_id context + if let Some(ref by_id) = by_id_clone { + setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?; + } + + // Determine if we need to add return statement. + let code = if should_add_return_quickjs(&transformed_expr) { + format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) + } else { + format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) + }; + + // Evaluate the expression (returns a Promise that resolves to a JSON string) + let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?; + + // Await the promise + let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?; + + let json_str = String::from_js(&ctx, result) + .unwrap_or_else(|_| "null".to_string()); + + Ok(unsafe_raw(json_str)) + }) + .await +} + +#[cfg(feature = "quickjs")] +fn setup_async_ops<'js>( + ctx: &rquickjs::Ctx<'js>, + globals: &Object<'js>, + state: Arc, +) -> anyhow::Result<()> { + const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; + + let state_for_var = state.clone(); + globals.set( + "__fetchVariable", + Func::from(Async(MutFn::new(move |path: String| { + let client = state_for_var.client.clone(); + async move { + match client.get_variable_value(&path).await { + Ok(value) => value, + Err(e) => format!("{}{}", ERR_PREFIX, e), + } + } + }))), + )?; + + let state_for_res = state.clone(); + globals.set( + "__fetchResource", + Func::from(Async(MutFn::new(move |path: String| { + let client = state_for_res.client.clone(); + async move { + match client + .get_resource_value_interpolated::(&path, None) + .await + { + Ok(value) => { + serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) + } + Err(e) => format!("{}{}", ERR_PREFIX, e), + } + } + }))), + )?; + + let wrapper_code = r#" + const __ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; + + async function variable(path) { + const result = await __fetchVariable(path); + if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { + throw new Error(result.substring(__ERR_PREFIX.length)); + } + return result; + } + + async function resource(path) { + const result = await __fetchResource(path); + if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { + throw new Error(result.substring(__ERR_PREFIX.length)); + } + return JSON.parse(result); + } + "#; + + ctx.eval::<(), _>(wrapper_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(()) +} + +#[cfg(feature = "quickjs")] +fn setup_stub_functions<'js>( + ctx: &rquickjs::Ctx<'js>, + _globals: &Object<'js>, +) -> anyhow::Result<()> { + let setup_code = r#" + function variable(path) { + return Promise.reject(new Error(`variable() is not available without an authenticated client`)); + } + + function resource(path) { + return Promise.reject(new Error(`resource() is not available without an authenticated client`)); + } + "#; + + ctx.eval::<(), _>(setup_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(()) +} + +#[cfg(feature = "quickjs")] +fn setup_results_proxy<'js>( + ctx: &rquickjs::Ctx<'js>, + globals: &Object<'js>, + by_id: &IdContext, + op_state: Option>, +) -> anyhow::Result<()> { + globals.set("__previous_id", by_id.previous_id.clone())?; + + if let Some(state) = op_state { + let by_id_for_result = by_id.clone(); + globals.set( + "__fetchResult", + Func::from(Async(MutFn::new(move |step_id: String| { + let client = state.client.clone(); + let by_id = by_id_for_result.clone(); + let step_id_clone = step_id.clone(); + + let job_result = by_id.steps_results.get(&step_id).cloned(); + let flow_job_id = by_id.flow_job.to_string(); + + async move { + const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; + + let result: Result = match job_result { + Some(jr) => match jr { + JobResult::SingleJob(job_id) => client + .get_completed_job_result::( + &job_id.to_string(), + None, + ) + .await + .map_err(|e| { + format!( + "Failed to fetch result for step '{}': {}", + step_id_clone, e + ) + }), + JobResult::ListJob(job_ids) => { + let futs = job_ids.iter().map(|job_id| { + let client = client.clone(); + let job_id_str = job_id.to_string(); + async move { + client + .get_completed_job_result::( + &job_id_str, + None, + ) + .await + } + }); + let results: Vec<_> = futures::future::join_all(futs).await; + let collected: Result, _> = results.into_iter().collect(); + collected.map(serde_json::Value::Array).map_err(|e| { + format!( + "Failed to fetch results for step '{}': {}", + step_id_clone, e + ) + }) + } + }, + None => Ok(client + .get_result_by_id::( + &flow_job_id, + &step_id_clone, + None, + ) + .await + .ok() + .unwrap_or(serde_json::Value::Null)), + }; + + match result { + Ok(value) => { + serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()) + } + Err(e) => format!("{}{}", ERR_PREFIX, e), + } + } + }))), + )?; + + let wrapper_code = r#" + const __RESULT_ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; + async function __getResult(stepId) { + const result = await __fetchResult(stepId); + if (typeof result === 'string' && result.startsWith(__RESULT_ERR_PREFIX)) { + throw new Error(result.substring(__RESULT_ERR_PREFIX.length)); + } + return JSON.parse(result); + } + "#; + ctx.eval::<(), _>(wrapper_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + } else { + let stub_code = r#" + function __getResult(stepId) { + return Promise.reject(new Error('Result fetching not available without authenticated client')); + } + "#; + ctx.eval::<(), _>(stub_code) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + } + + let proxy_setup = r#" + const results = new Proxy({}, { + get: function(target, name, receiver) { + if (typeof name === 'symbol') { + return undefined; + } + if (name === __previous_id && typeof previous_result !== 'undefined') { + return Promise.resolve(previous_result); + } + return __getResult(name); + } + }); + "#; + ctx.eval::<(), _>(proxy_setup) + .catch(ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(()) +} + +#[cfg(feature = "quickjs")] +fn json_to_js<'js>( + ctx: &rquickjs::Ctx<'js>, + val: &serde_json::Value, +) -> rquickjs::Result> { + match val { + serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), + serde_json::Value::Bool(b) => Ok(Value::new_bool(ctx.clone(), *b)), + serde_json::Value::Number(n) => { + if let Some(i) = n.as_i64() { + if i >= i32::MIN as i64 && i <= i32::MAX as i64 { + Ok(Value::new_int(ctx.clone(), i as i32)) + } else { + Ok(Value::new_float(ctx.clone(), i as f64)) + } + } else if let Some(f) = n.as_f64() { + Ok(Value::new_float(ctx.clone(), f)) + } else { + Ok(Value::new_float(ctx.clone(), 0.0)) + } + } + serde_json::Value::String(s) => s.clone().into_js(ctx), + serde_json::Value::Array(arr) => { + let js_arr = rquickjs::Array::new(ctx.clone())?; + for (i, item) in arr.iter().enumerate() { + js_arr.set(i, json_to_js(ctx, item)?)?; + } + Ok(js_arr.into_value()) + } + serde_json::Value::Object(obj) => { + let js_obj = Object::new(ctx.clone())?; + for (k, v) in obj { + js_obj.set(k.as_str(), json_to_js(ctx, v)?)?; + } + Ok(js_obj.into_value()) + } + } +} + +#[cfg(feature = "quickjs")] +fn should_add_return_quickjs(expr: &str) -> bool { + let trimmed = expr.trim(); + + if trimmed.is_empty() { + return true; + } + + if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" { + return false; + } + + let statement_prefixes = [ + "const ", + "let ", + "var ", + "if ", + "if(", + "for ", + "for(", + "while ", + "while(", + "switch ", + "switch(", + "try ", + "try{", + "throw ", + "function ", + "class ", + "async ", + "await ", + ]; + + for prefix in &statement_prefixes { + if trimmed.starts_with(prefix) { + return false; + } + } + + if contains_semicolon_outside_strings(trimmed) { + return false; + } + + true +} + +#[cfg(feature = "quickjs")] +fn contains_semicolon_outside_strings(expr: &str) -> bool { + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut in_template = false; + let mut prev_char = '\0'; + + for ch in expr.chars() { + match ch { + '\'' if prev_char != '\\' && !in_double_quote && !in_template => { + in_single_quote = !in_single_quote; + } + '"' if prev_char != '\\' && !in_single_quote && !in_template => { + in_double_quote = !in_double_quote; + } + '`' if prev_char != '\\' && !in_single_quote && !in_double_quote => { + in_template = !in_template; + } + ';' if !in_single_quote && !in_double_quote && !in_template => { + return true; + } + _ => {} + } + prev_char = ch; + } + + false +} + +#[cfg(feature = "quickjs")] +fn quickjs_error_to_anyhow(err: rquickjs::CaughtError<'_>) -> anyhow::Error { + anyhow::anyhow!("QuickJS evaluation error: {}", err) +} + +// ── eval_simple_js for windmill-api batch rerun ────────────────────── + +#[cfg(feature = "quickjs")] +pub async fn eval_simple_js( + expr: String, + globals: HashMap, +) -> anyhow::Result> { + tokio::time::timeout( + std::time::Duration::from_millis(10000), + tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + rt.block_on(async move { + let runtime = AsyncRuntime::new()?; + runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await; + let context = AsyncContext::full(&runtime).await?; + + async_with!(context => |ctx| { + let js_globals = ctx.globals(); + + // Set up each named global + for (name, value) in &globals { + let js_val = json_to_js(&ctx, value)?; + js_globals.set(name.as_str(), js_val)?; + } + + // Wrap expression to return JSON string + let code = format!("JSON.stringify(({}) ?? null)", expr); + let result: String = ctx.eval(code) + .catch(&ctx) + .map_err(quickjs_error_to_anyhow)?; + + Ok(unsafe_raw(result)) + }) + .await + }) + }), + ) + .await + .map_err(|_| { + anyhow::anyhow!("The expression evaluation took too long to execute (>10000ms)") + })?? +} + +// ── Fallback stubs when quickjs is disabled ────────────────────────── + +#[cfg(not(feature = "quickjs"))] +pub async fn eval_timeout_quickjs( + _expr: String, + _transform_context: HashMap>>, + _flow_input: Option>>>, + _flow_env: Option<&HashMap>>, + _authed_client: Option<&windmill_common::client::AuthedClient>, + _by_id: Option<&IdContext>, + _ctx: Option>, +) -> anyhow::Result> { + anyhow::bail!("JavaScript expression evaluation requires the `quickjs` feature. Enable it with: cargo build --features quickjs") +} + +#[cfg(not(feature = "quickjs"))] +pub async fn eval_simple_js( + _expr: String, + _globals: HashMap, +) -> anyhow::Result> { + anyhow::bail!("JavaScript expression evaluation requires the `quickjs` feature. Enable it with: cargo build --features quickjs") +} + +// ── Tests ──────────────────────────────────────────────────────────── + +#[cfg(all(test, feature = "quickjs"))] +mod tests { + use super::*; + use serde_json::json; + use windmill_common::worker::to_raw_value; + + /// Helper: evaluate an expression with a transform context and assert against expected JSON. + async fn assert_eval( + expr: &str, + ctx: HashMap>>, + expected: serde_json::Value, + ) { + assert_eval_full(expr, ctx, None, None, expected).await; + } + + /// Helper: evaluate with full context (transform_context, flow_input, flow_env). + async fn assert_eval_full( + expr: &str, + ctx: HashMap>>, + flow_input: Option>>>, + flow_env: Option<&HashMap>>, + expected: serde_json::Value, + ) { + let result = eval_timeout_quickjs( + expr.to_string(), + ctx, + flow_input, + flow_env, + None, + None, + None, + ) + .await + .unwrap_or_else(|e| panic!("eval_timeout_quickjs failed for '{}': {}", expr, e)); + + let actual: serde_json::Value = serde_json::from_str(result.get()).unwrap_or_else(|e| { + panic!( + "Failed to parse result '{}' for '{}': {}", + result.get(), + expr, + e + ) + }); + + assert_eq!(actual, expected, "Mismatch for expression '{}'", expr); + } + + // ===================================================================== + // HELPER FUNCTION TESTS + // ===================================================================== + + #[test] + fn test_should_add_return_quickjs() { + assert!(should_add_return_quickjs("5")); + assert!(should_add_return_quickjs("x + y")); + assert!(should_add_return_quickjs("foo()")); + assert!(should_add_return_quickjs("obj.method()")); + assert!(should_add_return_quickjs("a > b ? 'yes' : 'no'")); + assert!(should_add_return_quickjs("({ key: 'value' })")); + + assert!(!should_add_return_quickjs("return 5")); + assert!(!should_add_return_quickjs("return x + y")); + assert!(!should_add_return_quickjs("const x = 5")); + assert!(!should_add_return_quickjs("let y = 10")); + assert!(!should_add_return_quickjs("if (x > 5) { return x; }")); + assert!(!should_add_return_quickjs("let x = 5; x + 1")); + assert!(!should_add_return_quickjs("try { return 1; } catch(e) {}")); + } + + #[test] + fn test_contains_semicolon_outside_strings() { + assert!(contains_semicolon_outside_strings("a; b")); + assert!(contains_semicolon_outside_strings("let x = 5; x + 1")); + + assert!(!contains_semicolon_outside_strings("'a;b'")); + assert!(!contains_semicolon_outside_strings("\"a;b\"")); + assert!(!contains_semicolon_outside_strings("`a;b`")); + assert!(!contains_semicolon_outside_strings("x + y")); + } + + #[test] + fn test_replace_with_await() { + assert_eq!( + replace_with_await("variable('test')".to_string(), "variable"), + "(await variable('test'))" + ); + assert_eq!( + replace_with_await("x + variable('a') + variable('b')".to_string(), "variable"), + "x + (await variable('a')) + (await variable('b'))" + ); + assert_eq!( + replace_with_await("no_match".to_string(), "variable"), + "no_match" + ); + } + + #[test] + fn test_replace_with_await_result() { + assert_eq!( + replace_with_await_result("results.step_a".to_string()), + "(await results.step_a)" + ); + assert_eq!( + replace_with_await_result("results.a + results.b".to_string()), + "(await results.a) + (await results.b)" + ); + assert_eq!( + replace_with_await_result("no_results_here".to_string()), + "no_results_here" + ); + } + + #[test] + fn test_try_exact_property_access_flow_input_dot() { + let mut fi = HashMap::new(); + fi.insert("name".to_string(), to_raw_value(&json!("hello"))); + let fi = mappable_rc::Marc::new(fi); + + let result = try_exact_property_access("flow_input.name", Some(&fi), None); + assert!(result.is_some()); + assert_eq!(result.unwrap().get(), "\"hello\""); + } + + #[test] + fn test_try_exact_property_access_flow_input_bracket() { + let mut fi = HashMap::new(); + fi.insert("my_key".to_string(), to_raw_value(&json!(42))); + let fi = mappable_rc::Marc::new(fi); + + let result = try_exact_property_access("flow_input[\"my_key\"]", Some(&fi), None); + assert!(result.is_some()); + assert_eq!(result.unwrap().get(), "42"); + } + + #[test] + fn test_try_exact_property_access_flow_env() { + let mut fe = HashMap::new(); + fe.insert("ENV".to_string(), to_raw_value(&json!("production"))); + + let result = try_exact_property_access("flow_env.ENV", None, Some(&fe)); + assert!(result.is_some()); + assert_eq!(result.unwrap().get(), "\"production\""); + } + + #[test] + fn test_try_exact_property_access_missing_key() { + let fi = mappable_rc::Marc::new(HashMap::new()); + let result = try_exact_property_access("flow_input.missing", Some(&fi), None); + assert!(result.is_none()); + } + + #[test] + fn test_try_exact_property_access_no_prefix() { + let result = try_exact_property_access("some_var.key", None, None); + assert!(result.is_none()); + } + + // ===================================================================== + // SIMPLE ARITHMETIC + // ===================================================================== + + #[tokio::test] + async fn test_simple_arithmetic() { + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3)))); + + assert_eval("x + y", env.clone(), json!(8)).await; + assert_eval("x - y", env.clone(), json!(2)).await; + assert_eval("x * y", env.clone(), json!(15)).await; + assert_eval("x % y", env.clone(), json!(2)).await; + assert_eval("x ** 2", env.clone(), json!(25)).await; + } + + // ===================================================================== + // OBJECT PROPERTY ACCESS + // ===================================================================== + + #[tokio::test] + async fn test_object_property_access() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({ + "name": "test", + "value": 42, + "nested": {"deep": {"property": "found"}} + }))), + ); + + assert_eval("obj.name", env.clone(), json!("test")).await; + assert_eval("obj.value", env.clone(), json!(42)).await; + assert_eval("obj.nested.deep.property", env.clone(), json!("found")).await; + assert_eval("obj['name']", env.clone(), json!("test")).await; + } + + // ===================================================================== + // ARRAY OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_array_operations() { + let mut env = HashMap::new(); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), + ); + + assert_eval("arr.length", env.clone(), json!(5)).await; + assert_eval("arr[0]", env.clone(), json!(1)).await; + assert_eval("arr.map(x => x * 2)", env.clone(), json!([2, 4, 6, 8, 10])).await; + assert_eval("arr.filter(x => x > 2)", env.clone(), json!([3, 4, 5])).await; + assert_eval("arr.reduce((a, b) => a + b, 0)", env.clone(), json!(15)).await; + assert_eval("arr.find(x => x > 3)", env.clone(), json!(4)).await; + assert_eval("arr.some(x => x > 4)", env.clone(), json!(true)).await; + assert_eval("arr.every(x => x > 0)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // STRING OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_string_operations() { + let mut env = HashMap::new(); + env.insert( + "s".to_string(), + Arc::new(to_raw_value(&json!("Hello World"))), + ); + + assert_eval("s.toLowerCase()", env.clone(), json!("hello world")).await; + assert_eval("s.toUpperCase()", env.clone(), json!("HELLO WORLD")).await; + assert_eval("s.length", env.clone(), json!(11)).await; + assert_eval("s.split(' ')", env.clone(), json!(["Hello", "World"])).await; + assert_eval( + "s.replace('World', 'QuickJS')", + env.clone(), + json!("Hello QuickJS"), + ) + .await; + assert_eval("s.includes('World')", env.clone(), json!(true)).await; + assert_eval("s.startsWith('Hello')", env.clone(), json!(true)).await; + assert_eval("s.trim()", env.clone(), json!("Hello World")).await; + } + + // ===================================================================== + // TERNARY AND CONDITIONALS + // ===================================================================== + + #[tokio::test] + async fn test_ternary_and_conditionals() { + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(10)))); + env.insert("y".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval("x > y ? 'bigger' : 'smaller'", env.clone(), json!("bigger")).await; + assert_eval("x === 10 ? true : false", env.clone(), json!(true)).await; + assert_eval("x > 5 && y < 10", env.clone(), json!(true)).await; + assert_eval("x > 20 || y < 10", env.clone(), json!(true)).await; + assert_eval("!false", env.clone(), json!(true)).await; + } + + // ===================================================================== + // OBJECT CREATION + // ===================================================================== + + #[tokio::test] + async fn test_object_creation() { + let mut env = HashMap::new(); + env.insert("name".to_string(), Arc::new(to_raw_value(&json!("test")))); + env.insert("value".to_string(), Arc::new(to_raw_value(&json!(42)))); + + assert_eval("({ foo: 'bar' })", env.clone(), json!({"foo": "bar"})).await; + assert_eval( + "({ name, value })", + env.clone(), + json!({"name": "test", "value": 42}), + ) + .await; + assert_eval( + "({ ...{ a: 1 }, b: 2 })", + env.clone(), + json!({"a": 1, "b": 2}), + ) + .await; + } + + // ===================================================================== + // NULL / UNDEFINED + // ===================================================================== + + #[tokio::test] + async fn test_null_undefined() { + assert_eval("null", HashMap::new(), json!(null)).await; + assert_eval("undefined", HashMap::new(), json!(null)).await; + + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(null)))); + assert_eval("x", env.clone(), json!(null)).await; + assert_eval("x ?? 'default'", env.clone(), json!("default")).await; + } + + // ===================================================================== + // FLOW INPUT + // ===================================================================== + + #[tokio::test] + async fn test_flow_input() { + let mut fi = HashMap::new(); + fi.insert("name".to_string(), to_raw_value(&json!("test_flow"))); + fi.insert("count".to_string(), to_raw_value(&json!(100))); + fi.insert( + "config".to_string(), + to_raw_value(&json!({"enabled": true})), + ); + + let fi = Some(mappable_rc::Marc::new(fi)); + + assert_eval_full( + "flow_input.name", + HashMap::new(), + fi.clone(), + None, + json!("test_flow"), + ) + .await; + assert_eval_full( + "flow_input.count", + HashMap::new(), + fi.clone(), + None, + json!(100), + ) + .await; + assert_eval_full( + "flow_input.config.enabled", + HashMap::new(), + fi.clone(), + None, + json!(true), + ) + .await; + } + + // ===================================================================== + // TEMPLATE LITERALS + // ===================================================================== + + #[tokio::test] + async fn test_template_literals() { + let mut env = HashMap::new(); + env.insert("name".to_string(), Arc::new(to_raw_value(&json!("World")))); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval("`Hello ${name}!`", env.clone(), json!("Hello World!")).await; + assert_eval( + "`The answer is ${x * 2}`", + env.clone(), + json!("The answer is 10"), + ) + .await; + } + + // ===================================================================== + // JSON OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_json_operations() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), + ); + + // JSON.stringify produces a string result + assert_eval( + "JSON.stringify(obj)", + env.clone(), + json!("{\"a\":1,\"b\":2}"), + ) + .await; + assert_eval("Object.keys(obj)", env.clone(), json!(["a", "b"])).await; + assert_eval("Object.values(obj)", env.clone(), json!([1, 2])).await; + } + + // ===================================================================== + // MATH OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_math_operations() { + let env = HashMap::new(); + + assert_eval("Math.max(1, 5, 3)", env.clone(), json!(5)).await; + assert_eval("Math.min(1, 5, 3)", env.clone(), json!(1)).await; + assert_eval("Math.abs(-5)", env.clone(), json!(5)).await; + assert_eval("Math.floor(3.7)", env.clone(), json!(3)).await; + assert_eval("Math.ceil(3.2)", env.clone(), json!(4)).await; + assert_eval("Math.round(3.5)", env.clone(), json!(4)).await; + } + + // ===================================================================== + // TYPE COERCION + // ===================================================================== + + #[tokio::test] + async fn test_type_coercion() { + let mut env = HashMap::new(); + env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); + env.insert("str".to_string(), Arc::new(to_raw_value(&json!("123")))); + + assert_eval("String(num)", env.clone(), json!("42")).await; + assert_eval("Number(str)", env.clone(), json!(123)).await; + assert_eval("Boolean(num)", env.clone(), json!(true)).await; + assert_eval("parseInt('42px')", env.clone(), json!(42)).await; + assert_eval("parseFloat('3.14')", env.clone(), json!(3.14)).await; + } + + // ===================================================================== + // ARRAY SPREAD + // ===================================================================== + + #[tokio::test] + async fn test_array_spread() { + let mut env = HashMap::new(); + env.insert( + "arr1".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 3]))), + ); + env.insert( + "arr2".to_string(), + Arc::new(to_raw_value(&json!([4, 5, 6]))), + ); + + assert_eval("[...arr1, ...arr2]", env.clone(), json!([1, 2, 3, 4, 5, 6])).await; + assert_eval("[0, ...arr1, 99]", env.clone(), json!([0, 1, 2, 3, 99])).await; + } + + // ===================================================================== + // MULTILINE STATEMENTS + // ===================================================================== + + #[tokio::test] + async fn test_multiline_statements() { + let mut env = HashMap::new(); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval( + r#"let y = x * 2; + return y + 1"#, + env.clone(), + json!(11), + ) + .await; + + assert_eval( + r#"const result = x > 3 ? 'big' : 'small'; + return result"#, + env.clone(), + json!("big"), + ) + .await; + } + + // ===================================================================== + // OPTIONAL CHAINING & NULLISH COALESCING + // ===================================================================== + + #[tokio::test] + async fn test_optional_chaining_nullish() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({"a": {"b": 1}}))), + ); + env.insert("empty".to_string(), Arc::new(to_raw_value(&json!(null)))); + + assert_eval("obj?.a?.b", env.clone(), json!(1)).await; + assert_eval("obj?.a?.c", env.clone(), json!(null)).await; + assert_eval("obj?.x?.y", env.clone(), json!(null)).await; + assert_eval("empty?.foo", env.clone(), json!(null)).await; + + assert_eval("null ?? 'default'", env.clone(), json!("default")).await; + assert_eval("undefined ?? 'default'", env.clone(), json!("default")).await; + assert_eval("0 ?? 'default'", env.clone(), json!(0)).await; + assert_eval("'' ?? 'default'", env.clone(), json!("")).await; + assert_eval("false ?? 'default'", env.clone(), json!(false)).await; + } + + // ===================================================================== + // DESTRUCTURING + // ===================================================================== + + #[tokio::test] + async fn test_destructuring() { + let mut env = HashMap::new(); + env.insert( + "obj".to_string(), + Arc::new(to_raw_value(&json!({"name": "test", "value": 42}))), + ); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), + ); + + assert_eval( + "const { name, value } = obj; return { name, value }", + env.clone(), + json!({"name": "test", "value": 42}), + ) + .await; + + assert_eval( + "const [first, second, ...rest] = arr; return { first, second, rest }", + env.clone(), + json!({"first": 1, "second": 2, "rest": [3, 4, 5]}), + ) + .await; + + assert_eval( + "const { missing = 'default' } = obj; return missing", + env.clone(), + json!("default"), + ) + .await; + } + + // ===================================================================== + // NUMBER EDGE CASES + // ===================================================================== + + #[tokio::test] + async fn test_number_edge_cases() { + let env = HashMap::new(); + + assert_eval( + "Number.MAX_SAFE_INTEGER", + env.clone(), + json!(9007199254740991_i64), + ) + .await; + assert_eval( + "Number.MIN_SAFE_INTEGER", + env.clone(), + json!(-9007199254740991_i64), + ) + .await; + assert_eval("Number.isInteger(5)", env.clone(), json!(true)).await; + assert_eval("Number.isInteger(5.5)", env.clone(), json!(false)).await; + assert_eval("Number.isFinite(Infinity)", env.clone(), json!(false)).await; + assert_eval("Number.isNaN(NaN)", env.clone(), json!(true)).await; + assert_eval("isNaN(NaN)", env.clone(), json!(true)).await; + assert_eval("isFinite(100)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // REGEX BASIC + // ===================================================================== + + #[tokio::test] + async fn test_regex_basic() { + let mut env = HashMap::new(); + env.insert( + "str".to_string(), + Arc::new(to_raw_value(&json!("hello world 123"))), + ); + + assert_eval("/hello/.test(str)", env.clone(), json!(true)).await; + assert_eval("str.match(/\\d+/)?.[0]", env.clone(), json!("123")).await; + assert_eval( + "str.replace(/world/, 'universe')", + env.clone(), + json!("hello universe 123"), + ) + .await; + assert_eval( + "str.split(/\\s+/)", + env.clone(), + json!(["hello", "world", "123"]), + ) + .await; + assert_eval("'aaa'.replace(/a/g, 'b')", env.clone(), json!("bbb")).await; + assert_eval("/HELLO/i.test(str)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // DATE OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_date_basic() { + let env = HashMap::new(); + + assert_eval( + "Date.parse('2024-01-15T00:00:00.000Z')", + env.clone(), + json!(1705276800000_i64), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').getUTCFullYear()", + env.clone(), + json!(2024), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').getUTCMonth()", + env.clone(), + json!(0), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').getUTCDate()", + env.clone(), + json!(15), + ) + .await; + assert_eval( + "new Date('2024-01-15T00:00:00.000Z').toISOString()", + env.clone(), + json!("2024-01-15T00:00:00.000Z"), + ) + .await; + } + + #[tokio::test] + async fn test_date_serialization() { + let env = HashMap::new(); + + // Direct Date → ISO string via toJSON + assert_eval( + "new Date('2024-01-15T12:30:00.000Z')", + env.clone(), + json!("2024-01-15T12:30:00.000Z"), + ) + .await; + + // Date within an object + assert_eval( + "({ date: new Date('2024-01-15T00:00:00.000Z'), name: 'test' })", + env.clone(), + json!({"date": "2024-01-15T00:00:00.000Z", "name": "test"}), + ) + .await; + + // Deeply nested Date + assert_eval( + "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })", + env.clone(), + json!({"level1": {"level2": {"date": "2024-01-15T00:00:00.000Z"}}}), + ) + .await; + } + + // ===================================================================== + // SPECIAL OBJECT SERIALIZATION + // ===================================================================== + + #[tokio::test] + async fn test_special_object_serialization() { + let env = HashMap::new(); + + // RegExp, Map, Set all serialize to {} + assert_eval("/test/gi", env.clone(), json!({})).await; + assert_eval("new Map([['key', 'value']])", env.clone(), json!({})).await; + assert_eval("new Set([1, 2, 3])", env.clone(), json!({})).await; + } + + // ===================================================================== + // ARRAY ADVANCED + // ===================================================================== + + #[tokio::test] + async fn test_array_advanced() { + let mut env = HashMap::new(); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9, 2, 6]))), + ); + env.insert( + "nested".to_string(), + Arc::new(to_raw_value(&json!([[1, 2], [3, 4], [5, 6]]))), + ); + + assert_eval( + "[...arr].sort((a, b) => a - b)", + env.clone(), + json!([1, 1, 2, 3, 4, 5, 6, 9]), + ) + .await; + assert_eval( + "[...arr].sort((a, b) => b - a)", + env.clone(), + json!([9, 6, 5, 4, 3, 2, 1, 1]), + ) + .await; + assert_eval("nested.flat()", env.clone(), json!([1, 2, 3, 4, 5, 6])).await; + assert_eval( + "nested.flatMap(x => x)", + env.clone(), + json!([1, 2, 3, 4, 5, 6]), + ) + .await; + assert_eval("arr.indexOf(5)", env.clone(), json!(4)).await; + assert_eval("arr.indexOf(99)", env.clone(), json!(-1)).await; + assert_eval("arr.includes(9)", env.clone(), json!(true)).await; + assert_eval("arr.slice(2, 5)", env.clone(), json!([4, 1, 5])).await; + assert_eval("arr.slice(-3)", env.clone(), json!([9, 2, 6])).await; + } + + // ===================================================================== + // LOGICAL OPERATORS + // ===================================================================== + + #[tokio::test] + async fn test_logical_operators() { + let mut env = HashMap::new(); + env.insert("a".to_string(), Arc::new(to_raw_value(&json!(true)))); + env.insert("b".to_string(), Arc::new(to_raw_value(&json!(false)))); + env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); + + assert_eval("a && 'yes'", env.clone(), json!("yes")).await; + assert_eval("b && 'yes'", env.clone(), json!(false)).await; + assert_eval("b || 'no'", env.clone(), json!("no")).await; + assert_eval("a || 'no'", env.clone(), json!(true)).await; + assert_eval("let y = null; y ??= 10; return y", env.clone(), json!(10)).await; + assert_eval("let y = 5; y ??= 10; return y", env.clone(), json!(5)).await; + assert_eval("(a && x > 3) || (b && x < 3)", env.clone(), json!(true)).await; + } + + // ===================================================================== + // TYPEOF + // ===================================================================== + + #[tokio::test] + async fn test_typeof() { + let mut env = HashMap::new(); + env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); + env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); + env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([1, 2, 3])))); + env.insert("obj".to_string(), Arc::new(to_raw_value(&json!({"a": 1})))); + env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); + + assert_eval("typeof str", env.clone(), json!("string")).await; + assert_eval("typeof num", env.clone(), json!("number")).await; + assert_eval("typeof arr", env.clone(), json!("object")).await; + assert_eval("typeof obj", env.clone(), json!("object")).await; + assert_eval("typeof n", env.clone(), json!("object")).await; + assert_eval("typeof undefined", env.clone(), json!("undefined")).await; + assert_eval("Array.isArray(arr)", env.clone(), json!(true)).await; + assert_eval("Array.isArray(obj)", env.clone(), json!(false)).await; + } + + // ===================================================================== + // COMPLEX MULTILINE EXPRESSIONS + // ===================================================================== + + #[tokio::test] + async fn test_multiline_complex_logic() { + let mut env = HashMap::new(); + env.insert( + "users".to_string(), + Arc::new(to_raw_value(&json!([ + {"name": "Alice", "age": 30, "role": "admin"}, + {"name": "Bob", "age": 25, "role": "user"}, + {"name": "Charlie", "age": 35, "role": "admin"}, + {"name": "Diana", "age": 28, "role": "user"} + ]))), + ); + + assert_eval( + r#" + const admins = users.filter(u => u.role === 'admin'); + const names = admins.map(u => u.name); + return names.join(', ') + "#, + env.clone(), + json!("Alice, Charlie"), + ) + .await; + + assert_eval( + r#" + const totalAge = users.reduce((sum, u) => sum + u.age, 0); + const avgAge = totalAge / users.length; + return Math.round(avgAge) + "#, + env.clone(), + json!(30), + ) + .await; + + assert_eval( + r#" + const grouped = users.reduce((acc, u) => { + if (!acc[u.role]) acc[u.role] = []; + acc[u.role].push(u.name); + return acc; + }, {}); + return grouped + "#, + env.clone(), + json!({"admin": ["Alice", "Charlie"], "user": ["Bob", "Diana"]}), + ) + .await; + } + + // ===================================================================== + // DATA TRANSFORMATION + // ===================================================================== + + #[tokio::test] + async fn test_multiline_data_transformation() { + let mut env = HashMap::new(); + env.insert( + "data".to_string(), + Arc::new(to_raw_value(&json!({ + "items": [ + {"id": 1, "price": 100, "quantity": 2}, + {"id": 2, "price": 50, "quantity": 5}, + {"id": 3, "price": 75, "quantity": 3} + ], + "discount": 0.1 + }))), + ); + + assert_eval( + r#" + const subtotals = data.items.map(item => item.price * item.quantity); + const total = subtotals.reduce((a, b) => a + b, 0); + const discounted = total * (1 - data.discount); + return { subtotals, total, discounted } + "#, + env.clone(), + json!({"subtotals": [200, 250, 225], "total": 675, "discounted": 607.5}), + ) + .await; + } + + // ===================================================================== + // TRY-CATCH + // ===================================================================== + + #[tokio::test] + async fn test_try_catch() { + let env = HashMap::new(); + + assert_eval( + r#" + try { + return JSON.parse('{"valid": true}'); + } catch (e) { + return { problem: e.message }; + } + "#, + env.clone(), + json!({"valid": true}), + ) + .await; + + assert_eval( + r#" + try { + return JSON.parse('invalid json'); + } catch (e) { + return { problem: 'parse_failed' }; + } + "#, + env.clone(), + json!({"problem": "parse_failed"}), + ) + .await; + + assert_eval( + r#" + let result = 'initial'; + try { + result = 'try'; + } catch (e) { + result = 'catch'; + } finally { + result = result + '_finally'; + } + return result + "#, + env.clone(), + json!("try_finally"), + ) + .await; + } + + // ===================================================================== + // OBJECT ADVANCED + // ===================================================================== + + #[tokio::test] + async fn test_object_advanced() { + let mut env = HashMap::new(); + env.insert( + "config".to_string(), + Arc::new(to_raw_value(&json!({ + "server": {"host": "localhost", "port": 8080}, + "database": {"host": "db.local", "port": 5432}, + "features": ["auth", "logging", "cache"] + }))), + ); + + assert_eval( + "Object.assign({}, config.server, { secure: true })", + env.clone(), + json!({"host": "localhost", "port": 8080, "secure": true}), + ) + .await; + + assert_eval( + "({ ...config.server, port: 443, secure: true })", + env.clone(), + json!({"host": "localhost", "port": 443, "secure": true}), + ) + .await; + + assert_eval( + "JSON.parse(JSON.stringify(config))", + env.clone(), + json!({ + "server": {"host": "localhost", "port": 8080}, + "database": {"host": "db.local", "port": 5432}, + "features": ["auth", "logging", "cache"] + }), + ) + .await; + + assert_eval( + r#" + const key = 'dynamic'; + return { [key]: 'value', [`${key}_2`]: 'value2' } + "#, + env.clone(), + json!({"dynamic": "value", "dynamic_2": "value2"}), + ) + .await; + } + + // ===================================================================== + // STRING ADVANCED + // ===================================================================== + + #[tokio::test] + async fn test_string_advanced() { + let mut env = HashMap::new(); + env.insert( + "text".to_string(), + Arc::new(to_raw_value(&json!(" Hello, World! "))), + ); + env.insert( + "path".to_string(), + Arc::new(to_raw_value(&json!("/api/v1/users/123/profile"))), + ); + + assert_eval("text.trim()", env.clone(), json!("Hello, World!")).await; + assert_eval("text.trimStart()", env.clone(), json!("Hello, World! ")).await; + assert_eval("text.trimEnd()", env.clone(), json!(" Hello, World!")).await; + assert_eval("'42'.padStart(5, '0')", env.clone(), json!("00042")).await; + assert_eval("'42'.padEnd(5, '-')", env.clone(), json!("42---")).await; + assert_eval("'ab'.repeat(3)", env.clone(), json!("ababab")).await; + assert_eval( + "path.split('/').filter(p => p.length > 0)", + env.clone(), + json!(["api", "v1", "users", "123", "profile"]), + ) + .await; + } + + // ===================================================================== + // ARRAY MANIPULATION + // ===================================================================== + + #[tokio::test] + async fn test_array_manipulation() { + let mut env = HashMap::new(); + env.insert( + "items".to_string(), + Arc::new(to_raw_value(&json!([ + {"id": 1, "name": "Apple", "category": "fruit"}, + {"id": 2, "name": "Carrot", "category": "vegetable"}, + {"id": 3, "name": "Banana", "category": "fruit"}, + {"id": 4, "name": "Broccoli", "category": "vegetable"} + ]))), + ); + + assert_eval( + "items.find(i => i.name === 'Banana')", + env.clone(), + json!({"id": 3, "name": "Banana", "category": "fruit"}), + ) + .await; + assert_eval( + "items.findIndex(i => i.name === 'Banana')", + env.clone(), + json!(2), + ) + .await; + assert_eval( + "items.filter(i => i.category === 'fruit').map(i => i.name).sort()", + env.clone(), + json!(["Apple", "Banana"]), + ) + .await; + assert_eval( + "Array.from({length: 5}, (_, i) => i * 2)", + env.clone(), + json!([0, 2, 4, 6, 8]), + ) + .await; + assert_eval("Array(3).fill(0)", env.clone(), json!([0, 0, 0])).await; + assert_eval( + "[1, 2].concat([3, 4], [5, 6])", + env.clone(), + json!([1, 2, 3, 4, 5, 6]), + ) + .await; + assert_eval( + "items.map(i => i.name).join(' | ')", + env.clone(), + json!("Apple | Carrot | Banana | Broccoli"), + ) + .await; + } + + // ===================================================================== + // SET AND MAP OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_set_operations() { + let mut env = HashMap::new(); + env.insert( + "arr".to_string(), + Arc::new(to_raw_value(&json!([1, 2, 2, 3, 3, 3, 4]))), + ); + + assert_eval("[...new Set(arr)]", env.clone(), json!([1, 2, 3, 4])).await; + assert_eval("new Set(arr).size", env.clone(), json!(4)).await; + assert_eval("new Set(arr).has(3)", env.clone(), json!(true)).await; + assert_eval("new Set(arr).has(99)", env.clone(), json!(false)).await; + } + + #[tokio::test] + async fn test_map_operations() { + let env = HashMap::new(); + + assert_eval( + r#" + const map = new Map([['a', 1], ['b', 2], ['c', 3]]); + return Object.fromEntries(map) + "#, + env.clone(), + json!({"a": 1, "b": 2, "c": 3}), + ) + .await; + + assert_eval( + r#" + const map = new Map(); + map.set('key1', 'value1'); + map.set('key2', 'value2'); + return map.get('key1') + "#, + env.clone(), + json!("value1"), + ) + .await; + + assert_eval( + r#" + const map = new Map([['a', 1], ['b', 2]]); + return map.size + "#, + env.clone(), + json!(2), + ) + .await; + } + + // ===================================================================== + // COMPARISONS + // ===================================================================== + + #[tokio::test] + async fn test_comparisons() { + let env = HashMap::new(); + + assert_eval("1 === 1", env.clone(), json!(true)).await; + assert_eval("1 === '1'", env.clone(), json!(false)).await; + assert_eval("null === undefined", env.clone(), json!(false)).await; + assert_eval("null === null", env.clone(), json!(true)).await; + assert_eval("1 == '1'", env.clone(), json!(true)).await; + assert_eval("null == undefined", env.clone(), json!(true)).await; + assert_eval("5 !== '5'", env.clone(), json!(true)).await; + assert_eval("5 > 3", env.clone(), json!(true)).await; + assert_eval("5 >= 5", env.clone(), json!(true)).await; + assert_eval("3 < 5", env.clone(), json!(true)).await; + assert_eval("5 <= 5", env.clone(), json!(true)).await; + assert_eval("'apple' < 'banana'", env.clone(), json!(true)).await; + } + + // ===================================================================== + // BITWISE OPERATIONS + // ===================================================================== + + #[tokio::test] + async fn test_bitwise() { + let env = HashMap::new(); + + assert_eval("5 & 3", env.clone(), json!(1)).await; + assert_eval("5 | 3", env.clone(), json!(7)).await; + assert_eval("5 ^ 3", env.clone(), json!(6)).await; + assert_eval("~5", env.clone(), json!(-6)).await; + assert_eval("5 << 2", env.clone(), json!(20)).await; + assert_eval("20 >> 2", env.clone(), json!(5)).await; + } + + // ===================================================================== + // REAL-WORLD FLOW PATTERNS + // ===================================================================== + + #[tokio::test] + async fn test_flow_patterns_api_response() { + let mut env = HashMap::new(); + env.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!({ + "status": 200, + "data": { + "users": [ + {"id": 1, "email": "alice@example.com", "active": true}, + {"id": 2, "email": "bob@example.com", "active": false}, + {"id": 3, "email": "charlie@example.com", "active": true} + ], + "pagination": {"page": 1, "total": 50, "per_page": 10} + } + }))), + ); + + assert_eval( + "previous_result.data.users.filter(u => u.active).map(u => u.email)", + env.clone(), + json!(["alice@example.com", "charlie@example.com"]), + ) + .await; + + assert_eval( + r#" + const { page, total, per_page } = previous_result.data.pagination; + return page * per_page < total + "#, + env.clone(), + json!(true), + ) + .await; + } + + #[tokio::test] + async fn test_flow_patterns_batch_processing() { + let mut env = HashMap::new(); + env.insert( + "jobs".to_string(), + Arc::new(to_raw_value(&json!([ + {"id": 1, "status": "completed", "result": 100}, + {"id": 2, "status": "failed", "reason": "timeout"}, + {"id": 3, "status": "completed", "result": 200}, + {"id": 4, "status": "failed", "reason": "connection"}, + {"id": 5, "status": "completed", "result": 150} + ]))), + ); + + assert_eval( + r#" + const completed = jobs.filter(j => j.status === 'completed'); + const failed = jobs.filter(j => j.status === 'failed'); + const totalResult = completed.reduce((sum, j) => sum + j.result, 0); + return { + totalJobs: jobs.length, + completedCount: completed.length, + failedCount: failed.length, + successRate: completed.length / jobs.length, + totalResult, + failureReasons: failed.map(j => j.reason) + } + "#, + env.clone(), + json!({ + "totalJobs": 5, + "completedCount": 3, + "failedCount": 2, + "successRate": 0.6, + "totalResult": 450, + "failureReasons": ["timeout", "connection"] + }), + ) + .await; + } + + #[tokio::test] + async fn test_flow_patterns_config_merge() { + let mut env = HashMap::new(); + env.insert( + "defaults".to_string(), + Arc::new(to_raw_value(&json!({ + "timeout": 5000, + "retries": 3, + "headers": {"Content-Type": "application/json"}, + "features": {"logging": true, "caching": false} + }))), + ); + env.insert( + "overrides".to_string(), + Arc::new(to_raw_value(&json!({ + "timeout": 10000, + "headers": {"Authorization": "Bearer token"}, + "features": {"caching": true} + }))), + ); + + assert_eval( + r#"({ + ...defaults, + ...overrides, + headers: { ...defaults.headers, ...overrides.headers }, + features: { ...defaults.features, ...overrides.features } + })"#, + env.clone(), + json!({ + "timeout": 10000, + "retries": 3, + "headers": {"Content-Type": "application/json", "Authorization": "Bearer token"}, + "features": {"logging": true, "caching": true} + }), + ) + .await; + } + + // ===================================================================== + // FLOW SIMULATION: Multi-step flow with various step results + // ===================================================================== + + fn create_multi_step_flow_context() -> ( + HashMap>>, + Option>>>, + HashMap>, + ) { + let mut ctx = HashMap::new(); + ctx.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); + ctx.insert( + "b".to_string(), + Arc::new(to_raw_value(&json!({ + "status": "success", + "data": { + "users": [ + {"id": 1, "name": "Alice", "active": true, "roles": ["admin", "user"]}, + {"id": 2, "name": "Bob", "active": false, "roles": ["user"]}, + {"id": 3, "name": "Charlie", "active": true, "roles": ["moderator", "user"]} + ], + "total": 3, + "metadata": {"page": 1, "hasMore": true} + } + }))), + ); + ctx.insert( + "c".to_string(), + Arc::new(to_raw_value(&json!([10, 20, 30, 40, 50]))), + ); + ctx.insert("d".to_string(), Arc::new(to_raw_value(&json!(null)))); + ctx.insert( + "f".to_string(), + Arc::new(to_raw_value(&json!({ + "level1": {"level2": {"level3": {"level4": {"value": "deeply_nested"}}}} + }))), + ); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!([ + "string", 123, true, null, {"key": "value"}, [1, 2, 3] + ]))), + ); + + let mut fi = HashMap::new(); + fi.insert("name".to_string(), to_raw_value(&json!("test_flow"))); + fi.insert("count".to_string(), to_raw_value(&json!(100))); + fi.insert("enabled".to_string(), to_raw_value(&json!(true))); + fi.insert( + "config".to_string(), + to_raw_value(&json!({ + "timeout": 30, "retries": 3, "options": ["fast", "secure"] + })), + ); + fi.insert( + "items".to_string(), + to_raw_value(&json!([ + {"id": 1, "value": "first"}, + {"id": 2, "value": "second"}, + {"id": 3, "value": "third"} + ])), + ); + + let mut fe = HashMap::new(); + fe.insert("ENV".to_string(), to_raw_value(&json!("production"))); + fe.insert("DEBUG".to_string(), to_raw_value(&json!(false))); + fe.insert("VERSION".to_string(), to_raw_value(&json!("1.2.3"))); + + (ctx, Some(mappable_rc::Marc::new(fi)), fe) + } + + #[tokio::test] + async fn test_flow_step_references() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full("a", ctx.clone(), fi.clone(), Some(&fe), json!(42)).await; + assert_eval_full( + "b.status", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("success"), + ) + .await; + assert_eval_full("b.data.total", ctx.clone(), fi.clone(), Some(&fe), json!(3)).await; + assert_eval_full( + "b.data.users[0].name", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("Alice"), + ) + .await; + assert_eval_full("c[0]", ctx.clone(), fi.clone(), Some(&fe), json!(10)).await; + assert_eval_full( + "c.map(x => x * 2)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!([20, 40, 60, 80, 100]), + ) + .await; + assert_eval_full( + "f.level1.level2.level3.level4.value", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("deeply_nested"), + ) + .await; + } + + #[tokio::test] + async fn test_flow_input_nested_and_combined() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full( + "flow_input.name", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("test_flow"), + ) + .await; + assert_eval_full( + "flow_input.config.timeout", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(30), + ) + .await; + assert_eval_full( + "flow_input.config.options[0]", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("fast"), + ) + .await; + assert_eval_full( + "flow_input.items.length", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(3), + ) + .await; + assert_eval_full( + "flow_input.items[0].id", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(1), + ) + .await; + assert_eval_full( + "flow_input.items.map(i => i.value)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(["first", "second", "third"]), + ) + .await; + + // Combining flow_input with step results + assert_eval_full( + "flow_input.count + a", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(142), + ) + .await; + assert_eval_full( + "flow_input.enabled ? b.data.users : []", + ctx.clone(), + fi.clone(), + Some(&fe), + json!([ + {"id": 1, "name": "Alice", "active": true, "roles": ["admin", "user"]}, + {"id": 2, "name": "Bob", "active": false, "roles": ["user"]}, + {"id": 3, "name": "Charlie", "active": true, "roles": ["moderator", "user"]} + ]), + ) + .await; + } + + #[tokio::test] + async fn test_flow_env_access() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full( + "flow_env.ENV", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("production"), + ) + .await; + assert_eval_full( + "flow_env.DEBUG", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(false), + ) + .await; + assert_eval_full( + "flow_env.VERSION", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("1.2.3"), + ) + .await; + } + + #[tokio::test] + async fn test_flow_branch_conditions() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full("a > 40", ctx.clone(), fi.clone(), Some(&fe), json!(true)).await; + assert_eval_full( + "b.status === 'success'", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "flow_input.enabled", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "a > 40 && b.status === 'success'", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "b.data.users.length > 0", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "b.data.users.some(u => u.active)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + assert_eval_full( + "c.includes(30)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(true), + ) + .await; + } + + #[tokio::test] + async fn test_flow_complex_data_extraction() { + let (ctx, fi, fe) = create_multi_step_flow_context(); + + assert_eval_full( + "b.data.users.filter(u => u.active).map(u => u.name)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(["Alice", "Charlie"]), + ) + .await; + + assert_eval_full( + "b.data.users.filter(u => u.roles.includes('admin'))[0]?.name", + ctx.clone(), + fi.clone(), + Some(&fe), + json!("Alice"), + ) + .await; + + assert_eval_full( + "b.data.users.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(2), + ) + .await; + + // Combining multiple step results + assert_eval_full("a + c[0]", ctx.clone(), fi.clone(), Some(&fe), json!(52)).await; + assert_eval_full( + "a * b.data.total", + ctx.clone(), + fi.clone(), + Some(&fe), + json!(126), + ) + .await; + } + + #[tokio::test] + async fn test_flow_forloop_inner_expressions() { + let mut ctx = HashMap::new(); + ctx.insert( + "previous_result".to_string(), + Arc::new(to_raw_value(&json!({"value": 42, "index": 2}))), + ); + + let mut fi = HashMap::new(); + fi.insert( + "iter".to_string(), + to_raw_value(&json!({"index": 2, "value": {"id": 3, "name": "test_item"}})), + ); + fi.insert("name".to_string(), to_raw_value(&json!("parent_flow"))); + let fi = Some(mappable_rc::Marc::new(fi)); + + assert_eval_full( + "flow_input.iter.index", + ctx.clone(), + fi.clone(), + None, + json!(2), + ) + .await; + assert_eval_full( + "flow_input.iter.value.id", + ctx.clone(), + fi.clone(), + None, + json!(3), + ) + .await; + assert_eval_full( + "flow_input.iter.value.name", + ctx.clone(), + fi.clone(), + None, + json!("test_item"), + ) + .await; + assert_eval_full( + "`Item ${flow_input.iter.index} of ${flow_input.name}`", + ctx.clone(), + fi.clone(), + None, + json!("Item 2 of parent_flow"), + ) + .await; + } + + // ===================================================================== + // EDGE CASES + // ===================================================================== + + #[tokio::test] + async fn test_edge_cases_empty_values() { + let mut env = HashMap::new(); + env.insert("emptyArray".to_string(), Arc::new(to_raw_value(&json!([])))); + env.insert( + "emptyObject".to_string(), + Arc::new(to_raw_value(&json!({}))), + ); + env.insert( + "emptyString".to_string(), + Arc::new(to_raw_value(&json!(""))), + ); + env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); + + assert_eval("emptyArray.length", env.clone(), json!(0)).await; + assert_eval("emptyArray.map(x => x * 2)", env.clone(), json!([])).await; + assert_eval( + "emptyArray.reduce((a, b) => a + b, 100)", + env.clone(), + json!(100), + ) + .await; + assert_eval("Object.keys(emptyObject)", env.clone(), json!([])).await; + assert_eval("emptyString.length", env.clone(), json!(0)).await; + assert_eval("emptyString || 'default'", env.clone(), json!("default")).await; + assert_eval("emptyString ?? 'default'", env.clone(), json!("")).await; + assert_eval("zero || 'default'", env.clone(), json!("default")).await; + assert_eval("zero ?? 'default'", env.clone(), json!(0)).await; + } + + #[tokio::test] + async fn test_edge_cases_deep_access() { + let mut env = HashMap::new(); + env.insert( + "deep".to_string(), + Arc::new(to_raw_value( + &json!({"a": {"b": {"c": {"d": {"e": "found!"}}}}}), + )), + ); + + assert_eval("deep.a.b.c.d.e", env.clone(), json!("found!")).await; + assert_eval("deep?.a?.b?.c?.d?.e", env.clone(), json!("found!")).await; + assert_eval("deep?.a?.b?.x?.y?.z", env.clone(), json!(null)).await; + assert_eval( + "deep?.a?.b?.x?.y?.z ?? 'not found'", + env.clone(), + json!("not found"), + ) + .await; + } + + #[tokio::test] + async fn test_edge_cases_special_characters() { + let mut env = HashMap::new(); + env.insert( + "data".to_string(), + Arc::new(to_raw_value(&json!({ + "key-with-dash": "value1", + "key.with.dots": "value2", + "key with spaces": "value3" + }))), + ); + + assert_eval("data['key-with-dash']", env.clone(), json!("value1")).await; + assert_eval("data['key.with.dots']", env.clone(), json!("value2")).await; + assert_eval("data['key with spaces']", env.clone(), json!("value3")).await; + } + + #[tokio::test] + async fn test_edge_cases_boolean_coercion() { + let env = HashMap::new(); + + assert_eval("Boolean(0)", env.clone(), json!(false)).await; + assert_eval("Boolean('')", env.clone(), json!(false)).await; + assert_eval("Boolean(null)", env.clone(), json!(false)).await; + assert_eval("Boolean(undefined)", env.clone(), json!(false)).await; + assert_eval("Boolean(NaN)", env.clone(), json!(false)).await; + assert_eval("Boolean(1)", env.clone(), json!(true)).await; + assert_eval("Boolean('hello')", env.clone(), json!(true)).await; + assert_eval("Boolean([])", env.clone(), json!(true)).await; + assert_eval("Boolean({})", env.clone(), json!(true)).await; + assert_eval("!!0", env.clone(), json!(false)).await; + assert_eval("!!1", env.clone(), json!(true)).await; + } + + // ===================================================================== + // PROMISE RESOLUTION + // ===================================================================== + + #[tokio::test] + async fn test_promise_resolve() { + let env = HashMap::new(); + + assert_eval("Promise.resolve(42)", env.clone(), json!(42)).await; + assert_eval( + "Promise.resolve({ key: 'value' })", + env.clone(), + json!({"key": "value"}), + ) + .await; + assert_eval( + "Promise.all([Promise.resolve(1), Promise.resolve(2), Promise.resolve(3)])", + env.clone(), + json!([1, 2, 3]), + ) + .await; + } + + // ===================================================================== + // eval_simple_js TESTS + // ===================================================================== + + #[tokio::test] + async fn test_eval_simple_js_basic() { + let mut globals = HashMap::new(); + globals.insert("job".to_string(), json!({"id": "abc", "input": {"x": 42}})); + + let result = eval_simple_js("job.input.x".to_string(), globals) + .await + .unwrap(); + assert_eq!(result.get(), "42"); + } + + #[tokio::test] + async fn test_eval_simple_js_string() { + let mut globals = HashMap::new(); + globals.insert("name".to_string(), json!("World")); + + let result = eval_simple_js("`Hello ${name}!`".to_string(), globals) + .await + .unwrap(); + assert_eq!(result.get(), "\"Hello World!\""); + } + + #[tokio::test] + async fn test_eval_simple_js_array_transform() { + let mut globals = HashMap::new(); + globals.insert("data".to_string(), json!([1, 2, 3, 4, 5])); + + let result = eval_simple_js( + "data.filter(x => x > 2).map(x => x * 10)".to_string(), + globals, + ) + .await + .unwrap(); + let actual: serde_json::Value = serde_json::from_str(result.get()).unwrap(); + assert_eq!(actual, json!([30, 40, 50])); + } + + #[tokio::test] + async fn test_eval_simple_js_null_handling() { + let mut globals = HashMap::new(); + globals.insert("x".to_string(), json!(null)); + + let result = eval_simple_js("x ?? 'default'".to_string(), globals) + .await + .unwrap(); + assert_eq!(result.get(), "\"default\""); + } + + #[tokio::test] + async fn test_eval_simple_js_multiple_globals() { + let mut globals = HashMap::new(); + globals.insert("a".to_string(), json!(10)); + globals.insert("b".to_string(), json!(20)); + globals.insert("prefix".to_string(), json!("result")); + + let result = eval_simple_js( + "({ label: `${prefix}_sum`, value: a + b })".to_string(), + globals, + ) + .await + .unwrap(); + let actual: serde_json::Value = serde_json::from_str(result.get()).unwrap(); + assert_eq!(actual, json!({"label": "result_sum", "value": 30})); + } + + #[tokio::test] + async fn test_eval_simple_js_error_handling() { + let globals = HashMap::new(); + + // Invalid JS should return an error + let result = eval_simple_js("this is not valid js @#$".to_string(), globals).await; + assert!(result.is_err()); + } +} diff --git a/backend/windmill-mcp/Cargo.toml b/backend/windmill-mcp/Cargo.toml index afe2547f19..3968ea0836 100644 --- a/backend/windmill-mcp/Cargo.toml +++ b/backend/windmill-mcp/Cargo.toml @@ -11,7 +11,7 @@ path = "src/lib.rs" [features] default = [] server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker", "dep:sqlx", "dep:async-trait", "dep:http", "dep:tokio-util", "dep:tokio"] -auth = ["rmcp/auth", "dep:oauth2"] +auth = ["rmcp/auth", "dep:oauth2", "dep:sqlx", "dep:chrono"] [dependencies] oauth2 = { version = "5.0", optional = true } @@ -23,6 +23,7 @@ serde_json.workspace = true tracing.workspace = true rmcp.workspace = true sqlx = { workspace = true, optional = true } +chrono = { workspace = true, optional = true } async-trait = { workspace = true, optional = true } http = { workspace = true, optional = true } tokio-util = { workspace = true, features = ["rt"], optional = true } diff --git a/backend/windmill-mcp/src/client_registration.rs b/backend/windmill-mcp/src/client_registration.rs new file mode 100644 index 0000000000..87736f2915 --- /dev/null +++ b/backend/windmill-mcp/src/client_registration.rs @@ -0,0 +1,208 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! MCP OAuth client registration and credential management. +//! +//! Handles dynamic client registration (DCR) and Client Identifier Metadata Document (CIMD) +//! flows for MCP OAuth, including caching registered credentials in the database. + +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use windmill_common::db::DB; +use windmill_common::error; +use windmill_common::variables::{build_crypt, decrypt, encrypt}; + +use crate::oauth::AuthorizationManager; + +/// MCP client credentials returned by [`get_or_refresh_mcp_client`]. +pub struct McpClientCredentials { + pub client_id: String, + pub client_secret: Option, + pub token_endpoint: String, +} + +#[derive(FromRow)] +struct McpOAuthClient { + #[allow(dead_code)] + mcp_server_url: String, + client_id: String, + client_secret: Option, + client_secret_expires_at: Option, + token_endpoint: String, +} + +impl McpOAuthClient { + fn is_expired(&self) -> bool { + if let Some(expires_at) = self.client_secret_expires_at { + expires_at < chrono::Utc::now().naive_utc() + } else { + false + } + } +} + +async fn encrypt_client_secret(db: &DB, client_secret: &str) -> error::Result { + let mc = build_crypt(db, "admins").await?; + Ok(encrypt(&mc, client_secret)) +} + +async fn decrypt_client_secret(db: &DB, encrypted_secret: &str) -> error::Result { + let mc = build_crypt(db, "admins").await?; + decrypt(&mc, encrypted_secret.to_string()) +} + +#[derive(Serialize)] +struct DcrRequest { + client_name: String, + redirect_uris: Vec, + grant_types: Vec, + response_types: Vec, + token_endpoint_auth_method: String, +} + +#[derive(Deserialize)] +struct DcrResponse { + client_id: String, + client_secret: Option, + client_secret_expires_at: Option, +} + +async fn register_client( + registration_endpoint: &str, + redirect_uri: &str, + client_name: &str, +) -> Result { + let client = reqwest::Client::new(); + let request = DcrRequest { + client_name: client_name.to_string(), + redirect_uris: vec![redirect_uri.to_string()], + grant_types: vec![ + "authorization_code".to_string(), + "refresh_token".to_string(), + ], + response_types: vec!["code".to_string()], + token_endpoint_auth_method: "none".to_string(), + }; + + let response = client + .post(registration_endpoint) + .json(&request) + .send() + .await + .map_err(|e| error::Error::BadRequest(format!("DCR request failed: {e}")))?; + + if !response.status().is_success() { + let error_text = response.text().await.unwrap_or_default(); + return Err(error::Error::BadRequest(format!( + "DCR failed: {error_text}" + ))); + } + + response + .json::() + .await + .map_err(|e| error::Error::BadRequest(format!("DCR response parse failed: {e}"))) +} + +/// Get MCP client credentials from cache, re-registering if expired. +/// +/// Supports two modes: +/// - DCR (Dynamic Client Registration): If the server's metadata includes a registration endpoint +/// - CIMD (Client Identifier Metadata Document): Otherwise, uses our metadata URL as client_id +pub async fn get_or_refresh_mcp_client( + db: &DB, + mcp_server_url: &str, +) -> Result { + let base_url = windmill_common::BASE_URL.read().await.clone(); + let redirect_uri = format!("{}/api/mcp/oauth/callback", base_url); + + let cached_client: Option = + sqlx::query_as("SELECT * FROM mcp_oauth_client WHERE mcp_server_url = $1") + .bind(mcp_server_url) + .fetch_optional(db) + .await + .map_err(|e| error::Error::InternalErr(format!("Database error: {e}")))?; + + if let Some(client) = cached_client { + if !client.is_expired() { + tracing::debug!("Using cached MCP client for {}", mcp_server_url); + let decrypted_secret = if let Some(ref encrypted_secret) = client.client_secret { + Some(decrypt_client_secret(db, encrypted_secret).await?) + } else { + None + }; + return Ok(McpClientCredentials { + client_id: client.client_id, + client_secret: decrypted_secret, + token_endpoint: client.token_endpoint, + }); + } + tracing::debug!("Cached MCP client expired, re-registering"); + } + + let manager = AuthorizationManager::new(mcp_server_url) + .await + .map_err(|e| error::Error::BadRequest(format!("Failed to create auth manager: {e}")))?; + + let metadata = manager + .discover_metadata() + .await + .map_err(|e| error::Error::BadRequest(format!("OAuth discovery failed: {e}")))?; + + let supports_dynamic_registration = metadata.registration_endpoint.is_some(); + + let (client_id, client_secret, expires_at) = if supports_dynamic_registration { + let reg_endpoint = metadata.registration_endpoint.as_ref().unwrap(); + tracing::debug!("Performing DCR at {}", reg_endpoint); + let dcr = register_client(reg_endpoint, &redirect_uri, "Windmill").await?; + + let expires_at = dcr.client_secret_expires_at.and_then(|ts| { + if ts == 0 { + None + } else { + chrono::DateTime::from_timestamp(ts, 0).map(|dt| dt.naive_utc()) + } + }); + + (dcr.client_id, dcr.client_secret, expires_at) + } else { + let client_metadata_url = format!("{}/api/mcp/oauth/client-metadata.json", base_url); + tracing::debug!( + "Using CIMD with metadata URL as client_id: {}", + client_metadata_url + ); + + (client_metadata_url, None, None) + }; + + let encrypted_secret = if let Some(ref secret) = client_secret { + Some(encrypt_client_secret(db, secret).await?) + } else { + None + }; + + sqlx::query( + "INSERT INTO mcp_oauth_client (mcp_server_url, client_id, client_secret, client_secret_expires_at, token_endpoint) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (mcp_server_url) DO UPDATE + SET client_id = EXCLUDED.client_id, + client_secret = EXCLUDED.client_secret, + client_secret_expires_at = EXCLUDED.client_secret_expires_at, + token_endpoint = EXCLUDED.token_endpoint", + ) + .bind(mcp_server_url) + .bind(&client_id) + .bind(&encrypted_secret) + .bind(expires_at) + .bind(&metadata.token_endpoint) + .execute(db) + .await + .map_err(|e| error::Error::InternalErr(format!("Database error: {e}")))?; + + Ok(McpClientCredentials { client_id, client_secret, token_endpoint: metadata.token_endpoint }) +} diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index 9026837493..a75b545154 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -29,6 +29,10 @@ pub use rmcp::model::Tool as McpTool; #[cfg(feature = "server")] pub mod server; +// MCP OAuth client registration (when auth feature is enabled) +#[cfg(feature = "auth")] +pub mod client_registration; + // Re-export rmcp auth types when auth feature is enabled #[cfg(feature = "auth")] pub mod oauth { diff --git a/backend/windmill-native-triggers/Cargo.toml b/backend/windmill-native-triggers/Cargo.toml new file mode 100644 index 0000000000..126c22c7c0 --- /dev/null +++ b/backend/windmill-native-triggers/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "windmill-native-triggers" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_native_triggers" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] +native_trigger = ["dep:windmill-oauth", "dep:backon"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true +windmill-queue.workspace = true +windmill-oauth = { workspace = true, optional = true } +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +http.workspace = true +tracing.workspace = true +tokio.workspace = true +async-trait.workspace = true +lazy_static.workspace = true +reqwest.workspace = true +chrono.workspace = true +itertools.workspace = true +anyhow.workspace = true +base64.workspace = true +hmac.workspace = true +sha2.workspace = true +urlencoding.workspace = true +uuid.workspace = true +strum.workspace = true +backon = { workspace = true, optional = true } diff --git a/backend/windmill-api/src/native_triggers/handler.rs b/backend/windmill-native-triggers/src/handler.rs similarity index 93% rename from backend/windmill-api/src/native_triggers/handler.rs rename to backend/windmill-native-triggers/src/handler.rs index 3c42b5261b..ef054d4664 100644 --- a/backend/windmill-api/src/native_triggers/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -1,13 +1,8 @@ use crate::{ - db::ApiAuthed, - native_triggers::{ - delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix, - get_workspace_integration, list_native_triggers, store_native_trigger, - update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, - NativeTriggerData, ServiceName, - }, - users::{create_token_internal, NewToken}, - utils::check_scopes, + delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix, + get_workspace_integration, list_native_triggers, store_native_trigger, + update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, NativeTriggerData, + ServiceName, }; use axum::{ extract::{Path, Query}, @@ -17,6 +12,9 @@ use axum::{ use serde::{Deserialize, Serialize}; use sqlx::PgConnection; use std::sync::Arc; +use windmill_api_auth::{ + check_scopes, create_token_internal, require_is_writer, ApiAuthed, NewToken, +}; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, @@ -33,9 +31,25 @@ async fn require_is_writer_on_runnable( db: DB, ) -> Result<()> { if is_flow { - crate::flows::require_is_writer(authed, path, w_id, db).await + require_is_writer( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM flow WHERE path = $1 AND workspace_id = $2", + "flow", + ) + .await } else { - crate::scripts::require_is_writer(authed, path, w_id, db).await + require_is_writer( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "script", + ) + .await } } @@ -548,11 +562,11 @@ pub fn generate_native_trigger_routers() -> Router { #[cfg(feature = "native_trigger")] { - use crate::native_triggers::nextcloud::NextCloud; + use crate::nextcloud::NextCloud; // Register all service routes here // When adding a new service: - // 1. Import the handler: use crate::native_triggers::newservice::NewServiceHandler; + // 1. Import the handler: use crate::newservice::NewServiceHandler; // 2. Add the route: .nest("/newservice", service_routes(NewServiceHandler)) return router.nest("/nextcloud", service_routes(NextCloud)); // Add new services here: diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs new file mode 100644 index 0000000000..456580b3ef --- /dev/null +++ b/backend/windmill-native-triggers/src/lib.rs @@ -0,0 +1,1053 @@ +//! Native Triggers Module +//! +//! This module provides integration with external services (like Nextcloud) that can +//! trigger Windmill scripts/flows via webhooks. +//! +//! ## Adding a New Native Trigger Service +//! +//! When adding a new service (e.g., "NewService"), you need to update the following locations: +//! +//! ### 1. This file (lib.rs): +//! - Add `pub mod newservice;` under the `#[cfg(feature = "native_trigger")]` block +//! - Add `NewService` variant to `ServiceName` enum +//! - Update `ServiceName::as_str()` - add match arm returning `"newservice"` +//! - Update `TryFrom for ServiceName` - add match arm for `"newservice"` +//! - Update `ServiceName::as_trigger_kind()` - add match arm (requires TriggerKind::NewService in windmill_common) +//! - Update `ServiceName::as_job_trigger_kind()` - add match arm (requires JobTriggerKind::NewService in windmill_common) +//! - Update `ServiceName::fmt()` (Display impl) - add match arm +//! +//! ### 2. sync.rs: +//! - Add `sync_service!()` macro call in `sync_all_triggers()` +//! +//! ### 3. handler.rs: +//! - Add `.nest("/newservice", service_routes(NewServiceHandler))` in `generate_native_trigger_routers()` +//! +//! ### 4. Database migration: +//! - Add `'newservice'` to the `native_trigger_service` enum type +//! +//! ### 5. windmill_common (if needed): +//! - Add `NewService` variant to `TriggerKind` enum +//! - Add `'newservice'` to `job_trigger_kind` enum type in migration +//! +//! The generic code (trait definitions, route handlers, database operations) does NOT +//! need modification when adding new services. + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use http::StatusCode; +use itertools::Itertools; +use reqwest::{Client, Method}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::json; +use serde_json::value::RawValue; +use sqlx::{FromRow, PgConnection, Postgres}; +use std::{collections::HashMap, fmt::Debug}; +use strum::{EnumIter, IntoEnumIterator}; +use tokio::task; +use windmill_common::{ + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + variables::{build_crypt, decrypt, encrypt}, + DB, +}; +use windmill_queue::PushArgsOwned; + +#[cfg(feature = "native_trigger")] +use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; + +use windmill_api_auth::ApiAuthed; +pub mod handler; +pub mod sync; +pub mod workspace_integrations; + +// Service modules - add new services here: +#[cfg(feature = "native_trigger")] +pub mod nextcloud; +// #[cfg(feature = "native_trigger")] +// pub mod newservice; + +/// Enum of all supported native trigger services. +/// When adding a new service, add a variant here (e.g., `NewService`). +#[derive(EnumIter, sqlx::Type, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[sqlx(type_name = "native_trigger_service", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum ServiceName { + Nextcloud, + // Add new services here: + // NewService, +} + +impl TryFrom for ServiceName { + type Error = Error; + fn try_from(value: String) -> std::result::Result { + // Add new service match arms here: + let service = match value.as_str() { + "nextcloud" => ServiceName::Nextcloud, + // "newservice" => ServiceName::NewService, + _ => { + return Err(anyhow::anyhow!( + "Unknown service, currently supported services are: [{}]", + ServiceName::iter().join(",") + ) + .into()) + } + }; + + Ok(service) + } +} + +impl ServiceName { + /// Returns the lowercase string identifier for this service. + /// Add new service match arms here. + pub fn as_str(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "nextcloud", + // ServiceName::NewService => "newservice", + } + } + + /// Returns the corresponding TriggerKind for this service. + /// Requires adding the variant to TriggerKind in windmill_common. + pub fn as_trigger_kind(&self) -> TriggerKind { + match self { + ServiceName::Nextcloud => TriggerKind::Nextcloud, + // ServiceName::NewService => TriggerKind::NewService, + } + } + + /// Returns the corresponding JobTriggerKind for this service. + /// Requires adding the variant to JobTriggerKind in windmill_common. + pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind { + match self { + ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud, + // ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService, + } + } + + /// Returns the OAuth token endpoint path for this service. + /// Used for building OAuth clients dynamically. + pub fn token_endpoint(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", + // ServiceName::NewService => "/oauth/token", + } + } + + /// Returns the OAuth authorization endpoint path for this service. + /// Used for building OAuth authorization URLs. + pub fn auth_endpoint(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "/apps/oauth2/authorize", + // ServiceName::NewService => "/oauth/authorize", + } + } +} + +impl std::fmt::Display for ServiceName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct NativeTrigger { + pub external_id: String, + pub workspace_id: String, + pub service_name: ServiceName, + pub script_path: String, + pub is_flow: bool, + pub webhook_token_prefix: String, + pub service_config: Option, + pub error: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NativeTriggerConfig { + pub script_path: String, + pub is_flow: bool, + pub webhook_token: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct NativeTriggerData { + pub script_path: String, + pub is_flow: bool, + pub service_config: C, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WorkspaceIntegration { + pub workspace_id: String, + pub service_name: ServiceName, + pub oauth_data: serde_json::Value, + pub created_at: DateTime, + pub updated_at: DateTime, + pub created_by: String, +} + +#[async_trait] +pub trait External: Send + Sync + 'static { + type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + type TriggerData: Debug + Serialize + Send + Sync; + type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; + type CreateResponse: DeserializeOwned + Send + Sync; + + const SUPPORT_WEBHOOK: bool; + const SERVICE_NAME: ServiceName; + const DISPLAY_NAME: &'static str; + const TOKEN_ENDPOINT: &'static str; + const REFRESH_ENDPOINT: &'static str; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()>; + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()>; + + #[allow(unused)] + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn list_all( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + db: &DB, + tx: &mut PgConnection, + ) -> Result>; + + async fn prepare_webhook( + &self, + _db: &DB, + _w_id: &str, + _header: HashMap, + _body: String, + _script_path: &str, + _is_flow: bool, + ) -> Result { + Ok(PushArgsOwned { extra: None, args: HashMap::new() }) + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option); + + fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String; + + /// Extracts the service-specific config from trigger data (from external service). + /// Used for comparison during sync to detect config drift. + /// Default implementation converts the trigger data to a JSON value + /// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method. + fn extract_service_config_from_trigger_data( + &self, + data: &Self::TriggerData, + ) -> Result { + serde_json::to_value(data).map_err(|e| { + Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e)) + }) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn http_client_request( + &self, + url: &str, + method: Method, + workspace_id: &str, + tx: &mut PgConnection, + db: &DB, + headers: Option>, + body: Option<&B>, + ) -> Result { + let oauth_config: OAuthConfig = + decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?; + + let result = make_http_request( + url, + method.clone(), + headers.clone(), + body.as_ref(), + &oauth_config.access_token, + ) + .await; + + match result { + Ok(response) => Ok(response), + Err(err) + if err.status() == Some(StatusCode::UNAUTHORIZED) + || err.status() == Some(StatusCode::FORBIDDEN) => + { + tracing::info!( + "HTTP auth error ({}), attempting token refresh", + err.status().unwrap() + ); + + let refreshed_oauth_config = + refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?; + + task::spawn({ + let db_clone = db.clone(); + let workspace_id_clone = workspace_id.to_string(); + let refreshed_json = oauth_config_to_json(&refreshed_oauth_config); + async move { + update_workspace_integration_tokens_helper( + db_clone, + workspace_id_clone, + Self::SERVICE_NAME, + refreshed_json, + ) + .await; + } + }); + + let response = make_http_request( + url, + method, + headers, + body.as_ref(), + &refreshed_oauth_config.access_token, + ) + .await + .map_err(to_anyhow)?; + Ok(response) + } + Err(e) => Err(to_anyhow(e).into()), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct OAuthConfig { + pub base_url: String, + pub access_token: String, + pub refresh_token: Option, + pub client_id: String, + pub client_secret: String, +} + +pub async fn make_http_request( + url: &str, + method: Method, + headers: Option>, + body: Option<&B>, + access_token: &str, +) -> std::result::Result { + let client = Client::new(); + let mut request = client.request(method, url); + + request = request + .header("Accept", "application/json") + .header("Authorization", format!("Bearer {}", access_token)); + + if body.is_some() { + request = request.header("Content-Type", "application/json"); + } + + if let Some(custom_headers) = headers { + for (key, value) in custom_headers { + request = request.header(key, value); + } + } + + if let Some(body_content) = body { + request = request.json(body_content); + } + + let response = request.send().await?.error_for_status()?; + + let response_json = response.json().await?; + + Ok(response_json) +} + +pub async fn decrypt_oauth_data< + 'c, + E: sqlx::Executor<'c, Database = Postgres>, + T: DeserializeOwned, +>( + tx: E, + db: &DB, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let integration = get_workspace_integration(tx, workspace_id, service_name).await?; + + let mc = build_crypt(db, workspace_id).await?; + let mut oauth_data: serde_json::Value = integration.oauth_data; + + if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) { + let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string()) + .map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?; + oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token); + } + + if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str()) + { + let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string()) + .map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?; + oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token); + } + + serde_json::from_value(oauth_data) + .map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e))) +} + +#[allow(unused)] +pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result { + let base_url = oauth_data + .get("base_url") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))? + .to_string(); + + let access_token = oauth_data + .get("access_token") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))? + .to_string(); + + let refresh_token = oauth_data + .get("refresh_token") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let client_id = oauth_data + .get("client_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))? + .to_string(); + + let client_secret = oauth_data + .get("client_secret") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))? + .to_string(); + + Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret }) +} + +#[inline] +pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value { + let mut json = json!({ + "base_url": config.base_url, + "access_token": config.access_token, + "client_id": config.client_id, + "client_secret": config.client_secret, + }); + + if let Some(refresh_token) = &config.refresh_token { + json["refresh_token"] = serde_json::Value::String(refresh_token.clone()); + } + + json +} + +/// Token refresh response +#[cfg(feature = "native_trigger")] +#[derive(Debug, Deserialize)] +struct RefreshTokenResponse { + access_token: String, + refresh_token: Option, +} + +/// Refresh OAuth tokens using windmill-oauth. +#[cfg(feature = "native_trigger")] +pub async fn refresh_oauth_tokens( + oauth_config: &OAuthConfig, + refresh_endpoint: &str, +) -> Result { + let refresh_token_str = oauth_config + .refresh_token + .as_ref() + .ok_or_else(|| Error::InternalErr("No refresh token available".to_string()))?; + + // Build OAuth client for token refresh + // Auth URL is not used for refresh, but required by the client constructor + let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url)) + .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; + let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint)) + .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; + + let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url); + client.set_client_secret(oauth_config.client_secret.clone()); + + let token_response: RefreshTokenResponse = client + .exchange_refresh_token(&RefreshToken::from(refresh_token_str.as_str())) + .with_client(&*OAUTH_HTTP_CLIENT) + .execute() + .await + .map_err(|e| Error::InternalErr(format!("Failed to refresh token: {:?}", e)))?; + + Ok(OAuthConfig { + base_url: oauth_config.base_url.clone(), + access_token: token_response.access_token, + refresh_token: token_response + .refresh_token + .or_else(|| oauth_config.refresh_token.clone()), + client_id: oauth_config.client_id.clone(), + client_secret: oauth_config.client_secret.clone(), + }) +} + +/// Fallback refresh without native_triggers feature +#[cfg(not(feature = "native_trigger"))] +pub async fn refresh_oauth_tokens( + _oauth_config: &OAuthConfig, + _refresh_endpoint: &str, +) -> Result { + Err(Error::InternalErr( + "Native triggers feature is not enabled".to_string(), + )) +} + +async fn update_workspace_integration_tokens_helper( + db: DB, + workspace_id: String, + service_name: ServiceName, + oauth_data: serde_json::Value, +) { + let result = async { + let mut tx = db.begin().await?; + let mc = build_crypt(&db, &workspace_id).await?; + let mut encrypted_oauth_data = oauth_data; + + if let Some(access_token) = encrypted_oauth_data + .get("access_token") + .and_then(|v| v.as_str()) + { + let encrypted_access_token = encrypt(&mc, access_token); + encrypted_oauth_data["access_token"] = + serde_json::Value::String(encrypted_access_token); + } + + if let Some(refresh_token) = encrypted_oauth_data + .get("refresh_token") + .and_then(|v| v.as_str()) + { + let encrypted_refresh_token = encrypt(&mc, refresh_token); + encrypted_oauth_data["refresh_token"] = + serde_json::Value::String(encrypted_refresh_token); + } + + sqlx::query!( + r#" + UPDATE workspace_integrations + SET oauth_data = $1, updated_at = now() + WHERE workspace_id = $2 AND service_name = $3 + "#, + encrypted_oauth_data, + workspace_id, + service_name as ServiceName, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok::<(), Error>(()) + } + .await; + + if let Err(e) = result { + tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}", + service_name, workspace_id, e); + } +} + +/// Look up the full token from the token table using its prefix +pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + token_prefix: &str, +) -> Result> { + let token = sqlx::query_scalar!( + r#" + SELECT token + FROM token + WHERE token LIKE concat($1::text, '%') + LIMIT 1 + "#, + token_prefix + ) + .fetch_optional(db) + .await?; + + Ok(token) +} + +/// Delete a token from the token table using its prefix +pub async fn delete_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + token_prefix: &str, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM token + WHERE token LIKE concat($1::text, '%') + "#, + token_prefix + ) + .execute(db) + .await? + .rows_affected(); + + Ok(deleted > 0) +} + +pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>, C: Serialize>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + config: &NativeTriggerConfig, + service_config: C, +) -> Result<()> { + // Store only the first 10 characters of the webhook token as a prefix + let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); + + sqlx::query!( + r#" + INSERT INTO native_trigger ( + external_id, + workspace_id, + service_name, + script_path, + is_flow, + webhook_token_prefix, + service_config + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7 + ) + ON CONFLICT (external_id, workspace_id, service_name) + DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_prefix = $6, service_config = $7, error = NULL, updated_at = NOW() + "#, + external_id, + workspace_id, + service_name as ServiceName, + config.script_path, + config.is_flow, + webhook_token_prefix, + sqlx::types::Json(service_config) as _, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + config: &NativeTriggerConfig, + service_config: Option<&RawValue>, +) -> Result<()> { + // Store only the first 10 characters of the webhook token as a prefix + let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); + + sqlx::query!( + r#" + UPDATE native_trigger + SET script_path = $1, is_flow = $2, webhook_token_prefix = $3, service_config = $4, error = NULL, updated_at = NOW() + WHERE + workspace_id = $5 + AND service_name = $6 + AND external_id = $7 + "#, + config.script_path, + config.is_flow, + webhook_token_prefix, + service_config.map(sqlx::types::Json) as _, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn delete_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await? + .rows_affected(); + + Ok(deleted > 0) +} +pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result> { + let trigger = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + external_id, + workspace_id, + service_name AS "service_name!: ServiceName", + script_path, + is_flow, + webhook_token_prefix, + service_config, + error, + created_at, + updated_at + FROM + native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id + ) + .fetch_optional(db) + .await?; + + Ok(trigger) +} + +pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + script_path: &str, + is_flow: bool, +) -> Result> { + let trigger = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + external_id, + workspace_id, + service_name AS "service_name!: ServiceName", + script_path, + is_flow, + webhook_token_prefix, + service_config, + error, + created_at, + updated_at + FROM + native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND script_path = $3 + AND is_flow = $4 + LIMIT 1 + "#, + workspace_id, + service_name as ServiceName, + script_path, + is_flow + ) + .fetch_optional(db) + .await?; + + Ok(trigger) +} + +pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + page: Option, + per_page: Option, + path: Option<&str>, + is_flow: Option, +) -> Result> { + let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; + let limit = per_page.unwrap_or(100) as i64; + + let triggers = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + nt.external_id, + nt.workspace_id, + nt.service_name AS "service_name!: ServiceName", + nt.script_path, + nt.is_flow, + nt.webhook_token_prefix, + nt.service_config, + nt.error, + nt.created_at, + nt.updated_at + FROM + native_trigger nt + WHERE + nt.workspace_id = $1 AND + nt.service_name = $2 AND + ($5::text IS NULL OR nt.script_path = $5) AND + ($6::bool IS NULL OR nt.is_flow = $6) AND + ( + (nt.is_flow = false AND EXISTS ( + SELECT 1 FROM script s + WHERE s.workspace_id = nt.workspace_id + AND s.path = nt.script_path + )) + OR + (nt.is_flow = true AND EXISTS ( + SELECT 1 FROM flow f + WHERE f.workspace_id = nt.workspace_id + AND f.path = nt.script_path + )) + ) + LIMIT $3 + OFFSET $4 + "#, + workspace_id, + service_name as ServiceName, + limit, + offset, + path, + is_flow + ) + .fetch_all(db) + .await?; + + Ok(triggers) +} + +pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + error: Option<&str>, +) -> Result<()> { + sqlx::query!( + r#" + UPDATE native_trigger + SET error = $1 + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + error, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn update_native_trigger_service_config< + 'c, + E: sqlx::Executor<'c, Database = Postgres>, +>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + service_config: &serde_json::Value, +) -> Result<()> { + sqlx::query!( + r#" + UPDATE native_trigger + SET service_config = $1, updated_at = NOW() + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + service_config, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn store_workspace_integration( + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + service_name: ServiceName, + oauth_data: serde_json::Value, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO workspace_integrations ( + workspace_id, + service_name, + oauth_data, + created_by, + created_at, + updated_at + ) VALUES ( + $1, $2, $3, $4, now(), now() + ) + ON CONFLICT (workspace_id, service_name) + DO UPDATE SET + oauth_data = $3, + updated_at = now() + "#, + workspace_id, + service_name as ServiceName, + oauth_data, + authed.username, + ) + .execute(&mut *tx) + .await?; + + Ok(()) +} + +pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let integration = sqlx::query_as!( + WorkspaceIntegration, + r#" + SELECT + workspace_id, + service_name AS "service_name!: ServiceName", + oauth_data, + created_at, + updated_at, + created_by + FROM + workspace_integrations + WHERE + workspace_id = $1 + AND service_name = $2 + "#, + workspace_id, + service_name as ServiceName, + ) + .fetch_one(db) + .await?; + + Ok(integration) +} + +pub async fn delete_workspace_integration( + tx: &mut PgConnection, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM workspace_integrations + WHERE + workspace_id = $1 + AND service_name = $2 + "#, + workspace_id, + service_name as ServiceName, + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) +} + +/// Generates the webhook URL that external services will call. +/// +/// `external_id` is optional because during CREATE we don't have it yet +/// (it's returned by the external service). During UPDATE, we have it. +pub fn generate_webhook_service_url( + base_url: &str, + w_id: &str, + script_path: &str, + is_flow: bool, + external_id: Option<&str>, + service_name: ServiceName, + webhook_token: &str, +) -> String { + let runnable_prefix = if is_flow { "f" } else { "p" }; + + let mut url = format!( + "{}/api/w/{}/jobs/run/{}/{}?token={}&service_name={}", + base_url, + w_id, + runnable_prefix, + script_path, + &webhook_token, + service_name.as_str(), + ); + + if let Some(id) = external_id { + url.push_str(&format!("&trigger_external_id={}", id)); + } + + url +} diff --git a/backend/windmill-api/src/native_triggers/nextcloud/external.rs b/backend/windmill-native-triggers/src/nextcloud/external.rs similarity index 99% rename from backend/windmill-api/src/native_triggers/nextcloud/external.rs rename to backend/windmill-native-triggers/src/nextcloud/external.rs index 028ba131c3..54b7578229 100644 --- a/backend/windmill-api/src/native_triggers/nextcloud/external.rs +++ b/backend/windmill-native-triggers/src/nextcloud/external.rs @@ -8,7 +8,7 @@ use windmill_common::{ BASE_URL, DB, }; -use crate::native_triggers::{ +use crate::{ generate_webhook_service_url, nextcloud::{ routes, NextCloud, NextCloudOAuthData, NextCloudTriggerData, NextcloudServiceConfig, diff --git a/backend/windmill-api/src/native_triggers/nextcloud/mod.rs b/backend/windmill-native-triggers/src/nextcloud/mod.rs similarity index 100% rename from backend/windmill-api/src/native_triggers/nextcloud/mod.rs rename to backend/windmill-native-triggers/src/nextcloud/mod.rs diff --git a/backend/windmill-api/src/native_triggers/nextcloud/routes.rs b/backend/windmill-native-triggers/src/nextcloud/routes.rs similarity index 90% rename from backend/windmill-api/src/native_triggers/nextcloud/routes.rs rename to backend/windmill-native-triggers/src/nextcloud/routes.rs index 40633dc248..29534b9642 100644 --- a/backend/windmill-api/src/native_triggers/nextcloud/routes.rs +++ b/backend/windmill-native-triggers/src/nextcloud/routes.rs @@ -9,13 +9,11 @@ use windmill_common::{ }; use crate::{ - db::ApiAuthed, - native_triggers::{ - get_workspace_integration, - nextcloud::{NextCloudEventType, OcsResponse}, - External, OAuthConfig, ServiceName, - }, + get_workspace_integration, + nextcloud::{NextCloudEventType, OcsResponse}, + External, OAuthConfig, ServiceName, }; +use windmill_api_auth::ApiAuthed; async fn list_available_events( authed: ApiAuthed, diff --git a/backend/windmill-api/src/native_triggers/sync.rs b/backend/windmill-native-triggers/src/sync.rs similarity index 98% rename from backend/windmill-api/src/native_triggers/sync.rs rename to backend/windmill-native-triggers/src/sync.rs index ca955630aa..6e7b70c098 100644 --- a/backend/windmill-api/src/native_triggers/sync.rs +++ b/backend/windmill-native-triggers/src/sync.rs @@ -4,9 +4,12 @@ use windmill_common::DB; use serde::Serialize; -use crate::native_triggers::{ +use crate::ServiceName; + +#[cfg(feature = "native_trigger")] +use crate::{ decrypt_oauth_data, list_native_triggers, update_native_trigger_error, - update_native_trigger_service_config, External, ServiceName, + update_native_trigger_service_config, External, }; #[derive(Debug, Serialize)] @@ -44,6 +47,7 @@ pub struct ServiceSyncResult { pub errors: Vec, } +#[allow(unused_variables, unused_mut)] pub async fn sync_all_triggers(db: &DB) -> Result { tracing::info!("Starting native triggers sync"); @@ -56,7 +60,7 @@ pub async fn sync_all_triggers(db: &DB) -> Result { // Each service only syncs workspaces that have the corresponding integration configured #[cfg(feature = "native_trigger")] { - use crate::native_triggers::nextcloud::NextCloud; + use crate::nextcloud::NextCloud; let (service_name, result) = sync_service_triggers(db, NextCloud).await; total_synced += result.synced_triggers.len(); @@ -64,7 +68,7 @@ pub async fn sync_all_triggers(db: &DB) -> Result { service_results.insert(service_name, result); // Add new services here: - // use crate::native_triggers::newservice::NewService; + // use crate::newservice::NewService; // let (service_name, result) = sync_service_triggers(db, NewService).await; // total_synced += result.synced_triggers.len(); // total_errors += result.errors.len(); @@ -93,6 +97,7 @@ pub async fn sync_all_triggers(db: &DB) -> Result { Ok(result) } +#[cfg(feature = "native_trigger")] async fn sync_service_triggers( db: &DB, handler: T, diff --git a/backend/windmill-api/src/native_triggers/workspace_integrations.rs b/backend/windmill-native-triggers/src/workspace_integrations.rs similarity index 87% rename from backend/windmill-api/src/native_triggers/workspace_integrations.rs rename to backend/windmill-native-triggers/src/workspace_integrations.rs index dd823b1344..3891905411 100644 --- a/backend/windmill-api/src/native_triggers/workspace_integrations.rs +++ b/backend/windmill-native-triggers/src/workspace_integrations.rs @@ -1,17 +1,25 @@ +use axum::Router; + +#[cfg(feature = "native_trigger")] use axum::{ extract::Path, routing::{delete, get, post}, - Extension, Json, Router, + Extension, Json, }; #[cfg(feature = "native_trigger")] use serde_json::to_value; -use sqlx::prelude::FromRow; -use strum::IntoEnumIterator; +#[cfg(feature = "native_trigger")] use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +#[cfg(feature = "native_trigger")] +use sqlx::prelude::FromRow; + #[cfg(feature = "native_trigger")] use windmill_audit::{audit_oss::audit_log, ActionKind}; + +#[cfg(feature = "native_trigger")] use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, @@ -21,20 +29,28 @@ use windmill_common::{ }; #[cfg(feature = "native_trigger")] -use crate::{ - db::ApiAuthed, - native_triggers::{delete_workspace_integration, store_workspace_integration, ServiceName}, -}; +use windmill_api_auth::ApiAuthed; + +#[cfg(feature = "native_trigger")] +use crate::ServiceName; + +#[cfg(feature = "native_trigger")] +use crate::{delete_workspace_integration, store_workspace_integration}; #[cfg(feature = "native_trigger")] use windmill_oauth::{OClient, Url, OAUTH_HTTP_CLIENT}; +#[cfg(feature = "native_trigger")] use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +#[cfg(feature = "native_trigger")] use hmac::{Hmac, Mac}; +#[cfg(feature = "native_trigger")] use sha2::Sha256; +#[cfg(feature = "native_trigger")] type HmacSha256 = Hmac; +#[cfg(feature = "native_trigger")] const STATE_EXPIRATION_SECONDS: i64 = 600; // 10 minutes /// Generate a signed OAuth state that is cluster-safe. @@ -60,8 +76,8 @@ async fn generate_signed_state( // Get workspace key for signing let key = get_workspace_key(workspace_id, db).await?; - let mut mac = - HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; + let mut mac = HmacSha256::new_from_slice(key.as_bytes()) + .map_err(|e| Error::InternalErr(e.to_string()))?; mac.update(payload.as_bytes()); let signature = mac.finalize().into_bytes(); @@ -121,8 +137,8 @@ async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Resu // Verify signature let key = get_workspace_key(workspace_id, db).await?; - let mut mac = - HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; + let mut mac = HmacSha256::new_from_slice(key.as_bytes()) + .map_err(|e| Error::InternalErr(e.to_string()))?; mac.update(payload.as_bytes()); let received_signature = match URL_SAFE_NO_PAD.decode(encoded_signature) { @@ -133,6 +149,7 @@ async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Resu Ok(mac.verify_slice(&received_signature).is_ok()) } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct IntegrationStatusResponse { pub connected: bool, @@ -141,24 +158,28 @@ pub struct IntegrationStatusResponse { pub created_by: Option, } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct ListIntegrationsResponse { pub integrations: Vec, } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct ConnectIntegrationResponse { pub auth_url: String, } +#[cfg(feature = "native_trigger")] #[derive(FromRow, Debug, Clone, Serialize, Deserialize)] pub struct WorkspaceOAuthConfig { pub client_id: String, pub client_secret: String, pub base_url: String, - pub access_token: Option + pub access_token: Option, } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct OAuthConfigResponse { pub configured: bool, @@ -222,6 +243,7 @@ async fn delete_integration( ))) } +#[cfg(feature = "native_trigger")] #[derive(FromRow, Debug, Deserialize, Serialize)] struct WorkspaceIntegrations { service_name: ServiceName, @@ -240,12 +262,12 @@ async fn list_integrations( let integrations = sqlx::query_as!( WorkspaceIntegrations, r#" - SELECT + SELECT oauth_data as "oauth_data!: sqlx::types::Json", service_name as "service_name!: ServiceName" - FROM - workspace_integrations - WHERE + FROM + workspace_integrations + WHERE workspace_id = $1 "#, workspace_id @@ -258,6 +280,7 @@ async fn list_integrations( .map(|integration| (integration.service_name, integration.oauth_data)) .collect::>(); + use strum::IntoEnumIterator; let integrations = ServiceName::iter() .map(|service_name| WorkspaceIntegrations { service_name: service_name, @@ -270,6 +293,7 @@ async fn list_integrations( Ok(Json(integrations)) } +#[cfg(feature = "native_trigger")] async fn integration_exist( authed: ApiAuthed, Extension(user_db): Extension, @@ -282,7 +306,7 @@ async fn integration_exist( SELECT 1 FROM workspace_integrations WHERE workspace_id = $1 - AND service_name = $2 + AND service_name = $2 AND oauth_data IS NOT NULL ) "#, @@ -296,6 +320,7 @@ async fn integration_exist( Ok(Json(exists)) } +#[cfg(feature = "native_trigger")] #[derive(Debug, Deserialize)] struct RedirectUri { redirect_uri: String, @@ -366,6 +391,7 @@ async fn oauth_callback( } /// Token response from OAuth token exchange +#[cfg(feature = "native_trigger")] #[derive(Debug, Deserialize)] struct TokenResponse { access_token: String, @@ -381,10 +407,18 @@ fn build_native_oauth_client( service_name: ServiceName, redirect_uri: &str, ) -> Result { - let auth_url = Url::parse(&format!("{}{}", config.base_url, service_name.auth_endpoint())) - .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; - let token_url = Url::parse(&format!("{}{}", config.base_url, service_name.token_endpoint())) - .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; + let auth_url = Url::parse(&format!( + "{}{}", + config.base_url, + service_name.auth_endpoint() + )) + .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; + let token_url = Url::parse(&format!( + "{}{}", + config.base_url, + service_name.token_endpoint() + )) + .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; let redirect = Url::parse(redirect_uri).map_err(|e| { Error::BadRequest(format!( "Invalid redirect URI '{}': {}. The redirect URI must be an absolute URL (e.g., https://example.com/callback)", @@ -419,6 +453,7 @@ async fn exchange_code_for_token( Ok(token_response) } +#[cfg(feature = "native_trigger")] async fn get_workspace_oauth_config( db: &DB, workspace_id: &str, @@ -426,11 +461,11 @@ async fn get_workspace_oauth_config( ) -> Result { let oauth_configs = sqlx::query_scalar!( r#" - SELECT - oauth_data - FROM - workspace_integrations - WHERE + SELECT + oauth_data + FROM + workspace_integrations + WHERE workspace_id = $1 AND service_name = $2 "#, @@ -461,7 +496,7 @@ pub async fn create_workspace_integration( let mut tx = user_db.begin(&authed).await?; - store_workspace_integration( + crate::store_workspace_integration( &mut tx, &authed, &workspace_id, @@ -475,6 +510,7 @@ pub async fn create_workspace_integration( Ok(()) } +#[cfg(feature = "native_trigger")] #[inline] async fn get_workspace_oauth_config_as_oauth_config( db: &DB, @@ -484,6 +520,7 @@ async fn get_workspace_oauth_config_as_oauth_config( get_workspace_oauth_config::(db, workspace_id, service_name).await } +#[cfg(feature = "native_trigger")] fn build_authorization_url( config: &WorkspaceOAuthConfig, state: &str, @@ -506,6 +543,7 @@ fn build_authorization_url( format!("{}/apps/oauth2/authorize?{}", config.base_url, query_string) } +#[cfg(feature = "native_trigger")] pub fn workspaced_service() -> Router { let router = Router::new() .route("/list", get(list_integrations)) @@ -520,3 +558,8 @@ pub fn workspaced_service() -> Router { Router::new().nest("/integrations", router) } + +#[cfg(not(feature = "native_trigger"))] +pub fn workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index c89cb190b7..5d530b05b5 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -36,17 +36,16 @@ pub type DB = sqlx::Pool; // Re-export oauth2 types that consumers need (also used internally) pub use oauth2::{ - AccessToken, AuthType, Client as OClient, RefreshToken, Scope, State, Url, - helpers, + helpers, AccessToken, AuthType, Client as OClient, RefreshToken, Scope, State, Url, }; // Re-export reqwest Client (version 0.12 compatible with async-oauth2) pub use reqwest::Client as HttpClient; +pub use windmill_common::utils::{COOKIE_DOMAIN, IS_SECURE}; + lazy_static::lazy_static! { pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); - pub static ref IS_SECURE: Arc> = Arc::new(RwLock::new(false)); - pub static ref COOKIE_DOMAIN: Option = std::env::var("COOKIE_DOMAIN").ok(); /// HTTP client for OAuth operations (reqwest 0.12, compatible with async-oauth2) pub static ref OAUTH_HTTP_CLIENT: reqwest::Client = reqwest::ClientBuilder::new() @@ -55,6 +54,12 @@ lazy_static::lazy_static! { .timeout(std::time::Duration::from_secs(30)) .build() .expect("Failed to create OAuth HTTP client"); + + pub static ref OAUTH_CLIENTS: Arc> = Arc::new(RwLock::new(AllClients { + logins: HashMap::new(), + connects: HashMap::new(), + slack: None + })); } /// OAuth client with associated scopes and configuration @@ -221,10 +226,7 @@ pub async fn build_oauth_clients( .collect() }; - tracing::info!( - "OAuth loaded clients: {}", - oauths.keys().join(", ") - ); + tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", ")); let logins = login_configs .into_iter() @@ -446,9 +448,7 @@ pub async fn build_client_credentials_oauth_client( } else { let static_configs = serde_json::from_str::>(connect_configs_json).map_err( - |e| { - error::Error::InternalErr(format!("Failed to parse oauth_connect.json: {}", e)) - }, + |e| error::Error::InternalErr(format!("Failed to parse oauth_connect.json: {}", e)), )?; static_configs.get(client_name).cloned().ok_or_else(|| { @@ -532,14 +532,12 @@ pub async fn exchange_token( http_client: &reqwest::Client, ) -> Result { let token_json = match grant_type { - "authorization_code" => { - client - .exchange_refresh_token(&RefreshToken::from(refresh_token)) - .with_client(http_client) - .execute::() - .await - .map_err(to_anyhow)? - } + "authorization_code" => client + .exchange_refresh_token(&RefreshToken::from(refresh_token)) + .with_client(http_client) + .execute::() + .await + .map_err(to_anyhow)?, "client_credentials" => { let mut token_request = client.exchange_client_credentials(); @@ -557,14 +555,12 @@ pub async fn exchange_token( .await .map_err(to_anyhow)? } - "" | _ if grant_type.is_empty() => { - client - .exchange_refresh_token(&RefreshToken::from(refresh_token)) - .with_client(http_client) - .execute::() - .await - .map_err(to_anyhow)? - } + "" | _ if grant_type.is_empty() => client + .exchange_refresh_token(&RefreshToken::from(refresh_token)) + .with_client(http_client) + .execute::() + .await + .map_err(to_anyhow)?, _ => { return Err(Error::BadRequest(format!( "Unsupported grant type: {}", @@ -829,11 +825,9 @@ pub async fn http_get_user_info( res.text().await.unwrap_or_default(), ))); } - Ok(res - .json::() - .await - .map_err(to_anyhow) - .map_err(|e| error::Error::InternalErr(format!("failed to decode json from user info: {}", e)))?) + Ok(res.json::().await.map_err(to_anyhow).map_err(|e| { + error::Error::InternalErr(format!("failed to decode json from user info: {}", e)) + })?) } /// GitHub email info response diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 16178502fe..ad485d33d8 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -14,6 +14,7 @@ private = [] enterprise = ["windmill-common/enterprise"] cloud = [] benchmark = ["windmill-common/benchmark"] +failpoints = [] prometheus = ["dep:prometheus"] smtp = [] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 912c279254..b298fed82c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -23,7 +23,7 @@ use reqwest::Client; use serde::Deserialize; use serde::{ser::SerializeMap, Serialize}; use serde_json::{json, value::RawValue}; -use sqlx::{types::Json, Pool, Postgres, Transaction}; +use sqlx::{types::Json, Acquire, Pool, Postgres, Transaction}; use sqlx::{Encode, PgExecutor}; use tokio::sync::mpsc::Sender; use tokio::sync::oneshot; @@ -41,22 +41,22 @@ use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; +use windmill_common::min_version::{ + MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2, +}; use windmill_common::runnable_settings::{ ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettings, RunnableSettingsTrait, }; use windmill_common::triggers::TriggerMetadata; use windmill_common::utils::{calculate_hash, configure_client, now_from_db}; -use windmill_common::min_version::{ - MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2, -}; use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY}; use windmill_common::{ - auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username}, + auth::permissioned_as_to_username, cache::{self, FlowData}, db::{Authed, UserDB}, - error::{self, to_anyhow, Error}, + error::{self, Error}, flow_status::{ BranchAllStatus, FlowCleanupModule, FlowStatus, FlowStatusModule, FlowStatusModuleWParent, Iterator as FlowIterator, JobResult, RestartedFrom, RetryStatus, MAX_RETRY_ATTEMPTS, @@ -67,11 +67,11 @@ use windmill_common::{ StopAfterIf, }, jobs::{get_payload_tag_from_prefixed_path, JobKind, JobPayload, QueuedJob, RawCode}, + min_version::{MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440}, schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, - min_version::{MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440}, worker::{ to_raw_value, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY, @@ -114,6 +114,26 @@ lazy_static::lazy_static! { } +#[cfg(feature = "failpoints")] +pub mod schedule_failpoints { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum ScheduleFailPoint { + SavepointCreate, + Push, + PushQuotaExceeded, + SavepointCommit, + ScheduleDisable, + } + + tokio::task_local! { + pub static ACTIVE: ScheduleFailPoint; + } + + pub fn is_active(point: ScheduleFailPoint) -> bool { + ACTIVE.try_with(|fp| *fp == point).unwrap_or(false) + } +} + lazy_static::lazy_static! { pub static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .user_agent("windmill/beta") @@ -762,7 +782,6 @@ pub async fn add_completed_job_error( flow_is_done, duration, false, - false, ) .warn_after_seconds(10) .await?; @@ -798,7 +817,6 @@ pub async fn add_completed_job( canceled_by: Option, flow_is_done: bool, duration: Option, - has_stream: bool, from_cache: bool, ) -> Result<(Uuid, i64), Error> { // tracing::error!("Start"); @@ -824,7 +842,6 @@ pub async fn add_completed_job( &canceled_by, flow_is_done, duration, - has_stream, from_cache, ) .warn_after_seconds(10) @@ -832,13 +849,14 @@ pub async fn add_completed_job( .retry( ConstantBuilder::default() .with_delay(std::time::Duration::from_secs(3)) - .with_max_times(5) + .with_max_times(10) .build(), ) .when(|err| { !matches!(err, Error::QuotaExceeded(_)) && !matches!(err, Error::ResultTooLarge(_)) && !matches!(err, Error::AlreadyCompleted(_)) + && !matches!(err, Error::NotFound(_)) }) .notify(|err, dur| { tracing::error!("Could not insert completed job, retrying in {dur:#?}, err: {err:#?}"); @@ -883,7 +901,6 @@ async fn commit_completed_job( canceled_by: &Option, flow_is_done: bool, duration: Option, - has_stream: bool, from_cache: bool, ) -> windmill_common::error::Result<(Option, i64, bool)> { // let start = std::time::Instant::now(); @@ -1100,22 +1117,12 @@ async fn commit_completed_job( .unwrap_or(false); if schedule_next_tick { - if let Err(err) = Box::pin(handle_maybe_scheduled_job( - db, - completed_job, - &schedule, - &script_path, - &completed_job.workspace_id, - )) - .warn_after_seconds(10) - .await - { - match err { - Error::QuotaExceeded(_) => (), - // scheduling next job failed and could not disable schedule => make zombie job to retry - _ => return Ok((Some(job_id), 0, true)), - } - }; + let (returned_tx, schedule_push_err) = + try_schedule_next_job(db, tx, completed_job, &schedule, &script_path).await; + tx = returned_tx; + if let Some(err) = schedule_push_err { + return Err(err); + } } #[cfg(all(feature = "enterprise", feature = "private"))] @@ -1164,13 +1171,28 @@ async fn commit_completed_job( } if completed_job.concurrent_limit.is_some() - || RunnableSettings::prefetch_cached_from_handle(completed_job.runnable_settings_handle, db) - .await? - .1 - .concurrent_limit - .is_some() + || windmill_common::runnable_settings::prefetch_cached_from_handle( + completed_job.runnable_settings_handle, + db, + ) + .await? + .1 + .concurrent_limit + .is_some() { - let concurrency_key = concurrency_key(db, &completed_job.id).await?; + let concurrency_key = sqlx::query_scalar!( + "SELECT key FROM concurrency_key WHERE job_id = $1", + &completed_job.id + ) + .fetch_optional(&mut *tx) + .warn_after_seconds(10) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not get concurrency key for job {}: {e:#}", + completed_job.id + )) + })?; if *DISABLE_CONCURRENCY_LIMIT || concurrency_key.is_none() { tracing::warn!("Concurrency limit is disabled, skipping"); } else { @@ -1207,18 +1229,6 @@ async fn commit_completed_job( tracing::debug!("decremented concurrency counter"); } - sqlx::query!("DELETE FROM job_perms WHERE job_id = $1", job_id) - .execute(&mut *tx) - .warn_after_seconds(10) - .await?; - - if !success || has_stream { - sqlx::query!("DELETE FROM job_result_stream_v2 WHERE job_id = $1", job_id) - .execute(&mut *tx) - .warn_after_seconds(10) - .await?; - } - tx.commit().warn_after_seconds(10).await?; tracing::info!( @@ -1232,7 +1242,7 @@ async fn commit_completed_job( email = ?completed_job.permissioned_as_email, created_by = completed_job.created_by, is_flow_step = completed_job.is_flow_step(), - language = ?completed_job.script_lang, + language = completed_job.script_lang.map(|x| x.as_str()).unwrap_or_default(), scheduled_for = ?completed_job.scheduled_for, workspace_id = ?completed_job.workspace_id, success, @@ -1579,7 +1589,15 @@ async fn fetch_error_handler_from_db( db: &Pool, w_id: &str, ) -> Result<(Option, Option>>, bool, bool), Error> { - sqlx::query_as::<_, (Option, Option>>, Option, Option)>( + sqlx::query_as::< + _, + ( + Option, + Option>>, + Option, + Option, + ), + >( r#" SELECT error_handler->>'path', @@ -1595,7 +1613,12 @@ async fn fetch_error_handler_from_db( .await .context("fetching error handler info from workspace_settings")? .map(|(path, extra_args, muted_on_cancel, muted_on_user_path)| { - (path, extra_args, muted_on_cancel.unwrap_or(false), muted_on_user_path.unwrap_or(false)) + ( + path, + extra_args, + muted_on_cancel.unwrap_or(false), + muted_on_user_path.unwrap_or(false), + ) }) .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}"))) } @@ -1609,19 +1632,14 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> let w_id = &queued_job.workspace_id; let now = chrono::Utc::now().timestamp(); - let (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path) = - if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { - if cached.4 > now { - (cached.0.clone(), cached.1.clone(), cached.2, cached.3) - } else { - let row = fetch_error_handler_from_db(db, w_id).await?; - let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; - WORKSPACE_ERROR_HANDLER_CACHE.insert( - w_id.clone(), - (row.0.clone(), row.1.clone(), row.2, row.3, expiry), - ); - row - } + let ( + error_handler, + error_handler_extra_args, + error_handler_muted_on_cancel, + error_handler_muted_on_user_path, + ) = if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { + if cached.4 > now { + (cached.0.clone(), cached.1.clone(), cached.2, cached.3) } else { let row = fetch_error_handler_from_db(db, w_id).await?; let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; @@ -1630,7 +1648,16 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> (row.0.clone(), row.1.clone(), row.2, row.3, expiry), ); row - }; + } + } else { + let row = fetch_error_handler_from_db(db, w_id).await?; + let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; + WORKSPACE_ERROR_HANDLER_CACHE.insert( + w_id.clone(), + (row.0.clone(), row.1.clone(), row.2, row.3, expiry), + ); + row + }; if is_canceled && error_handler_muted_on_cancel { return Ok(()); @@ -1730,19 +1757,15 @@ pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Syn } else { let row = fetch_success_handler_from_db(db, w_id).await?; let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; - WORKSPACE_SUCCESS_HANDLER_CACHE.insert( - w_id.clone(), - (row.0.clone(), row.1.clone(), expiry), - ); + WORKSPACE_SUCCESS_HANDLER_CACHE + .insert(w_id.clone(), (row.0.clone(), row.1.clone(), expiry)); row } } else { let row = fetch_success_handler_from_db(db, w_id).await?; let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; - WORKSPACE_SUCCESS_HANDLER_CACHE.insert( - w_id.clone(), - (row.0.clone(), row.1.clone(), expiry), - ); + WORKSPACE_SUCCESS_HANDLER_CACHE + .insert(w_id.clone(), (row.0.clone(), row.1.clone(), expiry)); row }; @@ -1768,91 +1791,184 @@ pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Syn Ok(()) } -pub async fn handle_maybe_scheduled_job<'c>( +pub async fn try_schedule_next_job<'c>( db: &Pool, + mut tx: Transaction<'c, Postgres>, job: &MiniCompletedJob, schedule: &Schedule, script_path: &str, - w_id: &str, -) -> windmill_common::error::Result<()> { +) -> (Transaction<'c, Postgres>, Option) { + if !schedule.enabled { + tracing::info!( + "Schedule {} in {} is disabled. Not scheduling again.", + schedule.path, + &job.workspace_id + ); + return (tx, None); + } + + if script_path != schedule.script_path { + tracing::warn!( + "Schedule {} in {} has a different script path than the job. Not scheduling again", + schedule.path, + &job.workspace_id + ); + return (tx, None); + } + tracing::info!( - "Schedule {} scheduling next job for {} in {w_id}", + "Schedule {} scheduling next job for {} in {}", schedule.path, - schedule.script_path + schedule.script_path, + &job.workspace_id ); - if schedule.enabled && script_path == schedule.script_path { - let push_next_job_future = (|| { - tokio::time::timeout(std::time::Duration::from_secs(5), async { - let mut tx = db.begin().await?; - tx = push_scheduled_job(db, tx, &schedule, None, Some(job.scheduled_for)).await?; - tx.commit().await?; - Ok::<(), Error>(()) - }) - .map_err(|e| Error::internal_err(format!("Pushing next scheduled job timedout: {e:#}"))) - .unwrap_or_else(|e| Err(e)) - }) - .retry( - ConstantBuilder::default() - .with_delay(std::time::Duration::from_secs(5)) - .with_max_times(10) - .build(), - ) - .when(|err| !matches!(err, Error::QuotaExceeded(_))) - .notify(|err, dur| { - tracing::error!( - "Could not push next scheduled job for schedule {}, retrying in {dur:#?}, err: {err:#?}", schedule.path - ); - }) - .sleep(tokio::time::sleep); - match push_next_job_future.await { - Ok(()) => Ok(()), - Err(err) => { - let update_schedule = sqlx::query!( - "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", - err.to_string(), - &schedule.workspace_id, - &schedule.path + let schedule_authed = windmill_common::auth::fetch_authed_from_permissioned_as( + &windmill_common::users::username_to_permissioned_as(&schedule.edited_by), + &schedule.email, + &job.workspace_id, + &mut *tx, + ) + .await + .ok(); + + let mut push_err = None; + + #[cfg(feature = "failpoints")] + if schedule_failpoints::is_active(schedule_failpoints::ScheduleFailPoint::SavepointCreate) { + push_err = Some(Error::internal_err( + "failpoint: savepoint create".to_string(), + )); + } + + if push_err.is_none() { + let savepoint_result = tx.begin().await; + match savepoint_result { + Ok(savepoint) => { + let push_result = match tokio::time::timeout( + std::time::Duration::from_secs(5), + push_scheduled_job( + db, + savepoint, + schedule, + schedule_authed.as_ref(), + Some(job.scheduled_for), + ), ) - .execute(db) - .await; - match update_schedule { - Ok(_) => { - match err { - Error::QuotaExceeded(_) => {} - _ => { - report_error_to_workspace_handler_or_critical_side_channel(job, db, - format!("Could not schedule next job for {} with err {}. Schedule disabled", schedule.path, err.to_string()), - ).await; + .await + { + Ok(result) => result, + Err(_elapsed) => Err(Error::internal_err( + "push_scheduled_job timed out after 5s".to_string(), + )), + }; + #[cfg(feature = "failpoints")] + let push_result = + if schedule_failpoints::is_active(schedule_failpoints::ScheduleFailPoint::Push) + { + if let Ok(sp) = push_result { + sp.rollback().await.ok(); + } + Err(Error::internal_err("failpoint: push".to_string())) + } else if schedule_failpoints::is_active( + schedule_failpoints::ScheduleFailPoint::PushQuotaExceeded, + ) { + if let Ok(sp) = push_result { + sp.rollback().await.ok(); + } + Err(Error::QuotaExceeded( + "failpoint: push quota exceeded".to_string(), + )) + } else { + push_result + }; + match push_result { + Ok(savepoint) => { + #[cfg(feature = "failpoints")] + let savepoint_commit_fail = schedule_failpoints::is_active( + schedule_failpoints::ScheduleFailPoint::SavepointCommit, + ); + #[cfg(not(feature = "failpoints"))] + let savepoint_commit_fail = false; + + if savepoint_commit_fail { + savepoint.rollback().await.ok(); + push_err = Some(Error::internal_err( + "failpoint: savepoint commit".to_string(), + )); + } else { + match savepoint.commit().await { + Ok(()) => {} + Err(e) => { + push_err = Some(Error::internal_err(format!( + "Could not commit savepoint: {e:#}" + ))); + } } } - Ok(()) } - Err(disable_err) => match err { - Error::QuotaExceeded(_) => Err(err), - _ => { - report_error_to_workspace_handler_or_critical_side_channel(job, db, - format!("Could not schedule next job for {} and could not disable schedule with err {}.", schedule.path, disable_err), - ).await; - Err(to_anyhow(disable_err).into()) - } - }, + Err(err) if matches!(err, Error::QuotaExceeded(_)) => { + push_err = Some(err); + } + Err(err) => { + tracing::warn!( + "Could not push next scheduled job for {}: {err}", + schedule.path, + ); + push_err = Some(err); + } } } + Err(e) => { + tracing::error!("Could not create savepoint for schedule push: {e:#}",); + push_err = Some(Error::internal_err(format!( + "Could not create savepoint: {e:#}" + ))); + } } - } else { - if script_path != schedule.script_path { - tracing::warn!( - "Schedule {} in {w_id} has a different script path than the job. Not scheduling again", schedule.path - ); - } else { - tracing::info!( - "Schedule {} in {w_id} is disabled. Not scheduling again.", + } + + if let Some(ref err) = push_err { + if matches!(err, Error::QuotaExceeded(_) | Error::NotFound(_)) { + tracing::error!( + "Could not push next scheduled job for {}: {err}. Disabling schedule.", schedule.path ); + let disable_result = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", + err.to_string(), + &schedule.workspace_id, + &schedule.path + ) + .execute(&mut *tx) + .await; + #[cfg(feature = "failpoints")] + let disable_result = if schedule_failpoints::is_active( + schedule_failpoints::ScheduleFailPoint::ScheduleDisable, + ) { + Err(sqlx::Error::Protocol( + "failpoint: schedule disable".to_string(), + )) + } else { + disable_result + }; + if let Err(disable_err) = disable_result { + report_error_to_workspace_handler_or_critical_side_channel( + job, + db, + format!( + "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + schedule.path, + ), + ) + .await; + } else { + push_err = None; + } } - Ok(()) } + + (tx, push_err) } pub const ERROR_HANDLER_PATH_TEAMS: &str = "/workspace-or-schedule-error-handler-teams"; @@ -2073,7 +2189,10 @@ pub async fn push_success_handler<'a, 'c, T: Serialize + Send + Sync>( on_behalf_of.permissioned_as.clone(), ) } else { - (SUCCESS_HANDLER_USER_EMAIL, SUCCESS_HANDLER_USER_GROUP.to_string()) + ( + SUCCESS_HANDLER_USER_EMAIL, + SUCCESS_HANDLER_USER_GROUP.to_string(), + ) }; let tx = PushIsolationLevel::IsolatedRoot(db.clone()); @@ -2756,9 +2875,12 @@ impl PulledJobResult { }; let DebouncingSettings { debounce_delay_s, debounce_args_to_accumulate, .. } = - RunnableSettings::prefetch_cached_from_handle(j.runnable_settings_handle, db) - .await? - .0; + windmill_common::runnable_settings::prefetch_cached_from_handle( + j.runnable_settings_handle, + db, + ) + .await? + .0; let (kind, j_id) = (j.kind, j.id); let is_djob_to_debounce = kind.is_dependency() @@ -3047,12 +3169,17 @@ pub async fn pull( #[cfg(feature = "private")] let concurrency_settings = if let Some(ref j) = job { - RunnableSettings::from_runnable_settings_handle(j.runnable_settings_handle, db) - .await? - .prefetch_cached(db) - .await? - .1 - .maybe_fallback(None, j.concurrent_limit, j.concurrency_time_window_s) + windmill_common::runnable_settings::prefetch_cached_from_handle( + j.runnable_settings_handle, + db, + ) + .await? + .1 + .maybe_fallback( + None, + j.concurrent_limit, + j.concurrency_time_window_s, + ) } else { Default::default() }; @@ -3116,13 +3243,13 @@ pub async fn pull( }); }; - let concurrency_settings = - RunnableSettings::from_runnable_settings_handle(job.runnable_settings_handle, db) - .await? - .prefetch_cached(db) - .await? - .1 - .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); + let concurrency_settings = windmill_common::runnable_settings::prefetch_cached_from_handle( + job.runnable_settings_handle, + db, + ) + .await? + .1 + .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); let has_concurent_limit = concurrency_settings.concurrent_limit.is_some(); @@ -4156,8 +4283,74 @@ async fn check_usage_limits( #[cfg(feature = "cloud")] use crate::cloud_usage::increment_usage_async; -// #[instrument(level = "trace", skip_all)] +// Thin wrapper that boxes the future to reduce async state machine sizes in callers. +// Without this, the ~13KB future of push_inner is inlined into every caller's state machine, +// causing stack overflows in deeply nested async call chains (e.g. flow execution). pub async fn push<'c, 'd>( + _db: &Pool, + tx: PushIsolationLevel<'c>, + workspace_id: &str, + job_payload: JobPayload, + args: PushArgs<'d>, + user: &str, + email: &str, + permissioned_as: String, + token_prefix: Option<&str>, + scheduled_for_o: Option>, + schedule_path: Option, + parent_job: Option, + root_job: Option, + flow_innermost_root_job: Option, + job_id: Option, + _is_flow_step: bool, + same_worker: bool, + pre_run_error: Option<&windmill_common::error::Error>, + visible_to_owner: bool, + tag: Option, + custom_timeout: Option, + flow_step_id: Option, + _priority_override: Option, + authed: Option<&Authed>, + running: bool, + end_user_email: Option, + trigger: Option, + suspended_mode: Option, +) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { + Box::pin(push_inner( + _db, + tx, + workspace_id, + job_payload, + args, + user, + email, + permissioned_as, + token_prefix, + scheduled_for_o, + schedule_path, + parent_job, + root_job, + flow_innermost_root_job, + job_id, + _is_flow_step, + same_worker, + pre_run_error, + visible_to_owner, + tag, + custom_timeout, + flow_step_id, + _priority_override, + authed, + running, + end_user_email, + trigger, + suspended_mode, + )) + .await +} + +// #[instrument(level = "trace", skip_all)] +async fn push_inner<'c, 'd>( _db: &Pool, mut tx: PushIsolationLevel<'c>, workspace_id: &str, @@ -5189,7 +5382,7 @@ pub async fn push<'c, 'd>( &runnable_path, job_kind, concurrency_settings.concurrency_key.clone(), - &mut tx, + &mut *tx, job_id, ) .await?; @@ -5221,11 +5414,11 @@ pub async fn push<'c, 'd>( if authed.is_some() { tracing::warn!("Authed passed to push is not the same as permissioned_as, refetching direclty permissions for job {job_id}...") } - fetch_authed_from_permissioned_as( - permissioned_as.clone(), - email.to_string(), + windmill_common::auth::fetch_authed_from_permissioned_as( + &permissioned_as, + email, workspace_id, - _db, + &mut *tx, ) .await .map_err(|e| { @@ -5282,11 +5475,13 @@ pub async fn push<'c, 'd>( (job_kind, scheduled_for_o) }; - let runnable_settings_handle = RunnableSettings { - debouncing_settings: debouncing_settings.insert_cached(_db).await?, - concurrency_settings: concurrency_settings.insert_cached(_db).await?, - } - .insert_cached(_db) + let runnable_settings_handle = windmill_common::runnable_settings::insert_rs( + RunnableSettings { + debouncing_settings: debouncing_settings.insert_cached(_db).await?, + concurrency_settings: concurrency_settings.insert_cached(_db).await?, + }, + _db, + ) .await?; let (guarded_concurrent_limit, guarded_concurrency_time_window_s) = @@ -5510,7 +5705,7 @@ pub async fn insert_concurrency_key<'d, 'c>( script_path: &Option, job_kind: JobKind, custom_concurrency_key: Option, - tx: &mut Transaction<'c, Postgres>, + db: impl PgExecutor<'c>, job_id: Uuid, ) -> Result<(), Error> { let concurrency_key = custom_concurrency_key @@ -5538,7 +5733,7 @@ pub async fn insert_concurrency_key<'d, 'c>( )); sqlx::query!( "WITH inserted_concurrency_counter AS ( - INSERT INTO concurrency_counter (concurrency_id, job_uuids) + INSERT INTO concurrency_counter (concurrency_id, job_uuids) VALUES ($1, '{}'::jsonb) ON CONFLICT DO NOTHING ) @@ -5546,7 +5741,7 @@ pub async fn insert_concurrency_key<'d, 'c>( concurrency_key, job_id, ) - .execute(&mut **tx) + .execute(db) .warn_after_seconds(3) .await .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?; diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 6a9ea1be82..d24d51cea9 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -24,7 +24,6 @@ use windmill_common::jobs::JobPayload; use windmill_common::jobs::JobTriggerKind; use windmill_common::runnable_settings::ConcurrencySettings; use windmill_common::runnable_settings::DebouncingSettings; -use windmill_common::runnable_settings::RunnableSettings; use windmill_common::schedule::schedule_to_user; use windmill_common::scripts::ScriptHash; use windmill_common::triggers::TriggerMetadata; @@ -345,9 +344,7 @@ pub async fn push_scheduled_job<'c>( .await?; let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle(runnable_settings_handle, db) - .await? - .prefetch_cached(db) + windmill_common::runnable_settings::prefetch_cached_from_handle(runnable_settings_handle, db) .await?; if schedule.retry.is_some() { @@ -467,6 +464,22 @@ pub async fn push_scheduled_job<'c>( ) }; + let obo_authed; + let push_authed = match push_authed { + Some(a) => Some(a), + None => { + obo_authed = windmill_common::auth::fetch_authed_from_permissioned_as( + &permissioned_as, + email, + &schedule.workspace_id, + &mut *tx, + ) + .await + .ok(); + obo_authed.as_ref() + } + }; + if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { check_tag_available_for_workspace_internal( &db, @@ -565,3 +578,32 @@ pub async fn exists_schedule( Ok(exists) } + +pub async fn clear_schedule<'c>( + tx: &mut Transaction<'c, Postgres>, + path: &str, + w_id: &str, +) -> Result<()> { + tracing::info!("Clearing schedule {}", path); + sqlx::query!( + "WITH to_delete AS ( + SELECT id FROM v2_job_queue + JOIN v2_job j USING (id) + WHERE trigger_kind = 'schedule' + AND trigger = $1 + AND j.workspace_id = $2 + AND flow_step_id IS NULL + AND running = false + FOR UPDATE + ), deleted AS ( + DELETE FROM v2_job_queue + WHERE id IN (SELECT id FROM to_delete) + RETURNING id + ) DELETE FROM v2_job WHERE id IN (SELECT id FROM deleted)", + path, + w_id + ) + .execute(&mut **tx) + .await?; + Ok(()) +} diff --git a/backend/windmill-runtime-nativets/Cargo.toml b/backend/windmill-runtime-nativets/Cargo.toml new file mode 100644 index 0000000000..417ec8de9d --- /dev/null +++ b/backend/windmill-runtime-nativets/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "windmill-runtime-nativets" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_runtime_nativets" +path = "src/lib.rs" + +[features] +default = [] +private = [] +enterprise = ["windmill-common/enterprise"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-parser-ts.workspace = true +deno_fetch.workspace = true +deno_webidl.workspace = true +deno_web.workspace = true +deno_net.workspace = true +deno_console.workspace = true +deno_url.workspace = true +deno_core.workspace = true +deno_ast.workspace = true +deno_tls.workspace = true +deno_permissions.workspace = true +deno_io.workspace = true +deno_telemetry.workspace = true +deno_error.workspace = true +deno_runtime.workspace = true +winapi.workspace = true + +itertools.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true +tracing.workspace = true +anyhow.workspace = true +uuid.workspace = true +reqwest.workspace = true +regex.workspace = true +lazy_static.workspace = true +const_format.workspace = true +futures.workspace = true +sqlx.workspace = true +rustls.workspace = true + +[build-dependencies] +deno_fetch.workspace = true +deno_webidl.workspace = true +deno_web.workspace = true +deno_net.workspace = true +deno_console.workspace = true +deno_url.workspace = true +deno_core.workspace = true +deno_ast.workspace = true +deno_tls.workspace = true +deno_permissions.workspace = true +deno_io.workspace = true +deno_runtime.workspace = true +deno_telemetry.workspace = true +winapi.workspace = true diff --git a/backend/windmill-runtime-nativets/build.rs b/backend/windmill-runtime-nativets/build.rs new file mode 100644 index 0000000000..2f4ec8c38d --- /dev/null +++ b/backend/windmill-runtime-nativets/build.rs @@ -0,0 +1,124 @@ +use deno_fetch::FetchPermissions; +use deno_net::NetPermissions; +use deno_web::{BlobStore, TimersPermission}; +use std::borrow::Cow; +use std::env; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +pub struct PermissionsContainer; + +impl FetchPermissions for PermissionsContainer { + #[inline(always)] + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + unreachable!("snapshotting") + } + + #[inline(always)] + fn check_read<'a>( + &mut self, + _resolved: bool, + _p: &'a std::path::Path, + _api_name: &str, + ) -> Result, deno_io::fs::FsError> { + unreachable!("snapshotting") + } +} + +impl TimersPermission for PermissionsContainer { + #[inline(always)] + fn allow_hrtime(&mut self) -> bool { + true + } +} + +impl NetPermissions for PermissionsContainer { + fn check_read<'a>( + &mut self, + _p: &'a str, + _api_name: &str, + ) -> Result { + unreachable!("snapshotting") + } + + fn check_write<'a>( + &mut self, + _p: &'a str, + _api_name: &str, + ) -> Result { + unreachable!("snapshotting") + } + + fn check_net>( + &mut self, + _host: &(T, Option), + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + unreachable!("snapshotting") + } + + fn check_write_path<'a>( + &mut self, + _: &'a Path, + _: &str, + ) -> Result, deno_permissions::PermissionCheckError> { + todo!() + } +} + +deno_core::extension!( + fetch, + esm_entry_point = "ext:fetch/src/runtime.js", + esm = ["src/runtime.js"], +); + +fn main() { + println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); + println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); + + let exts = vec![ + deno_telemetry::deno_telemetry::init_ops_and_esm(), + deno_webidl::deno_webidl::init_ops_and_esm(), + deno_url::deno_url::init_ops_and_esm(), + deno_console::deno_console::init_ops_and_esm(), + deno_web::deno_web::init_ops_and_esm::( + Arc::new(BlobStore::default()), + None, + ), + deno_fetch::deno_fetch::init_ops_and_esm::(Default::default()), + deno_net::deno_net::init_ops_and_esm::(None, None), + fetch::init_ops_and_esm(), + ]; + + // Build the file path to the snapshot. + let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let snapshot_path = o.join("FETCH_SNAPSHOT.bin"); + + // Create the snapshot. + let output = deno_core::snapshot::create_snapshot( + deno_core::snapshot::CreateSnapshotOptions { + cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), + startup_snapshot: None, + extension_transpiler: Some(std::rc::Rc::new(|specifier, source| { + deno_runtime::transpile::maybe_transpile_source(specifier, source) + })), + extensions: exts, + with_runtime_cb: None, + skip_op_registration: false, + }, + None, + ) + .unwrap(); + + let mut file = std::fs::File::create(snapshot_path).unwrap(); + file.write_all(&output.output).unwrap(); + + for path in output.files_loaded_during_snapshot { + println!("cargo:rerun-if-changed={}", path.display()); + } +} diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs new file mode 100644 index 0000000000..4f1f88daa8 --- /dev/null +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -0,0 +1,757 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Isolated deno_core runtime for NativeTS script execution. +//! +//! This crate encapsulates all deno_core/V8 dependencies for executing +//! TypeScript scripts via the nativets runtime. By isolating this here, +//! deno_core compilation no longer blocks windmill-worker or windmill-api. + +use std::{ + borrow::Cow, + cell::RefCell, + path::PathBuf, + rc::Rc, + sync::{Arc, Mutex}, +}; + +// Re-export deno_telemetry for use by windmill-worker's otel proxy +pub use deno_telemetry; + +use deno_ast::ParseParams; +use deno_core::{ + op2, serde_v8, url, + v8::{self, IsolateHandle}, + Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, +}; +use deno_fetch::FetchPermissions; +use deno_net::NetPermissions; +use deno_web::{BlobStore, TimersPermission}; +use itertools::Itertools; +use lazy_static::lazy_static; +use regex::Regex; +use serde_json::value::RawValue; +use sqlx::types::Json; +use tokio::sync::mpsc; +use uuid::Uuid; + +use windmill_common::error::Error; +use windmill_common::result_stream::append_result_stream_db; +use windmill_common::worker::{write_file, Connection, TMP_DIR}; + +// ── Permission container ───────────────────────────────────────────── + +pub struct PermissionsContainer; + +impl FetchPermissions for PermissionsContainer { + #[inline(always)] + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + Ok(()) + } + + #[inline(always)] + fn check_read<'a>( + &mut self, + _resolved: bool, + p: &'a std::path::Path, + _api_name: &str, + ) -> Result, deno_io::fs::FsError> { + Ok(Cow::Borrowed(p)) + } +} + +impl TimersPermission for PermissionsContainer { + #[inline(always)] + fn allow_hrtime(&mut self) -> bool { + true + } +} + +impl NetPermissions for PermissionsContainer { + fn check_read<'a>( + &mut self, + p: &'a str, + _api_name: &str, + ) -> Result { + Ok(PathBuf::from(p)) + } + + fn check_write<'a>( + &mut self, + p: &'a str, + _api_name: &str, + ) -> Result { + Ok(PathBuf::from(p)) + } + + fn check_net>( + &mut self, + _host: &(T, Option), + _api_name: &str, + ) -> Result<(), deno_permissions::PermissionCheckError> { + Ok(()) + } + + fn check_write_path<'a>( + &mut self, + p: &'a std::path::Path, + _api_name: &str, + ) -> Result, deno_permissions::PermissionCheckError> { + Ok(Cow::Borrowed(p)) + } +} + +// ── Types ──────────────────────────────────────────────────────────── + +struct MainArgs { + args: Vec>>, +} + +struct LogString { + pub s: mpsc::UnboundedSender, +} + +pub struct NativeAnnotation { + pub useragent: Option, + pub proxy: Option<(String, Option<(String, String)>)>, +} + +/// Serializes V8 isolate creation as defense-in-depth against concurrent +/// creation races on x86_64 Linux. The primary fix is using the unprotected +/// V8 platform (see `setup_deno_runtime`). +static V8_ISOLATE_CREATE_LOCK: Mutex<()> = Mutex::new(()); + +/// Guard that terminates a running V8 isolate when dropped (e.g. on job cancellation). +struct IsolateDropGuard(Arc>>); + +impl Drop for IsolateDropGuard { + fn drop(&mut self) { + if let Some(handle) = self.0.lock().unwrap().take() { + handle.terminate_execution(); + } + } +} + +// ── Statics ────────────────────────────────────────────────────────── + +static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); + +const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); + +const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); + +lazy_static! { + static ref RE_PROXY: Regex = + Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); +} + +// ── Public interface ───────────────────────────────────────────────── + +/// Set up the deno_core/V8 runtime. Idempotent — safe to call multiple times. +/// Called automatically before JsRuntime creation, but can also be called +/// eagerly at startup for predictable initialization order. +pub fn setup_deno_runtime() -> anyhow::Result<()> { + use std::sync::Once; + static INIT: Once = Once::new(); + + let mut init_err: Option = None; + INIT.call_once(|| { + // deno_fetch requires a TLS provider; install ring as default (idempotent). + let _ = rustls::crypto::ring::default_provider().install_default(); + + let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ + "--stack-size=1024".to_string(), + "--no-harmony-import-assertions".to_string(), + ]) + .into_iter() + .skip(1) + .collect::>(); + + if !unrecognized_v8_flags.is_empty() { + init_err = Some(format!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags)); + } + + // Use an unprotected platform that doesn't enforce thread-isolated allocations + // via Memory Protection Keys (pkeys). The default platform requires all V8-using + // threads to be descendants of the thread that called v8::Initialize, but tokio's + // spawn_blocking pool threads don't satisfy this. Without this, V8 crashes with + // SIGSEGV in WasmCodePointerTable::AllocateUninitializedEntry() on x86_64 Linux. + // See: https://github.com/denoland/deno_core/issues/952 + let platform = deno_core::v8::new_unprotected_default_platform(0, false).make_shared(); + deno_core::JsRuntime::init_platform(Some(platform), false); + }); + + if let Some(msg) = init_err { + println!("{msg}"); + } + Ok(()) +} + +pub fn transpile_ts(expr: String) -> anyhow::Result { + let parsed = deno_ast::parse_module(ParseParams { + specifier: url::Url::parse("file:///eval.ts")?, + capture_tokens: false, + scope_analysis: false, + media_type: deno_ast::MediaType::TypeScript, + maybe_syntax: None, + text: deno_core::ModuleCodeString::from(expr).into(), + })?; + Ok(parsed + .transpile( + &Default::default(), + &Default::default(), + &Default::default(), + )? + .into_source() + .text) +} + +pub fn get_annotation(inner_content: &str) -> NativeAnnotation { + let mut res = NativeAnnotation { useragent: None, proxy: None }; + + let anns = inner_content + .lines() + .take_while(|x| x.starts_with("//")) + .map(|x| x.to_string().trim_start_matches("//").trim().to_string()) + .collect_vec(); + + for ann in anns.iter() { + if ann.starts_with("useragent") { + res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); + } else if ann.starts_with("proxy") { + res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); + } + } + res +} + +fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { + RE_PROXY.captures(s).map(|x| { + ( + format!( + "{}://{}{}", + x.get(1).map(|x| x.as_str()).unwrap_or_default(), + x.get(5).map(|x| x.as_str()).unwrap_or_default(), + x.get(7) + .map(|x| format!(":{}", x.as_str())) + .unwrap_or_default(), + ), + x.get(3).map(|y| { + ( + y.as_str().to_string(), + x.get(4).map(|x| x.as_str().to_string()).unwrap_or_default(), + ) + }), + ) + }) +} + +fn write_error_expr(expr: &str, uuid: &Uuid) { + if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { + tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); + return; + } + let dir_entries = match std::fs::read_dir(ERROR_DIR) { + Ok(entries) => entries.count(), + Err(_) => { + tracing::error!("failed to read error dir {ERROR_DIR}"); + return; + } + }; + + if std::env::var("PRINT_NATIVE_ERRORS").is_ok() { + tracing::info!("native error for job {uuid}: {expr}"); + } + if dir_entries >= 100 { + tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); + return; + } + + let path = format!("/{uuid}.js"); + tracing::info!( + "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" + ); + if let Err(e) = write_file(ERROR_DIR, &path, expr) { + tracing::error!("failed to write error expr to file {path}: {e}"); + } +} + +use windmill_common::utils::unsafe_raw; + +async fn append_result_stream( + conn: &Connection, + workspace_id: &str, + job_id: &Uuid, + nstream: &str, + offset: i32, +) -> windmill_common::error::Result<()> { + match conn { + Connection::Sql(db) => { + append_result_stream_db(db, workspace_id, job_id, nstream, offset).await?; + } + Connection::Http(client) => { + #[derive(serde::Serialize)] + struct ResultStreamBody<'a> { + result_stream: &'a str, + offset: i32, + } + let body = ResultStreamBody { result_stream: nstream, offset }; + if let Err(e) = client + .post::<_, String>( + &format!( + "/api/w/{}/agent_workers/push_result_stream/{}", + workspace_id, job_id + ), + None, + &body, + ) + .await + { + tracing::error!(%job_id, %e, "error sending result stream for job {job_id}: {e}"); + } + } + } + Ok(()) +} + +// ── ops ────────────────────────────────────────────────────────────── + +#[op2] +#[serde] +fn op_get_static_args(op_state: Rc>) -> Vec> { + op_state + .borrow() + .borrow::() + .args + .iter() + .map(|x| x.as_ref().map(|y| y.get().to_string())) + .collect_vec() +} + +#[op2(fast)] +fn op_log(op_state: Rc>, #[string] log: &str) { + if let Err(e) = op_state + .borrow_mut() + .borrow_mut::() + .s + .send(log.to_string()) + { + tracing::error!("failed to send log: {e}"); + } +} + +// ── eval_fetch_timeout ─────────────────────────────────────────────── + +/// Execute a NativeTS script using deno_core/V8. +/// +/// Returns `(result, has_stream)` where `has_stream` indicates if the result +/// came from an async iterable stream. +/// +/// `otel_initialized` should be `DENO_OTEL_INITIALIZED.load(SeqCst)`. +/// +/// The caller (windmill-worker) is responsible for wrapping this in +/// `run_future_with_polling_update_job_poller` for job cancellation/polling. +#[allow(clippy::too_many_arguments)] +pub async fn eval_fetch_timeout( + env_code: String, + ts_expr: String, + js_expr: String, + args: Option<&Json>>>, + script_entrypoint_override: Option, + job_id: Uuid, + conn: &Connection, + w_id: &str, + load_client: bool, + otel_initialized: bool, + stream_notifier_update: Option>, +) -> windmill_common::error::Result<(Box, bool)> { + let isolate_handle: Arc>> = Arc::new(Mutex::new(None)); + let _isolate_guard = IsolateDropGuard(isolate_handle.clone()); + let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); + let (result_stream_sender, mut result_stream_receiver) = mpsc::unbounded_channel::(); + + let conn_ = conn.clone(); + let w_id_ = w_id.to_string(); + tokio::spawn(async move { + while let Some(log) = append_logs_receiver.recv().await { + windmill_queue::append_logs(&job_id, &w_id_, log, &conn_).await + } + }); + + let append_result_stream_fn = append_result_stream; + let conn_ = conn.clone(); + let w_id_ = w_id.to_string(); + tokio::spawn(async move { + let mut offset = -1; + while let Some(stream) = result_stream_receiver.recv().await { + offset += 1; + if let Err(e) = append_result_stream_fn(&conn_, &w_id_, &job_id, &stream, offset).await + { + tracing::error!("failed to append result stream: {e}"); + } + } + }); + + let parsed_args = windmill_parser_ts::parse_deno_signature( + &ts_expr, + true, + false, + script_entrypoint_override.clone(), + )? + .args; + let spread = parsed_args + .into_iter() + .map(|x| { + args.as_ref() + .and_then(|args| args.0.get(&x.name).map(|x| x.clone())) + }) + .collect::>(); + + let ann = get_annotation(&ts_expr); + + #[cfg(not(feature = "enterprise"))] + if ann.proxy.is_some() { + return Err(Error::ExecutionErr("Proxy is an EE feature".to_string()).into()); + } + + let mut extra_logs = String::new(); + if ann.useragent.is_some() { + extra_logs.push_str(&format!("useragent: {}\n", ann.useragent.as_ref().unwrap())); + } + if ann.proxy.is_some() { + let (proxy, auth) = ann.proxy.as_ref().unwrap(); + extra_logs.push_str(&format!( + "proxy: {proxy} (basic auth: {})\n", + auth.is_some() + )); + } + + let result_f = tokio::task::spawn_blocking(move || { + let ops = vec![op_get_static_args(), op_log()]; + let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; + + let fetch_options = deno_fetch::Options { + root_cert_store_provider: None, + user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), + proxy: ann.proxy.map(|x| deno_tls::Proxy { + url: x.0, + basic_auth: x + .1 + .map(|(username, password)| deno_tls::BasicAuth { username, password }), + }), + ..Default::default() + }; + + let exts: Vec = vec![ + deno_telemetry::deno_telemetry::init_ops(), + deno_webidl::deno_webidl::init_ops(), + deno_url::deno_url::init_ops(), + deno_console::deno_console::init_ops(), + deno_web::deno_web::init_ops::( + Arc::new(BlobStore::default()), + None, + ), + deno_fetch::deno_fetch::init_ops::(fetch_options), + deno_net::deno_net::init_ops::(None, None), + ext, + ]; + + let options = RuntimeOptions { + is_main: true, + extensions: exts, + create_params: Some( + deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128), + ), + startup_snapshot: Some(RUNTIME_SNAPSHOT), + module_loader: Some(Rc::new(deno_core::FsModuleLoader)), + extension_transpiler: None, + ..Default::default() + }; + + let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>(); + + // Ensure V8 platform is initialized (idempotent, no-op if already done). + setup_deno_runtime().expect("V8 platform init failed"); + + // Serialize isolate creation as extra safety net against concurrent V8 + // isolate creation races. The main fix is the unprotected platform in + // setup_deno_runtime(), but this provides defense in depth. + let mut js_runtime = { + let _v8_lock = V8_ISOLATE_CREATE_LOCK + .lock() + .unwrap_or_else(|e| e.into_inner()); + JsRuntime::new(options) + }; + + // Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized. + if otel_initialized { + if let Err(e) = + js_runtime.execute_script("", "globalThis.__bootstrapOtel()") + { + tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e); + } + } + + js_runtime.add_near_heap_limit_callback(move |x, y| { + tracing::error!("heap limit reached: {x} {y}"); + if memory_limit_tx.send(()).is_err() { + tracing::error!("failed to send memory limit reached notification - isolate may already be terminating"); + }; + y * 2 + }); + + let (log_sender, mut log_receiver) = mpsc::unbounded_channel::(); + + { + let op_state = js_runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(PermissionsContainer {}); + op_state.put(MainArgs { args: spread }); + op_state.put(LogString { s: log_sender }); + } + + *isolate_handle.lock().unwrap_or_else(|e| e.into_inner()) = + Some(js_runtime.v8_isolate().thread_safe_handle()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let future = async { + if !extra_logs.is_empty() { + if let Err(e) = append_logs_sender.send(extra_logs) { + tracing::error!("failed to send extra logs: {e}"); + } + } + let handle = tokio::spawn(async move { + let mut result_stream = String::new(); + let mut is_stream = false; + while let Some(log) = log_receiver.recv().await { + use windmill_common::result_stream::extract_stream_from_logs; + + if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { + if !is_stream { + is_stream = true; + if let Some(ref f) = stream_notifier_update { + f(); + } + } + + result_stream.push_str(&stream); + if let Err(e) = result_stream_sender.send(stream) { + tracing::error!("failed to send result stream: {e}"); + } + } else { + if let Err(e) = append_logs_sender.send(log) { + tracing::error!("failed to send log: {e}"); + } + } + } + if !result_stream.is_empty() { + Some(result_stream) + } else { + None + } + }); + + let r = tokio::select! { + r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), script_entrypoint_override, load_client, &job_id, otel_initialized) => Ok(r), + _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string())) + }; + *isolate_handle.lock().unwrap_or_else(|e| e.into_inner()) = None; + drop(js_runtime); + if let Ok(r) = r { + match handle.await { + Ok(Some(logs)) => { + // merge_result_stream: if main result is null but stream exists, use stream + match r { + Ok(raw) if raw.get() == "null" => Ok((unsafe_raw(logs), true)), + Ok(raw) => Ok((raw, true)), + Err(e) => Err(e), + } + } + Ok(None) => Ok(r.map(|r| (r, false))?), + Err(e) => Err(Error::ExecutionErr(e.to_string())), + } + } else { + r.map(|r| r.map(|r| (r, false)))? + } + }; + let r = runtime.block_on(future)?; + + Ok(r) as windmill_common::error::Result<(Box, bool)> + }); + + result_f.await.map_err(windmill_common::error::to_anyhow)? +} + +async fn eval_fetch( + js_runtime: &mut JsRuntime, + expr: &str, + env_code: Option, + script_entrypoint_override: Option, + load_client: bool, + job_id: &Uuid, + _otel_initialized: bool, +) -> windmill_common::error::Result> { + if load_client { + if let Some(env_code) = env_code.as_ref() { + let _ = js_runtime + .load_side_es_module_from_code( + &deno_core::resolve_url("file:///windmill.ts") + .map_err(windmill_common::error::to_anyhow)?, + format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()), + ) + .await + .map_err(windmill_common::error::to_anyhow)?; + } + } + use anyhow::Context; + use deno_core::error::CoreError; + use windmill_common::worker::to_raw_value; + let source = format!("{}\n{expr}", env_code.unwrap_or_default()); + let _ = js_runtime + .load_side_es_module_from_code( + &deno_core::resolve_url("file:///eval.ts") + .map_err(windmill_common::error::to_anyhow)?, + source.to_string(), + ) + .await + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("failed to load module")?; + + let main_override = script_entrypoint_override.unwrap_or("main".to_string()); + + #[cfg(all(feature = "private", feature = "enterprise"))] + let otel_context_inject = if _otel_initialized { + let trace_id = job_id.as_simple().to_string(); + format!( + r#"globalThis.__enterSpan?.({{ + isRecording: () => true, + spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }}) +}});"# + ) + } else { + String::new() + }; + + #[cfg(not(all(feature = "private", feature = "enterprise")))] + let otel_context_inject = ""; + + let script = js_runtime + .execute_script( + "", + format!( + r#" +function isAsyncIterable(obj) {{ + return obj != null && typeof obj[Symbol.asyncIterator] === 'function'; +}} + +function processStreamIterative(res) {{ + const iterator = res[Symbol.asyncIterator](); + + function processLoop() {{ + return new Promise(function(resolve) {{ + function step() {{ + iterator.next().then(function(result) {{ + if (!result.done) {{ + const chunk = result.value; + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); + step(); + }} else {{ + resolve("null"); + }} + }}).catch(function(error) {{ + resolve("null"); + }}); + }} + step(); + }}); + }} + + return processLoop(); +}} + +{otel_context_inject} + +let args = Deno.core.ops.op_get_static_args().map(JSON.parse) +import("file:///eval.ts").then((module) => module.{main_override}(...args)) + .then(res => {{ + if (isAsyncIterable(res)) {{ + return processStreamIterative(res) + }} else {{ + return JSON.stringify(res ?? null); + }} + }}) +"# + ), + ) + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }) + .context("native script initialization")?; + + let fut = js_runtime.resolve(script); + let global = js_runtime + .with_event_loop_promise(fut, PollEventLoopOptions::default()) + .await + .map_err(|e| { + write_error_expr(expr, &job_id); + e + }); + + match global { + Ok(global) => { + let scope = &mut js_runtime.handle_scope(); + let local = v8::Local::new(scope, global); + let r = serde_v8::from_v8::>(scope, local) + .map_err(windmill_common::error::to_anyhow)?; + Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) + } + Err(CoreError::Js(e)) => { + let stack_head = e.frames.first().and_then(|f| { + if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") { + Some(format!( + "{}\n", + source + .lines() + .nth((f.line_number.unwrap_or(1)) as usize - 1) + .unwrap_or("") + .to_string() + )) + } else { + None + } + }); + let stack_s = format!( + "{}{}", + stack_head.unwrap_or("".to_string()), + e.stack.unwrap_or("".to_string()) + ); + let stack = if stack_s.is_empty() { + None + } else { + Some(stack_s) + }; + Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({ + "message": e.message, + "stack": stack, + "name": e.name, + })))) + } + Err(e) => Err(Error::ExecutionErr(e.print_with_cause())), + } +} diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js new file mode 100644 index 0000000000..2639353a04 --- /dev/null +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -0,0 +1,62 @@ +import * as abortSignal from "ext:deno_web/03_abort_signal.js"; +import * as base64 from "ext:deno_web/05_base64.js"; +import * as console from "ext:deno_console/01_console.js"; +import * as encoding from "ext:deno_web/08_text_encoding.js"; +import * as event from "ext:deno_web/02_event.js"; +import * as fetch from "ext:deno_fetch/26_fetch.js"; +import * as file from "ext:deno_web/09_file.js"; +import * as fileReader from "ext:deno_web/10_filereader.js"; +import * as formData from "ext:deno_fetch/21_formdata.js"; +import * as headers from "ext:deno_fetch/20_headers.js"; +import * as streams from "ext:deno_web/06_streams.js"; +import * as timers from "ext:deno_web/02_timers.js"; +import * as url from "ext:deno_url/00_url.js"; +import * as net from "ext:deno_net/01_net.js"; +import * as tls from "ext:deno_net/02_tls.js"; +import * as urlPattern from "ext:deno_url/01_urlpattern.js"; +import * as webidl from "ext:deno_webidl/00_webidl.js"; +import * as response from "ext:deno_fetch/23_response.js"; +import * as request from "ext:deno_fetch/23_request.js"; +import "ext:deno_web/02_structured_clone.js"; +import "ext:deno_web/04_global_interfaces.js"; +import "ext:deno_web/13_message_port.js"; +import "ext:deno_web/14_compression.js"; +import "ext:deno_web/15_performance.js"; +import "ext:deno_web/16_image_data.js"; +import "ext:deno_fetch/27_eventsource.js"; + +globalThis.atob = base64.atob; +globalThis.btoa = base64.btoa; +globalThis.fetch = fetch.fetch; +globalThis.Request = request.Request; +globalThis.Response = response.Response; +globalThis.Blob = file.Blob; +globalThis.URL = url.URL; +globalThis.FormData = formData.FormData; +globalThis.URLSearchParams = url.URLSearchParams; +globalThis.Headers = headers.Headers; +globalThis.FileReader = fileReader.FileReader; +globalThis.console = new console.Console((msg, level) => + globalThis.Deno.core.ops.op_log(msg) +); +globalThis.AbortController = abortSignal.AbortController; +globalThis.AbortSignal = abortSignal.AbortSignal; + +Object.assign(globalThis, { + clearInterval: timers.clearInterval, + clearTimeout: timers.clearTimeout, + setInterval: timers.setInterval, + setTimeout: timers.setTimeout, +}); + +// Expose bootstrapOtel globally so it can be called from Rust after runtime creation. +// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. +// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] +// consoleConfig: 0=ignore, 1=capture, 2=replace +globalThis.__bootstrapOtel = () => { + import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { + bootstrap([1, 0, 1, 0]); + // Expose enterSpan for setting parent trace context + globalThis.__enterSpan = enterSpan; + }); +}; diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js similarity index 100% rename from backend/windmill-worker/src/windmill-client.js rename to backend/windmill-runtime-nativets/src/windmill-client.js diff --git a/backend/windmill-store/Cargo.toml b/backend/windmill-store/Cargo.toml new file mode 100644 index 0000000000..3eddd95a39 --- /dev/null +++ b/backend/windmill-store/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "windmill-store" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_store" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] +private = ["windmill-common/private"] +no_auth = [] +oauth2 = ["dep:windmill-oauth"] +mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] +http_trigger = [] +postgres_trigger = [] +mqtt_trigger = [] +sqs_trigger = [] +gcp_trigger = [] +kafka = [] +nats = [] +openidconnect = ["windmill-common/openidconnect"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +windmill-mcp = { workspace = true, optional = true } +windmill-oauth = { workspace = true, optional = true } +axum.workspace = true +hyper.workspace = true +http.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +uuid.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true +sql-builder.workspace = true +async-recursion.workspace = true +futures.workspace = true +chrono.workspace = true +reqwest.workspace = true +anyhow.workspace = true diff --git a/backend/windmill-store/src/lib.rs b/backend/windmill-store/src/lib.rs new file mode 100644 index 0000000000..c7e606de43 --- /dev/null +++ b/backend/windmill-store/src/lib.rs @@ -0,0 +1,16 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(all(feature = "oauth2", feature = "private"))] +mod oauth_refresh_ee; +#[cfg(feature = "oauth2")] +pub mod oauth_refresh_oss; +pub mod resources; +pub mod secret_backend_ext; +pub mod var_resource_cache; +pub mod variables; diff --git a/backend/windmill-store/src/oauth_refresh_oss.rs b/backend/windmill-store/src/oauth_refresh_oss.rs new file mode 100644 index 0000000000..7402e66e49 --- /dev/null +++ b/backend/windmill-store/src/oauth_refresh_oss.rs @@ -0,0 +1,38 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(feature = "private")] +pub use crate::oauth_refresh_ee::_refresh_token; + +#[cfg(not(feature = "private"))] +use sqlx::{Postgres, Transaction}; +#[cfg(not(feature = "private"))] +use windmill_common::db::DB; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn _refresh_token<'c>( + tx: Transaction<'c, Postgres>, + path: &str, + w_id: &str, + id: i32, + db: &DB, +) -> error::Result { + windmill_oauth::refresh_token( + tx, + path, + w_id, + id, + db, + &*windmill_oauth::OAUTH_CLIENTS.read().await, + &windmill_oauth::OAUTH_HTTP_CLIENT, + include_str!("../../oauth_connect.json"), + ) + .await +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs new file mode 100644 index 0000000000..262908d649 --- /dev/null +++ b/backend/windmill-store/src/resources.rs @@ -0,0 +1,1732 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::collections::HashMap; + +use windmill_api_auth::{ + check_scopes, maybe_refresh_folders, require_owner_of_path, require_super_admin, ApiAuthed, + Tokened, +}; +use windmill_common::db::DB; + +use crate::secret_backend_ext::rename_vault_secret; +use crate::var_resource_cache::{cache_resource, get_cached_resource}; +use windmill_common::utils::BulkDeleteRequest; +use windmill_common::webhook::{WebhookMessage, WebhookShared}; + +use axum::{ + body::Body, + extract::{Extension, Path, Query}, + response::Response, + routing::{delete, get, post}, + Json, Router, +}; +use futures::future::try_join_all; +use hyper::{header, StatusCode}; +use serde::{Deserialize, Serialize}; +use serde_json::{value::RawValue, Value}; +use sql_builder::{bind::Bind, quote, SqlBuilder}; +use sqlx::{Acquire, FromRow, Postgres, Transaction}; +use std::process::Stdio; +use tokio::process::Command; +use uuid::Uuid; +use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; +use windmill_audit::ActionKind; +use windmill_common::{ + db::{DbWithOptAuthed, UserDB}, + error::{self, Error, JsonResult, Result}, + get_database_url, + utils::{ + get_custom_pg_instance_password, not_found_if_none, paginate, require_admin, Pagination, + StripPath, + }, + variables, + worker::{CLOUD_HOSTED, TMP_DIR}, + PgDatabase, +}; + +use async_recursion::async_recursion; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_resources)) + .route("/list_search", get(list_search_resources)) + .route("/list_names/:type", get(list_names)) + .route("/get/*path", get(get_resource)) + .route("/exists/*path", get(exists_resource)) + .route("/get_value/*path", get(get_resource_value)) + .route( + "/get_value_interpolated/*path", + get(get_resource_value_interpolated), + ) + .route("/update/*path", post(update_resource)) + .route("/update_value/*path", post(update_resource_value)) + .route("/delete/*path", delete(delete_resource)) + .route("/delete_bulk", delete(delete_resources_bulk)) + .route("/create", post(create_resource)) + .route("/git_commit_hash/*path", get(get_git_commit_hash)) + .route("/type/list", get(list_resource_types)) + .route("/type/listnames", get(list_resource_types_names)) + .route("/type/get/:name", get(get_resource_type)) + .route("/type/exists/:name", get(exists_resource_type)) + .route("/type/update/:name", post(update_resource_type)) + .route("/type/delete/:name", delete(delete_resource_type)) + .route( + "/file_resource_type_to_file_ext_map", + get(file_resource_ext_to_resource_type), + ) + .route("/type/create", post(create_resource_type)) +} + +pub fn public_service() -> Router { + Router::new().route("/custom_component/:name", get(custom_component)) +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct ResourceType { + pub workspace_id: String, + pub name: String, + pub schema: Option, + pub description: Option, + pub created_by: Option, + pub edited_at: Option>, + pub format_extension: Option, +} + +#[derive(Deserialize)] +pub struct CreateResourceType { + pub name: String, + pub schema: Option, + pub description: Option, + pub format_extension: Option, +} + +#[derive(Deserialize)] +pub struct EditResourceType { + pub schema: Option, + pub description: Option, +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct Resource { + pub workspace_id: String, + pub path: String, + pub value: Option, + pub description: Option, + pub resource_type: String, + pub extra_perms: serde_json::Value, + pub created_by: Option, + pub edited_at: Option>, +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct ListableResource { + pub workspace_id: String, + pub path: String, + pub value: Option, + pub description: Option, + pub resource_type: String, + pub extra_perms: serde_json::Value, + pub created_by: Option, + pub edited_at: Option>, + pub is_linked: Option, + pub is_refreshed: Option, + pub is_oauth: Option, + pub is_expired: Option, + pub refresh_error: Option, + pub account: Option, +} + +#[derive(Deserialize)] +pub struct CreateResource { + pub path: String, + pub value: Option>, + pub description: Option, + pub resource_type: String, +} +#[derive(Deserialize)] +struct EditResource { + path: Option, + description: Option, + value: Option>, +} + +#[derive(Deserialize)] +pub struct ListResourceQuery { + resource_type: Option, + resource_type_exclude: Option, + path_start: Option, +} + +#[derive(Serialize, FromRow)] +pub struct NamePath { + name: String, + path: String, +} +async fn list_names( + authed: ApiAuthed, + Path((w_id, rt)): Path<(String, String)>, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2", + rt, + &w_id + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .filter_map(|x| x.name.map(|name| NamePath { name, path: x.path })) + .collect::>(); + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Serialize, FromRow)] +pub struct SearchResource { + path: String, + value: serde_json::Value, +} +async fn list_search_resources( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + #[cfg(feature = "enterprise")] + let n = 1000; + + #[cfg(not(feature = "enterprise"))] + let n = 3; + + let rows = sqlx::query_as!( + SearchResource, + "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2", + &w_id, + n + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_resources( + authed: ApiAuthed, + Query(lq): Query, + Query(pagination): Query, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + + let mut sqlb = SqlBuilder::select_from("resource") + .fields(&[ + "resource.workspace_id", + "resource.path", + "null::JSONB as value", + "resource.description", + "resource_type", + "resource.extra_perms", + "(now() > account.expires_at) as is_expired", + "variable.path IS NOT NULL as is_linked", + "account.refresh_token != '' as is_refreshed", + "variable.is_oauth", + "variable.account", + "account.refresh_error", + "resource.created_by", + "resource.edited_at", + ]) + .left() + .join("variable") + .on("variable.path = resource.path AND variable.workspace_id = resource.workspace_id") + .left() + .join("account") + .on("variable.account = account.id AND account.workspace_id = variable.workspace_id") + .order_by("path", true) + .and_where("resource.workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + + if let Some(rt) = &lq.resource_type { + let resource_type_filters = rt.split(',').collect::>(); + if resource_type_filters.len() == 1 { + sqlb.and_where_eq("resource_type", "?".bind(rt)); + } else { + let mut list = Vec::new(); + for rt in resource_type_filters { + let quoted_value = quote(rt); + list.push(quoted_value); + } + sqlb.and_where_in("resource_type", list.as_slice()); + } + } + if let Some(rt) = &lq.resource_type_exclude { + for rt in rt.split(',') { + sqlb.and_where_ne("resource_type", "?".bind(&rt)); + } + } + + if let Some(path_start) = &lq.path_start { + sqlb.and_where_like_left("resource.path", path_start); + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as::<_, ListableResource>(&sql) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(rows)) +} + +async fn get_resource( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let resource_o = sqlx::query_as!( + ListableResource, + "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed, + account.refresh_error, + variable.path IS NOT NULL as is_linked, + variable.is_oauth as \"is_oauth?\", + variable.account + FROM resource + LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2 + LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2 + WHERE resource.path = $1 AND resource.workspace_id = $2", + path.to_owned(), + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if resource_o.is_none() { + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + } + let resource = not_found_if_none(resource_o, "Resource", path)?; + Ok(Json(resource)) +} + +async fn exists_resource( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn get_resource_value( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; + + let value_o = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path.to_owned(), + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + if value_o.is_none() { + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + } + + let value = not_found_if_none(value_o, "Resource", path)?; + + Ok(Json(value)) +} + +pub async fn explain_resource_perm_error( + path: &str, + w_id: &str, + db: &sqlx::Pool, + authed: &ApiAuthed, +) -> windmill_common::error::Result<()> { + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Resource {} not found", path)))?; + if path.starts_with("f/") { + let folder = path.split("/").nth(1).ok_or_else(|| { + Error::BadRequest(format!( + "path {} should have at least 2 components separated by /", + path + )) + })?; + let folder_extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(db) + .await?; + return Err(Error::NotAuthorized(format!( + "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() + ))); + } else { + return Err(Error::NotAuthorized(format!( + "Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default() + ))); + } +} + +async fn custom_component( + Extension(db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> Result { + let cc_o = sqlx::query_scalar!( + "SELECT value->>'js' FROM resource + WHERE path = $1 AND workspace_id = $2", + format!("f/app_custom/{name}"), + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + + let cc = not_found_if_none(cc_o, "Custom Component", name)?; + let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript"); + + Ok(res.body(Body::from(cc)).unwrap()) +} + +#[derive(Deserialize)] +pub struct JobInfo { + pub job_id: Option, + pub allow_cache: Option, +} + +async fn get_resource_value_interpolated( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Tokened { token }: Tokened, + Path((w_id, path)): Path<(String, StripPath)>, + Query(job_info): Query, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let db_with_opt_authed = + DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + get_resource_value_interpolated_internal( + &db_with_opt_authed, + w_id.as_str(), + path, + job_info.job_id, + Some(token.as_str()), + job_info.allow_cache.unwrap_or(false), + ) + .await + .map(|success| Json(success)) +} + +pub async fn get_resource_value_interpolated_internal<'a>( + db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, + workspace: &str, + path: &str, + job_id: Option, + token_for_context: Option<&str>, + allow_cache: bool, +) -> Result> { + // This is a special syntax to help debugging custom instance databases + if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") { + let db = db_with_opt_authed.db(); + require_super_admin(db_with_opt_authed.db(), &db_with_opt_authed.email()).await?; + let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + pg_creds.dbname = dbname.to_string(); + pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); + pg_creds.user = Some("custom_instance_user".to_string()); + let pg_creds = serde_json::to_value(&pg_creds) + .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?; + return Ok(Some(pg_creds)); + } + + if allow_cache { + if let Some(cached_value) = get_cached_resource(&workspace, &path) { + return Ok(Some(cached_value)); + } + } + use sqlx::Acquire; + let mut tx = db_with_opt_authed.begin().await?; + + let value_o = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path, + workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if value_o.is_none() { + if let Some(authed) = db_with_opt_authed.authed() { + let db = db_with_opt_authed.db(); + explain_resource_perm_error(path, workspace, db, authed).await?; + } + } + + let value = not_found_if_none(value_o, "Resource", path)?; + if let Some(value) = value { + let r = transform_json_value( + &db_with_opt_authed, + workspace, + value, + &job_id, + token_for_context, + ) + .await?; + if allow_cache { + cache_resource(&workspace, &path, r.clone()); + } + Ok(Some(r)) + } else { + Ok(None) + } +} + +#[async_recursion] +pub async fn transform_json_value( + db_with_opt_authed: &DbWithOptAuthed, + workspace: &str, + v: Value, + job_id: &Option, + token: Option<&str>, +) -> Result { + match v { + Value::String(y) if y.starts_with("$var:") => { + let path = y.strip_prefix("$var:").unwrap(); + + let v = + crate::variables::get_value_internal(&db_with_opt_authed, workspace, path, false) + .await?; + Ok(Value::String(v)) + } + Value::String(y) if y.starts_with("$res:") => { + let path = y.strip_prefix("$res:").unwrap(); + if path.split("/").count() < 2 { + return Err(Error::internal_err(format!( + "Invalid resource path: {path}" + ))); + } + let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; + let v = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path, + &workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let v = not_found_if_none(v, "Resource", path)?; + if let Some(v) = v { + transform_json_value(db_with_opt_authed, workspace, v, job_id, token).await + } else { + Ok(Value::Null) + } + } + Value::String(y) if y.starts_with("$") && job_id.is_some() => { + let mut tx = db_with_opt_authed.begin().await?; + let job_id = job_id.unwrap(); + let job = sqlx::query!( + "SELECT + v2_job.permissioned_as_email, + v2_job.created_by, + v2_job.parent_job, + v2_job.permissioned_as, + v2_job.runnable_path, + CASE WHEN v2_job.trigger_kind = 'schedule'::job_trigger_kind THEN v2_job.trigger END AS schedule_path, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job.root_job, + v2_job_queue.scheduled_for AS \"scheduled_for: chrono::DateTime\" + FROM v2_job INNER JOIN v2_job_queue ON v2_job.id = v2_job_queue.id + WHERE v2_job.id = $1 AND v2_job.workspace_id = $2", + job_id, + workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let job = not_found_if_none(job, "Job", job_id.to_string())?; + + let flow_path = if let Some(uuid) = job.parent_job { + let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; + let p = sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) + .fetch_optional(&mut *tx) + .await? + .flatten(); + tx.commit().await?; + p + } else { + None + }; + + let variables = variables::get_reserved_variables( + &db_with_opt_authed.db().into(), + workspace, + token.unwrap_or_else(|| "no_token_available"), + &job.permissioned_as_email, + &job.created_by, + &job_id.to_string(), + &job.permissioned_as, + job.runnable_path.clone(), + job.parent_job.map(|x| x.to_string()), + flow_path, + job.schedule_path.clone(), + job.flow_step_id.clone(), + job.flow_innermost_root_job.map(|x| x.to_string()), + job.root_job.map(|x| x.to_string()), + Some(job.scheduled_for.clone()), + None, + None, + ) + .await; + + let name = y.strip_prefix("$").unwrap(); + + let value = variables + .iter() + .find(|x| x.name == name) + .map(|x| x.value.clone()) + .unwrap_or_else(|| y); + Ok(serde_json::json!(value)) + } + Value::Object(mut m) => { + for (a, b) in m.clone().into_iter() { + let v = + transform_json_value(db_with_opt_authed, workspace, b, job_id, token).await?; + m.insert(a.clone(), v); + } + Ok(Value::Object(m)) + } + a @ _ => Ok(a), + } +} + +async fn check_path_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + path: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!( + "Resource {} already exists", + path + ))); + } + return Ok(()); +} + +#[derive(Deserialize)] +struct CreateResourceQuery { + update_if_exists: Option, +} +async fn create_resource( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Query(q): Query, + Json(resource): Json, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("resources:write:{}", resource.path))?; + if *CLOUD_HOSTED { + let nb_resources = sqlx::query_scalar!( + "SELECT COUNT(*) FROM resource WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + if nb_resources.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of resources (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await; + + let mut tx = user_db.begin(&authed).await?; + + let update_if_exists = q.update_if_exists.unwrap_or(false); + if !update_if_exists { + check_path_conflict(&mut tx, &w_id, &resource.path).await?; + } + + let res_value = resource.value.unwrap_or_default(); + let raw_json = sqlx::types::Json(res_value.as_ref()); + + if resource.path.starts_with("f/app_themes/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } else if resource.path.starts_with("f/app_custom/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_custom', 'App Custom Components', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } else if resource.path.starts_with("f/app_groups/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } + sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now()", + w_id, + resource.path, + raw_json as sqlx::types::Json<&RawValue>, + resource.description, + resource.resource_type, + authed.username + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "resources.create", + ActionKind::Create, + &w_id, + Some(&resource.path), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: resource.path.clone(), parent_path: None }, + Some(format!("Resource '{}' created", resource.path.clone())), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, + ); + + Ok(( + StatusCode::CREATED, + format!("resource {} created", resource.path), + )) +} + +async fn delete_resource( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + + check_scopes(&authed, || format!("resources:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let deleted_path = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + not_found_if_none(deleted_path, "Resource", &path)?; + sqlx::query!( + "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "resources.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, + Some(format!("Resource '{}' deleted", path)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() }, + ); + + Ok(format!("resource {} deleted", path)) +} + +async fn delete_resources_bulk( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult> { + for path in &request.paths { + check_scopes(&authed, || format!("resources:write:{}", path))?; + } + + let mut tx = user_db.begin(&authed).await?; + + let deleted_paths = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", + &request.paths, + w_id + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "resources.delete_bulk", + ActionKind::Delete, + &w_id, + Some(&deleted_paths.join(", ")), + None, + ) + .await?; + + tx.commit().await?; + + try_join_all(deleted_paths.iter().map(|path| { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { + path: path.to_string(), + parent_path: Some(path.to_string()), + }, + Some(format!("Resource '{}' deleted", path)), + true, + None, + ) + })) + .await?; + + for path in &deleted_paths { + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteResource { workspace: w_id.clone(), path: path.to_owned() }, + ); + } + + Ok(Json(deleted_paths)) +} + +async fn update_resource( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(ns): Json, +) -> Result { + use sql_builder::prelude::*; + + let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; + + let mut sqlb = SqlBuilder::update_table("resource"); + sqlb.and_where_eq("path", "?".bind(&path)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + + if let Some(npath) = &ns.path { + sqlb.set_str("path", npath); + } + if let Some(nvalue) = ns.value { + sqlb.set_str("value", nvalue.to_string()); + } + if let Some(ndesc) = ns.description { + sqlb.set_str("description", ndesc); + } + + sqlb.set_str("edited_at", "now()"); + + sqlb.returning("path"); + let authed = maybe_refresh_folders(path, &w_id, authed, &db).await; + + let mut tx = user_db.begin(&authed).await?; + + if let Some(npath) = ns.path.clone() { + if npath != path { + check_path_conflict(&mut tx, &w_id, &npath).await?; + + require_owner_of_path(&authed, path)?; + + // Handle Vault secret rename if the linked variable is a Vault-stored secret + let linked_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = linked_var { + if var.is_secret { + // Check if this is a Vault-stored secret and rename it + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + + sqlx::query!( + "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", + npath, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; + + let npath = not_found_if_none(npath_o, "Resource", path)?; + + audit_log( + &mut *tx, + &authed, + "resources.update", + ActionKind::Update, + &w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + + // Detect if this was a rename operation + let old_path_if_renamed = if npath != path { Some(path) } else { None }; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: npath.to_string(), parent_path: Some(path.to_string()) }, + Some(format!("Resource '{}' updated", npath)), + true, + old_path_if_renamed, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateResource { + workspace: w_id, + old_path: path.to_owned(), + new_path: npath.clone(), + }, + ); + + Ok(format!("resource {} updated (npath: {:?})", path, npath)) +} + +#[derive(FromRow, Serialize, Deserialize)] +struct UpdateResource { + value: Option, +} + +async fn update_resource_value( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(nv): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "UPDATE resource SET value = $1, edited_at = now() WHERE path = $2 AND workspace_id = $3", + nv.value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "resources.update", + ActionKind::Update, + &w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, + None, + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateResource { + workspace: w_id, + old_path: path.to_owned(), + new_path: path.to_owned(), + }, + ); + + Ok(format!("value of resource {} updated", path)) +} + +async fn file_resource_ext_to_resource_type( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + #[derive(Serialize, sqlx::FromRow)] + struct LocalFileResourceExtension { + name: String, + format_extension: Option, + } + + let r = sqlx::query_as!(LocalFileResourceExtension, " + SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')", w_id) + .fetch_all(&db) + .await?; + + let hashmap: HashMap = r + .into_iter() + .filter_map(|entry| { + if let Some(format_extension) = entry.format_extension { + Some((entry.name, format_extension)) + } else { + None + } + }) + .collect(); + + Ok(Json(hashmap)) +} + +async fn list_resource_types( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let rows = sqlx::query_as!( + ResourceType, + "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + BY name", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(rows)) +} + +async fn list_resource_types_names( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let rows = sqlx::query_scalar!( + "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') \ + ORDER BY name", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(rows)) +} + +async fn get_resource_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + + let resource_type_o = sqlx::query_as!( + ResourceType, + "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + &name, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let resource_type = not_found_if_none(resource_type_o, "ResourceType", name)?; + Ok(Json(resource_type)) +} + +async fn exists_resource_type( + Extension(db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins'))", + name, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn create_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(resource_type): Json, +) -> Result<(StatusCode, String)> { + let mut tx = user_db.begin(&authed).await?; + + check_rt_path_conflict(&mut tx, &w_id, &resource_type.name).await?; + + sqlx::query!( + "INSERT INTO resource_type + (workspace_id, name, schema, description, created_by, format_extension, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, now())", + w_id, + resource_type.name, + resource_type.schema, + resource_type.description, + authed.username, + resource_type.format_extension, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "resource_types.create", + ActionKind::Create, + &w_id, + Some(&resource_type.name), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::ResourceType { path: resource_type.name.clone() }, + Some(format!( + "Resource Type '{}' created", + resource_type.name.clone() + )), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateResourceType { name: resource_type.name.clone() }, + ); + + Ok(( + StatusCode::CREATED, + format!("resource_type {} created", resource_type.name), + )) +} + +async fn check_rt_path_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + name: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)", + name, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!( + "Resource type {} already exists", + name + ))); + } + return Ok(()); +} + +async fn delete_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + let deleted_name = sqlx::query_scalar!( + "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", + name, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(deleted_name, "ResourceType", &name)?; + + audit_log( + &mut *tx, + &authed, + "resource_types.delete", + ActionKind::Delete, + &w_id, + Some(&name), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::ResourceType { path: name.clone() }, + None, + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteResourceType { name: name.clone() }, + ); + + Ok(format!("resource_type {} deleted", name)) +} + +async fn update_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(ns): Json, +) -> Result { + use sql_builder::prelude::*; + + let mut sqlb = SqlBuilder::update_table("resource_type"); + sqlb.and_where_eq("name", "?".bind(&name)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + if let Some(nschema) = ns.schema { + sqlb.set_str("schema", nschema); + } + if let Some(ndesc) = ns.description { + sqlb.set_str("description", ndesc); + } + sqlb.set_str("edited_at", "now()"); + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let mut tx = user_db.begin(&authed).await?; + + sqlx::query(&sql).execute(&mut *tx).await?; + audit_log( + &mut *tx, + &authed, + "resource_types.update", + ActionKind::Update, + &w_id, + Some(&name), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::ResourceType { path: name.clone() }, + None, + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateResourceType { name: name.clone() }, + ); + + Ok(format!("resource_type {} updated", name)) +} + +#[cfg(any( + feature = "http_trigger", + feature = "postgres_trigger", + feature = "mqtt_trigger", + all( + feature = "enterprise", + any( + feature = "sqs_trigger", + feature = "gcp_trigger", + feature = "kafka", + feature = "nats" + ) + ) +))] +pub async fn try_get_resource_from_db_as( + authed: &ApiAuthed, + user_db: Option, + db: &DB, + resource_path: &str, + w_id: &str, +) -> Result +where + T: serde::de::DeserializeOwned, +{ + let resource = get_resource_value_interpolated_internal( + &DbWithOptAuthed::from_authed(authed, db.clone(), user_db), + &w_id, + &resource_path, + None, + None, + false, + ) + .await?; + + let resource = match resource { + Some(resource) => serde_json::from_value::(resource) + .map_err(|e| Error::SerdeJson { error: e, location: "resources.rs".to_string() })?, + None => { + return { + Err(Error::NotFound(format!( + "resource at path :{} do not exist", + &resource_path + ))) + } + } + }; + + Ok(resource) +} + +#[derive(Deserialize, Serialize)] +struct GitRepositoryResource { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, +} + +/// Validates a git URL to prevent git option injection attacks. +/// Git URLs starting with '-' could be interpreted as command-line options. +fn validate_git_url(url: &str) -> Result<()> { + let url = url.trim(); + if url.is_empty() { + return Err(Error::BadRequest("Git URL cannot be empty".to_string())); + } + if url.starts_with('-') { + return Err(Error::BadRequest( + "Git URL cannot start with '-' (potential option injection)".to_string(), + )); + } + // Block other potentially dangerous patterns + if url.contains('\0') || url.contains('\n') || url.contains('\r') { + return Err(Error::BadRequest( + "Git URL contains invalid characters".to_string(), + )); + } + Ok(()) +} + +/// Validates a git branch/ref name to prevent injection attacks. +fn validate_git_ref(ref_name: &str) -> Result<()> { + let ref_name = ref_name.trim(); + if ref_name.is_empty() { + return Err(Error::BadRequest("Git ref cannot be empty".to_string())); + } + if ref_name.starts_with('-') { + return Err(Error::BadRequest( + "Git ref cannot start with '-' (potential option injection)".to_string(), + )); + } + // Git ref names have specific rules - block dangerous characters + if ref_name.contains('\0') + || ref_name.contains('\n') + || ref_name.contains('\r') + || ref_name.contains("..") + || ref_name.contains("@{") + || ref_name.ends_with('.') + || ref_name.ends_with('/') + || ref_name.contains("//") + { + return Err(Error::BadRequest( + "Git ref contains invalid characters or patterns".to_string(), + )); + } + Ok(()) +} + +#[derive(Serialize)] +struct GitCommitHashResponse { + commit_hash: String, +} + +#[derive(Deserialize)] +struct GitCommitHashQuery { + git_ssh_identity: Option, +} + +async fn get_git_commit_hash( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> JsonResult { + let path = path.to_path(); + + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let db_with_opt_authed = + DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + let git_repo_resource_value = get_resource_value_interpolated_internal( + &db_with_opt_authed, + &w_id, + path, + None, + None, + false, + ) + .await + .map_err(|e| Error::NotFound(format!("Access to resource {} denied: ({e})", path)))?; + + let git_resource: GitRepositoryResource = match git_repo_resource_value { + Some(value) => serde_json::from_value(value).map_err(|e| { + Error::BadRequest(format!("Invalid git repository resource format: {}", e)) + })?, + None => return Err(Error::NotFound(format!("Resource {} not found", path)).into()), + }; + + let identities: Vec = query + .git_ssh_identity + .map(|s| { + s.split(",") + .filter_map(|s| { + if !s.is_empty() { + Some(s.to_string()) + } else { + None + } + }) + .collect() + }) + .unwrap_or(vec![]); + + let (git_ssh_cmd, filenames) = + get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; + + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + + delete_paths(&filenames).await; + + Ok(Json(GitCommitHashResponse { commit_hash: commit_hash? })) +} + +async fn write_ssh_file( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + var_path: &str, +) -> std::result::Result { + let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); + let loc = std::path::Path::new(TMP_DIR) + .join("ssh_ids") + .join(id_file_name); + + let userdb_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + let mut content = crate::variables::get_value_internal(&userdb_authed, &w_id, &var_path, false) + .await + .map_err(|e| { + ( + error::Error::NotFound(format!( + "Variable {var_path} not found for git ssh identity: {e:#}" + )), + loc.clone(), + ) + })?; + content.push_str("\n"); + + if let Some(p) = &loc.parent() { + tokio::fs::create_dir_all(p) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + tokio::fs::write(&loc, content) + .await + .map_err(|e| (e.into(), loc.clone()))?; + + #[cfg(unix)] + { + let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); + tokio::fs::set_permissions(&loc, perm) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + + return Ok(loc); +} + +async fn delete_paths(paths: &Vec) { + for path in paths { + let _ = tokio::fs::remove_file(&path).await; + } +} + +async fn get_git_ssh_cmd( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + git_ssh_identity: Vec, +) -> error::Result<(Option, Vec)> { + if git_ssh_identity.len() > 5 { + return Err(error::Error::BadRequest( + "Too many ssh identities, try using at most 1".to_string(), + )); + } + if git_ssh_identity.len() == 0 { + return Ok((None, vec![])); + } + + let mut ssh_id_files = vec![]; + let mut file_paths = vec![]; + for var_path in git_ssh_identity.iter() { + match write_ssh_file(authed, user_db, db, w_id, &var_path).await { + Ok(loc) => { + ssh_id_files.push(format!( + " -i '{}'", + loc.to_string_lossy().replace('\'', r"'\''") + )); + file_paths.push(loc); + } + Err((e, loc)) => { + file_paths.push(loc); + delete_paths(&file_paths).await; + return Err(e); + } + } + } + + let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); + Ok((Some(git_ssh_cmd), file_paths)) +} + +async fn get_repo_latest_commit_hash( + git_resource: &GitRepositoryResource, + git_ssh_command: Option, +) -> Result { + // Validate URL and branch to prevent option injection attacks + validate_git_url(&git_resource.url)?; + + let ref_spec = git_resource + .branch + .as_deref() + .filter(|s| !s.is_empty()) + .unwrap_or("HEAD"); + + // Validate ref_spec if it's not the default HEAD + if ref_spec != "HEAD" { + validate_git_ref(ref_spec)?; + } + + let mut git_cmd = Command::new("git"); + git_cmd.args(["ls-remote", &git_resource.url, ref_spec]); + if let Some(git_ssh_command) = git_ssh_command { + git_cmd.env("GIT_SSH_COMMAND", git_ssh_command); + } + git_cmd.stderr(Stdio::piped()); + + let output = git_cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8(output.stderr) + .unwrap_or_else(|_| "Failed to decode stderr".to_string()); + return Err(Error::BadRequest(format!( + "Error getting git repo commit hash: {}", + stderr + ))); + } + + let stdout = String::from_utf8(output.stdout) + .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; + + let lines: Vec<&str> = stdout.lines().collect(); + + if lines.is_empty() { + return Err(Error::BadRequest(format!( + "No commits found for reference '{}' in repository '{}'", + ref_spec, git_resource.url + ))); + } + + let commit_hash = lines + .first() + .and_then(|line| line.split_whitespace().next()) + .map(|s| s.to_string()) + .ok_or_else(|| { + Error::BadRequest("Unexpected output format for git ls-remote".to_string()) + })?; + + Ok(commit_hash) +} + +#[cfg(all( + feature = "enterprise", + any(feature = "nats", feature = "kafka", feature = "sqs_trigger") +))] +pub async fn interpolate( + authed: &ApiAuthed, + db: &DB, + w_id: &str, + s: String, +) -> std::result::Result { + use serde_json::Value; + use windmill_common::db::DbWithOptAuthed; + let value = Value::String(s); + match transform_json_value( + &DbWithOptAuthed::from_authed(authed, db.clone(), None), + w_id, + value, + &None, + None, + ) + .await? + { + Value::String(s) => Ok(s), + v => Err(anyhow::anyhow!("Expected string, got {:?}", v)), + } +} diff --git a/backend/windmill-store/src/secret_backend_ext.rs b/backend/windmill-store/src/secret_backend_ext.rs new file mode 100644 index 0000000000..5fdc4fdb38 --- /dev/null +++ b/backend/windmill-store/src/secret_backend_ext.rs @@ -0,0 +1,371 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret backend extension for the API layer +//! +//! This module provides helper functions for integrating the SecretBackend +//! trait with variable operations in the API. +//! +//! Note: HashiCorp Vault integration requires Enterprise Edition. +//! The OSS version only supports the database backend. + +use std::sync::Arc; + +use windmill_common::{ + db::DB, + error::{Error, Result}, + secret_backend::{database::DatabaseBackend, SecretBackend}, + variables::{build_crypt, decrypt, encrypt}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::{ + global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, + secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use tokio::sync::RwLock; + +// Cached Vault backend to avoid recreating it for every request +// This enables connection pooling and avoids repeated setup overhead +#[cfg(all(feature = "private", feature = "enterprise"))] +struct CachedVaultBackend { + backend: Arc, + settings: VaultSettings, +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +lazy_static::lazy_static! { + static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); +} + +/// Get the current secret backend based on global settings +/// +/// OSS: Always returns DatabaseBackend +/// EE: Returns configured backend (Database or Vault) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn get_secret_backend(db: &DB) -> Result> { + Ok(Arc::new(DatabaseBackend::new(db.clone()))) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn get_secret_backend(db: &DB) -> Result> { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))), + SecretBackendConfig::HashiCorpVault(settings) => { + get_or_create_vault_backend(db, settings).await + } + } +} + +/// Get a cached Vault backend or create a new one if settings changed +#[cfg(all(feature = "private", feature = "enterprise"))] +async fn get_or_create_vault_backend( + _db: &DB, + settings: VaultSettings, +) -> Result> { + // Check if we have a cached backend with matching settings (read lock) + { + let cache = VAULT_BACKEND_CACHE.read().await; + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + } + + // Need to create a new backend - acquire write lock + let mut cache = VAULT_BACKEND_CACHE.write().await; + + // Double-check (another task may have created it while we waited) + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + + // Create new backend + let backend: Arc = { + #[cfg(feature = "openidconnect")] + if settings.token.is_none() { + Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone())) + } else { + Arc::new(VaultBackend::new(settings.clone())) + } + + #[cfg(not(feature = "openidconnect"))] + Arc::new(VaultBackend::new(settings.clone())) + }; + + // Cache it + *cache = Some(CachedVaultBackend { backend: backend.clone(), settings }); + + Ok(backend) +} + +/// Check if a Vault backend is currently configured +/// +/// OSS: Always returns false +/// EE: Checks global settings +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn is_vault_backend_configured(_db: &DB) -> Result { + Ok(false) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn is_vault_backend_configured(db: &DB) -> Result { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) +} + +/// Get a secret value using the configured backend +/// +/// For database backend: decrypts using workspace key +/// For vault backend (EE only): fetches from Vault directly +pub async fn get_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + encrypted_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database decryption + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value.to_string()).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + "hashicorp_vault" => { + // Fetch from Vault directly + backend.get_secret(workspace_id, path).await + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Store a secret value using the configured backend +/// +/// For database backend: encrypts using workspace key and returns encrypted value +/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage +pub async fn store_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + plain_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database encryption + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, plain_value)) + } + "hashicorp_vault" => { + // Store in Vault and return a marker for DB + backend.set_secret(workspace_id, path, plain_value).await?; + Ok(format!("$vault:{}", path)) + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Delete a secret from the configured backend (if using Vault) +/// +/// For database backend: no-op (DB delete is handled separately) +/// For vault backend (EE only): deletes from Vault +pub async fn delete_secret_from_backend(db: &DB, workspace_id: &str, path: &str) -> Result<()> { + if is_vault_backend_configured(db).await? { + let backend = get_secret_backend(db).await?; + // Ignore NotFound errors during deletion (secret might not exist in Vault) + match backend.delete_secret(workspace_id, path).await { + Ok(()) => Ok(()), + Err(Error::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } else { + Ok(()) + } +} + +/// Check if a value is stored in Vault (indicated by the $vault: prefix) +pub fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") +} + +/// Rename a secret in Vault when a variable path changes (EE only) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secret( + _db: &DB, + _workspace_id: &str, + _old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + if is_vault_stored_value(current_value) { + tracing::warn!( + "Variable has $vault: prefix but Vault requires Enterprise Edition. \ + Updating DB reference to {}", + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Ok(None) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secret( + db: &DB, + workspace_id: &str, + old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + if !is_vault_stored_value(current_value) { + return Ok(None); + } + + if !is_vault_backend_configured(db).await? { + tracing::warn!( + "Variable value has $vault: prefix but Vault is not configured. \ + Updating DB reference from {} to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + + let backend = get_secret_backend(db).await?; + + let secret_value = match backend.get_secret(workspace_id, old_path).await { + Ok(value) => value, + Err(Error::NotFound(_)) => { + tracing::warn!( + "Secret not found in Vault at path {} during rename to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Err(e) => return Err(e), + }; + + backend + .set_secret(workspace_id, new_path, &secret_value) + .await?; + + if let Err(e) = backend.delete_secret(workspace_id, old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename to {}: {}", + old_path, + new_path, + e + ); + } + + Ok(Some(format!("$vault:{}", new_path))) +} + +/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secrets_with_prefix( + _db: &DB, + _workspace_id: &str, + _old_prefix: &str, + _new_prefix: &str, + _variables: Vec<(String, String)>, +) -> Result> { + Ok(vec![]) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secrets_with_prefix( + db: &DB, + workspace_id: &str, + old_prefix: &str, + new_prefix: &str, + variables: Vec<(String, String)>, +) -> Result> { + if !is_vault_backend_configured(db).await? { + return Ok(vec![]); + } + + let backend = get_secret_backend(db).await?; + let mut updates = Vec::new(); + + for (old_path, value) in variables { + if !is_vault_stored_value(&value) { + continue; + } + + let new_path = if old_path.starts_with(old_prefix) { + format!("{}{}", new_prefix, &old_path[old_prefix.len()..]) + } else { + continue; + }; + + let secret_value = match backend.get_secret(workspace_id, &old_path).await { + Ok(v) => v, + Err(Error::NotFound(_)) => { + updates.push((old_path, format!("$vault:{}", new_path))); + continue; + } + Err(e) => { + tracing::error!( + "Failed to read secret at {} during bulk rename: {}", + old_path, + e + ); + continue; + } + }; + + if let Err(e) = backend + .set_secret(workspace_id, &new_path, &secret_value) + .await + { + tracing::error!( + "Failed to write secret to {} during bulk rename: {}", + new_path, + e + ); + continue; + } + + if let Err(e) = backend.delete_secret(workspace_id, &old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename: {}", + old_path, + e + ); + } + + updates.push((old_path, format!("$vault:{}", new_path))); + } + + Ok(updates) +} diff --git a/backend/windmill-api/src/var_resource_cache.rs b/backend/windmill-store/src/var_resource_cache.rs similarity index 96% rename from backend/windmill-api/src/var_resource_cache.rs rename to backend/windmill-store/src/var_resource_cache.rs index d97416b871..f7ce2aeecf 100644 --- a/backend/windmill-api/src/var_resource_cache.rs +++ b/backend/windmill-store/src/var_resource_cache.rs @@ -40,22 +40,20 @@ impl CacheEntry { } } - lazy_static::lazy_static! { /// Cache for individual variable values: key = "workspace_id:path" pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); - - /// Cache for resource values: key = "workspace_id:path" + + /// Cache for resource values: key = "workspace_id:path" pub static ref RESOURCE_CACHE: Cache> = Cache::new(1000); } - /// Generate cache key for variables and resources pub fn cache_key(workspace_id: &str, path: &str) -> String { format!("{}:{}", workspace_id, path) } -/// Get cached variable if available and not expired +/// Get cached variable if available and not expired pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { let key = cache_key(workspace_id, path); VARIABLE_CACHE.get(&key).and_then(|entry| { @@ -119,4 +117,4 @@ pub fn clear_all_caches() { VARIABLE_CACHE.clear(); RESOURCE_CACHE.clear(); tracing::debug!("All variable/resource caches cleared"); -} \ No newline at end of file +} diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs new file mode 100644 index 0000000000..9c3dd652db --- /dev/null +++ b/backend/windmill-store/src/variables.rs @@ -0,0 +1,926 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_api_auth::{check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed}; +use windmill_common::db::DB; + +use crate::secret_backend_ext::{ + delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, + store_secret_value, +}; +use windmill_common::utils::BulkDeleteRequest; +use windmill_common::webhook::{WebhookMessage, WebhookShared}; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use futures::future::try_join_all; +use hyper::StatusCode; +use serde_json::Value; + +use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; +use windmill_audit::ActionKind; +use windmill_common::{ + db::{DbWithOptAuthed, UserDB}, + error::{Error, JsonResult, Result}, + scripts::ScriptHash, + utils::{not_found_if_none, paginate, Pagination, StripPath, WarnAfterExt}, + variables::{ + build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable, + }, + worker::CLOUD_HOSTED, +}; + +use crate::var_resource_cache::{cache_variable, get_cached_variable}; +use lazy_static::lazy_static; +use serde::Deserialize; +use sqlx::{Acquire, Postgres, Transaction}; +use windmill_common::variables::encrypt; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; + +lazy_static! { + pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_variables)) + .route("/list_contextual", get(list_contextual_variables)) + .route("/get/*path", get(get_variable)) + .route("/get_value/*path", get(get_value)) + .route("/exists/*path", get(exists_variable)) + .route("/update/*path", post(update_variable)) + .route("/delete/*path", delete(delete_variable)) + .route("/delete_bulk", delete(delete_variables_bulk)) + .route("/create", post(create_variable)) + .route("/encrypt", post(encrypt_value)) +} + +async fn list_contextual_variables( + Path(w_id): Path, + ApiAuthed { username, email, .. }: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + Ok(Json( + get_reserved_variables( + &db.into(), + &w_id, + "q1A0qcPuO00yxioll7iph76N9CJDqn", + &email, + &username, + "017e0ad5-f499-73b6-5488-92a61c5196dd", + format!("u/{username}").as_str(), + Some("u/user/script_path".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some("u/user/encapsulating_flow_path".to_string()), + Some("u/user/triggering_flow_path".to_string()), + Some("c".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some(chrono::offset::Utc::now()), + Some(ScriptHash(1234567890)), + None, + ) + .await + .to_vec(), + )) +} + +#[derive(Deserialize)] +struct ListVariableQuery { + path_start: Option, +} + +async fn list_variables( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(lq): Query, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + + let mut tx = user_db.begin(&authed).await?; + + let rows = sqlx::query_as::<_, ListableVariable>( + "SELECT variable.workspace_id, variable.path, CASE WHEN is_secret IS TRUE THEN null ELSE variable.value::text END as value, + is_secret, variable.description, variable.extra_perms, account, is_oauth, (now() > account.expires_at) as is_expired, + account.refresh_error, + resource.path IS NOT NULL as is_linked, + account.refresh_token != '' as is_refreshed, + variable.expires_at + from variable + LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $1 + LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $1 + WHERE variable.workspace_id = $1 AND variable.path NOT LIKE 'u/' || $2 || '/secret_arg/%' + AND variable.path LIKE $3 || '%' + ORDER BY path + LIMIT $4 OFFSET $5 +", + ) + .bind(&w_id) + .bind(&authed.username) + .bind(&lq.path_start.unwrap_or_default()) + .bind(per_page as i32) + .bind(offset as i32) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct GetVariableQuery { + decrypt_secret: Option, + include_encrypted: Option, +} + +async fn get_variable( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Query(q): Query, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("variables:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; + + let variable_o = sqlx::query_as::<_, ListableVariable>( + "SELECT variable.*, (now() > account.expires_at) as is_expired, account.refresh_error, + resource.path IS NOT NULL as is_linked, + account.refresh_token != '' as is_refreshed + from variable + LEFT JOIN account ON variable.account = account.id + LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $2 + WHERE variable.path = $1 AND variable.workspace_id = $2 + LIMIT 1", + ) + .bind(&path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + let variable = if let Some(variable) = variable_o { + variable + } else { + explain_variable_perm_error(&path, &w_id, &db).await?; + unreachable!() + }; + + let decrypt_secret = q.decrypt_secret.unwrap_or(true); + + let r = if variable.is_secret { + if decrypt_secret { + audit_log( + &mut *tx, + &authed, + "variables.decrypt_secret", + ActionKind::Execute, + &w_id, + Some(&variable.path), + None, + ) + .await?; + } + + let value = variable.value.unwrap_or_else(|| "".to_string()); + ListableVariable { + value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() { + #[cfg(feature = "oauth2")] + { + let refresh_tx = db + .begin() + .await + .map_err(|e| Error::InternalErr(e.to_string()))?; + Some( + crate::oauth_refresh_oss::_refresh_token( + refresh_tx, + &variable.path, + &w_id, + variable.account.unwrap(), + &db, + ) + .await?, + ) + } + #[cfg(not(feature = "oauth2"))] + return Err(Error::internal_err("Require oauth2 feature".to_string())); + } else if !value.is_empty() && decrypt_secret { + let _ = tx.commit().await; + // Use secret backend for decryption (supports both DB and Vault) + Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) + } else if q.include_encrypted.unwrap_or(false) { + Some(value) + } else { + None + }, + ..variable + } + } else { + variable + }; + + Ok(Json(r)) +} + +#[derive(Deserialize)] +struct GetValueQuery { + allow_cache: Option, +} +async fn get_value( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(q): Query, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("variables:read:{}", path))?; + let userdb_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + + return get_value_internal(&userdb_authed, &w_id, &path, q.allow_cache.unwrap_or(false)) + .warn_after_seconds(10) + .await + .map(Json); +} + +async fn explain_variable_perm_error( + path: &str, + w_id: &str, + db: &sqlx::Pool, +) -> windmill_common::error::Result<()> { + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Variable {} not found", path)))?; + if path.starts_with("f/") { + let folder = path.split("/").nth(1).ok_or_else(|| { + Error::BadRequest(format!( + "path {} should have at least 2 components separated by /", + path + )) + })?; + let folder_extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(db) + .await?; + return Err(Error::NotAuthorized(format!( + "Variable exists but you don't have access to it:\nvariable perms: {}\nfolder perms: {}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() + ))); + } else { + return Err(Error::NotAuthorized(format!( + "Variable exists but you don't have access to it:\nvariable perms: {}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default() + ))); + } +} + +async fn exists_variable( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!( + "Variable {} already exists", + path + ))); + } + return Ok(()); +} + +async fn create_variable( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Query(AlreadyEncrypted { already_encrypted }): Query, + Json(variable): Json, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("variables:write:{}", variable.path))?; + if *CLOUD_HOSTED { + let nb_variables = sqlx::query_scalar!( + "SELECT COUNT(*) FROM variable WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + if nb_variables.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of variables (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await; + + check_path_conflict(&db, &w_id, &variable.path).await?; + let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { + // Use secret backend for encryption (supports both DB and Vault) + store_secret_value(&db, &w_id, &variable.path, &variable.value).await? + } else { + variable.value + }; + + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "INSERT INTO variable + (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + &w_id, + variable.path, + value, + variable.is_secret, + variable.description, + variable.account, + variable.is_oauth.unwrap_or(false), + variable.expires_at + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "variables.create", + ActionKind::Create, + &w_id, + Some(&variable.path), + None, + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: variable.path.clone(), parent_path: None }, + Some(format!("Variable '{}' created", variable.path.clone())), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateVariable { workspace: w_id, path: variable.path.clone() }, + ); + + Ok(( + StatusCode::CREATED, + format!("variable {} created", variable.path), + )) +} + +async fn encrypt_value( + Extension(db): Extension, + Path(w_id): Path, + Json(variable): Json, +) -> Result { + let mc = build_crypt(&db, &w_id).await?; + let value = encrypt(&mc, &variable); + + Ok(value) +} + +async fn delete_variable( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + + check_scopes(&authed, || format!("variables:write:{}", path))?; + + // Check if variable is a secret before deleting (for Vault cleanup) + let is_secret = sqlx::query_scalar!( + "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + // If variable was a secret, also delete from Vault backend (if configured) + if is_secret { + delete_secret_from_backend(&db, &w_id, path).await?; + } + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: Some(path.to_string()) }, + Some(format!("Variable '{}' deleted", path)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() }, + ); + + Ok(format!("variable {} deleted", path)) +} + +async fn delete_variables_bulk( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult> { + for path in &request.paths { + check_scopes(&authed, || format!("variables:write:{}", path))?; + } + + // Query which paths are secrets before deletion (for Vault cleanup) + let secret_paths: Vec = sqlx::query_scalar!( + "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + &request.paths, + &w_id + ) + .fetch_all(&db) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let deleted_paths = sqlx::query_scalar!( + "DELETE FROM variable WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", + &request.paths, + w_id + ) + .fetch_all(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2", + &deleted_paths, + w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "variables.delete_bulk", + ActionKind::Delete, + &w_id, + Some(&deleted_paths.join(", ")), + None, + ) + .await?; + + tx.commit().await?; + + // Delete secrets from Vault backend (if configured) + for path in &secret_paths { + if deleted_paths.contains(path) { + delete_secret_from_backend(&db, &w_id, path).await?; + } + } + + try_join_all(deleted_paths.iter().map(|path| { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { + path: path.to_string(), + parent_path: Some(path.to_string()), + }, + Some(format!("Variable '{}' deleted", path)), + true, + None, + ) + })) + .await?; + + for path in &deleted_paths { + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteVariable { workspace: w_id.clone(), path: path.to_owned() }, + ); + } + + Ok(Json(deleted_paths)) +} + +#[derive(Deserialize)] +struct EditVariable { + path: Option, + value: Option, + is_secret: Option, + description: Option, + account: Option, +} + +#[derive(Deserialize)] +struct AlreadyEncrypted { + already_encrypted: Option, +} + +async fn update_variable( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(AlreadyEncrypted { already_encrypted }): Query, + Json(ns): Json, +) -> Result { + use sql_builder::prelude::*; + + let path = path.to_path(); + check_scopes(&authed, || format!("variables:write:{}", path))?; + let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; + + let mut sqlb = SqlBuilder::update_table("variable"); + sqlb.and_where_eq("path", "?".bind(&path)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + + if let Some(npath) = &ns.path { + sqlb.set_str("path", npath); + } + let ns_value_is_none = ns.value.is_none(); + // Determine the target path for storing secrets (use new path if provided) + let target_path = ns.path.as_deref().unwrap_or(path); + if let Some(nvalue) = ns.value.clone() { + let is_secret = if ns.is_secret.is_some() { + ns.is_secret.unwrap() + } else { + sqlx::query_scalar!( + "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false) + }; + + let value = if is_secret && !already_encrypted.unwrap_or(false) { + // Use secret backend for encryption (supports both DB and Vault) + // Store at target_path (new path if renaming, otherwise current path) + store_secret_value(&db, &w_id, target_path, &nvalue).await? + } else { + nvalue + }; + sqlb.set_str("value", &value); + } + + if let Some(desc) = ns.description { + sqlb.set_str("description", &desc); + } + + if let Some(account_id) = ns.account { + sqlb.set_str("account", account_id); + } + + if let Some(nbool) = ns.is_secret { + let old_secret = sqlx::query_scalar!( + "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if old_secret != nbool && ns_value_is_none { + return Err(Error::BadRequest( + "cannot change is_secret without updating value too".to_string(), + )); + } + sqlb.set_str("is_secret", nbool); + } + sqlb.returning("path"); + + // Get old account_id if we're updating the account field + let old_account_id = if ns.account.is_some() { + sqlx::query_scalar!( + "SELECT account FROM variable WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + } else { + None + }; + + let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + + if let Some(npath) = ns.path.clone() { + if npath != path { + check_path_conflict(&db, &w_id, &npath).await?; + require_owner_of_path(&authed, path)?; + + // Handle Vault secret rename if the variable is a secret stored in Vault + let current_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = current_var { + if var.is_secret && is_vault_stored_value(&var.value) { + if ns.value.is_some() { + // New value was provided and already stored at new path + // Just delete the old secret from Vault + delete_secret_from_backend(&db, &w_id, path).await?; + } else { + // No new value - rename the secret in Vault + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlb.set_str("value", &new_value); + } + } + } + } + + let mut v = sqlx::query_scalar!( + "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + if let Some(old_v) = v { + v = Some(replace_path( + old_v, + &format!("$var:{path}"), + &format!("$var:{npath}"), + )) + } + + sqlx::query!( + "UPDATE resource SET path = $1, value = $2, edited_at = now() WHERE path = $3 AND workspace_id = $4", + npath, + v, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + + let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; + + let npath = not_found_if_none(npath_o, "Variable", path)?; + + audit_log( + &mut *tx, + &authed, + "variables.update", + ActionKind::Update, + &w_id, + Some(path), + None, + ) + .await?; + + // Clean up old account if it's no longer referenced and different from new account + if let Some(old_acc_id) = old_account_id { + if ns.account.is_some() && ns.account != Some(old_acc_id) { + // Check if old account is still referenced by other variables or resources + let account_still_used = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM variable WHERE account = $1 AND workspace_id = $2)", + old_acc_id, + &w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(true); + + if !account_still_used { + // Delete the orphaned account + sqlx::query!( + "DELETE FROM account WHERE id = $1 AND workspace_id = $2", + old_acc_id, + &w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + + tx.commit().await?; + + // Detect if this was a rename operation + let old_path_if_renamed = if npath != path { Some(path) } else { None }; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: npath.clone(), parent_path: Some(path.to_string()) }, + None, + true, + old_path_if_renamed, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateVariable { + workspace: w_id, + old_path: path.to_owned(), + new_path: npath.clone(), + }, + ); + + Ok(format!("variable {} updated (npath: {:?})", path, npath)) +} + +fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { + match v { + Value::Object(v) => Value::Object( + v.into_iter() + .map(|(k, v)| (k, replace_path(v, path, npath))) + .collect(), + ), + Value::Array(arr) => Value::Array( + arr.into_iter() + .map(|v| replace_path(v, path, npath)) + .collect(), + ), + Value::String(s) if s == path => Value::String(npath.to_owned()), + _ => v, + } +} + +pub async fn get_value_internal<'a>( + db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, + w_id: &str, + path: &str, + allow_cache: bool, +) -> Result { + if allow_cache { + if let Some(cached_variable) = get_cached_variable(&w_id, &path) { + return Ok(cached_variable); + } + } + + let mut tx = db_with_opt_authed.begin().await?; + let variable_o = sqlx::query!( + "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable + LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id + ) + .fetch_optional(&mut *tx) + .warn_after_seconds(5) + .await?; + drop(tx); + + let variable = if let Some(variable) = variable_o { + variable + } else { + explain_variable_perm_error(path, w_id, &db_with_opt_authed.db()).await?; + unreachable!() + }; + + let r = if variable.is_secret { + // let audit_author = + let mut tx = db_with_opt_authed.db().begin().await?; + audit_log( + &mut *tx, + db_with_opt_authed, + "variables.decrypt_secret", + ActionKind::Execute, + &w_id, + Some(&variable.path), + None, + ) + .await?; + tx.commit().await?; + + let value = variable.value; + if variable.is_expired.unwrap_or(false) && variable.account.is_some() { + #[cfg(feature = "oauth2")] + { + let db = db_with_opt_authed.db(); + let refresh_tx = db + .begin() + .await + .map_err(|e| Error::InternalErr(e.to_string()))?; + crate::oauth_refresh_oss::_refresh_token( + refresh_tx, + &variable.path, + &w_id, + variable.account.unwrap(), + db, + ) + .await? + } + #[cfg(not(feature = "oauth2"))] + return Err(Error::internal_err("Require oauth2 feature".to_string())); + } else if !value.is_empty() { + // Use secret backend for decryption (supports both DB and Vault) + get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? + } else { + "".to_string() + } + } else { + variable.value + }; + + // Cache the result when explicitly allowed and caching appropriate + if allow_cache { + cache_variable(&w_id, &path, db_with_opt_authed.email(), r.clone()); + } + + Ok(r) +} diff --git a/backend/windmill-trigger-email/Cargo.toml b/backend/windmill-trigger-email/Cargo.toml new file mode 100644 index 0000000000..0ed00fa2dd --- /dev/null +++ b/backend/windmill-trigger-email/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "windmill-trigger-email" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_email" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tracing.workspace = true +async-trait.workspace = true +anyhow.workspace = true +base64.workspace = true +regex.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api/src/triggers/email/handler_oss.rs b/backend/windmill-trigger-email/src/handler_oss.rs similarity index 90% rename from backend/windmill-api/src/triggers/email/handler_oss.rs rename to backend/windmill-trigger-email/src/handler_oss.rs index 9a688c2cf1..c2e450d052 100644 --- a/backend/windmill-api/src/triggers/email/handler_oss.rs +++ b/backend/windmill-trigger-email/src/handler_oss.rs @@ -1,5 +1,5 @@ #[cfg(not(feature = "private"))] -use crate::triggers::TriggerData; +use windmill_trigger::TriggerData; #[allow(unused)] #[cfg(feature = "private")] @@ -8,14 +8,15 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::EmailTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::TriggerCrud, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + db::DB, + error::{Error, Result}, + }, windmill_git_sync::DeployedObject, + windmill_trigger::TriggerCrud, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-email/src/lib.rs b/backend/windmill-trigger-email/src/lib.rs new file mode 100644 index 0000000000..836af77fa0 --- /dev/null +++ b/backend/windmill-trigger-email/src/lib.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct EmailTrigger; diff --git a/backend/windmill-trigger-gcp/Cargo.toml b/backend/windmill-trigger-gcp/Cargo.toml new file mode 100644 index 0000000000..1b106171c6 --- /dev/null +++ b/backend/windmill-trigger-gcp/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "windmill-trigger-gcp" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_gcp" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["gcp_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +google-cloud-pubsub.workspace = true +google-cloud-googleapis.workspace = true +tonic.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-util.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +base64.workspace = true +bytes.workspace = true +http.workspace = true +jsonwebtoken.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true +reqwest.workspace = true +tower-http.workspace = true +chrono.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-trigger-gcp/src/handler_oss.rs similarity index 91% rename from backend/windmill-api/src/triggers/gcp/handler_oss.rs rename to backend/windmill-trigger-gcp/src/handler_oss.rs index b75a9721eb..0b9de33ce6 100644 --- a/backend/windmill-api/src/triggers/gcp/handler_oss.rs +++ b/backend/windmill-trigger-gcp/src/handler_oss.rs @@ -5,14 +5,15 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::GcpTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{TriggerCrud, TriggerData}, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + error::{Error, Result}, + DB, + }, windmill_git_sync::DeployedObject, + windmill_trigger::{TriggerCrud, TriggerData}, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-gcp/src/lib.rs b/backend/windmill-trigger-gcp/src/lib.rs new file mode 100644 index 0000000000..ae52ce1eb8 --- /dev/null +++ b/backend/windmill-trigger-gcp/src/lib.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Clone, Copy)] +pub struct GcpTrigger; diff --git a/backend/windmill-api/src/triggers/gcp/listener_oss.rs b/backend/windmill-trigger-gcp/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/gcp/listener_oss.rs rename to backend/windmill-trigger-gcp/src/listener_oss.rs index a14168968a..b71cf305a1 100644 --- a/backend/windmill-api/src/triggers/gcp/listener_oss.rs +++ b/backend/windmill-trigger-gcp/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::GcpTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for GcpTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Gcp; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for GcpTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for GcpTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-http/Cargo.toml b/backend/windmill-trigger-http/Cargo.toml new file mode 100644 index 0000000000..43420793cf --- /dev/null +++ b/backend/windmill-trigger-http/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "windmill-trigger-http" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_http" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-trigger.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +http.workspace = true +tracing.workspace = true +tokio.workspace = true +async-trait.workspace = true +lazy_static.workspace = true +quick_cache.workspace = true +regex.workspace = true +matchit.workspace = true +hyper.workspace = true +futures.workspace = true + +# http_trigger_auth deps +base64.workspace = true +hmac.workspace = true +sha1.workspace = true +sha2.workspace = true +constant_time_eq.workspace = true +itertools.workspace = true +thiserror.workspace = true +anyhow.workspace = true +hex.workspace = true diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs new file mode 100644 index 0000000000..43d75f7368 --- /dev/null +++ b/backend/windmill-trigger-http/src/handler.rs @@ -0,0 +1,583 @@ +use super::{ + validate_authentication_method, HttpConfig, HttpConfigRequest, HttpMethod, HttpTrigger, + RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, +}; +use axum::{async_trait, extract::Path, routing::post, Extension, Json, Router}; +use http::StatusCode; +use sqlx::PgConnection; +use std::collections::HashSet; +use windmill_api_auth::ApiAuthed; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + utils::require_admin, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; + +pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { + sqlx::query!("SELECT nextval('http_trigger_version_seq')") + .fetch_one(tx) + .await?; + Ok(()) +} + +pub fn generate_route_path_key(route_path: &str) -> String { + ROUTE_PATH_KEY_RE + .replace_all(route_path, "${1}${2}key") + .to_string() +} + +pub async fn route_path_key_exists( + route_path_key: &str, + http_method: &HttpMethod, + w_id: &str, + trigger_path: Option<&str>, + workspaced_route: Option, + db: &DB, +) -> Result { + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + route_path_key = $1 + AND workspace_id = $2 + AND http_method = $3 + AND ($4::TEXT IS NULL OR path != $4) + ) + "#, + &route_path_key, + w_id, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + } else { + let route_path_key = match workspaced_route { + Some(true) => { + std::borrow::Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))) + } + _ => std::borrow::Cow::Borrowed(route_path_key), + }; + + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) + OR (workspaced_route IS FALSE AND route_path_key = $1)) + AND http_method = $2 + AND ($3::TEXT IS NULL OR path != $3) + ) + "#, + &route_path_key, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; + + Ok(exists) +} + +pub async fn exists_route( + Extension(db): Extension, + Path(w_id): Path, + Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< + RouteExists, + >, +) -> Result> { + let route_path_key = generate_route_path_key(&route_path); + + let exists = route_path_key_exists( + &route_path_key, + &http_method, + &w_id, + trigger_path.as_deref(), + workspaced_route, + &db, + ) + .await?; + + Ok(Json(exists)) +} + +fn check_no_duplicates( + new_http_triggers: &[TriggerData], + route_path_key: &[String], +) -> Result<()> { + let mut seen = HashSet::with_capacity(new_http_triggers.len()); + + for (i, trigger) in new_http_triggers.iter().enumerate() { + if !seen.insert(( + &route_path_key[i], + trigger.config.http_method, + trigger.config.workspaced_route, + )) { + return Err(Error::BadRequest(format!( + "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", + &trigger.config.route_path + ))); + } + } + + Ok(()) +} + +pub async fn insert_new_trigger_into_db( + authed: &ApiAuthed, + tx: &mut PgConnection, + w_id: &str, + trigger: &TriggerData, + route_path_key: &str, +) -> Result<()> { + require_admin(authed.is_admin, &authed.username)?; + + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + INSERT INTO http_trigger ( + workspace_id, + path, + route_path, + route_path_key, + workspaced_route, + authentication_resource_path, + wrap_body, + raw_string, + script_path, + summary, + description, + is_flow, + mode, + request_type, + authentication_method, + http_method, + static_asset_config, + edited_by, + email, + edited_at, + is_static_website, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 + ) + "#, + w_id, + trigger.base.path, + trigger.config.route_path, + route_path_key, + trigger.config.workspaced_route.unwrap_or(false), + trigger.config.authentication_resource_path, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), + trigger.base.script_path, + trigger.config.summary, + trigger.config.description, + trigger.base.is_flow, + trigger.base.mode() as _, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn create_many_http_triggers( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_triggers): Json>>, +) -> Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let handler = HttpTrigger; + + let error_wrapper = |route_path: &str, error: Error| -> Error { + anyhow::anyhow!( + "Error occurred for HTTP route at route path: {}, error: {}", + route_path, + error + ) + .into() + }; + + let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); + + for new_http_trigger in new_http_triggers.iter() { + handler + .validate_new(&db, &w_id, &new_http_trigger.config) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + let route_path_key = + check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; + + route_path_keys.push(route_path_key.clone()); + } + + check_no_duplicates(&new_http_triggers, &route_path_keys)?; + + let mut tx = user_db.begin(&authed).await?; + + for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { + insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + audit_log( + &mut *tx, + &authed, + "http_trigger.create", + ActionKind::Create, + &w_id, + Some(&new_http_trigger.base.path), + None, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + + increase_trigger_version(&mut tx) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + } + + tx.commit().await?; + + for http_trigger in new_http_triggers.into_iter() { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, + Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), + true, + None, + ) + .await + .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; + } + Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) +} + +async fn check_if_route_exist( + db: &DB, + config: &HttpConfigRequest, + workspace_id: &str, + trigger_path: Option<&str>, +) -> Result { + let route_path_key = generate_route_path_key(&config.route_path); + + let exists = route_path_key_exists( + &route_path_key, + &config.http_method, + workspace_id, + trigger_path, + config.workspaced_route, + db, + ) + .await?; + + if exists { + return Err(Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + Ok(route_path_key) +} + +#[async_trait] +impl TriggerCrud for HttpTrigger { + type TriggerConfig = HttpConfig; + type Trigger = Trigger; + type TriggerConfigRequest = HttpConfigRequest; + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = "http_trigger"; + const TRIGGER_TYPE: &'static str = "http"; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/http_triggers"; + const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; + const IS_ALLOWED_ON_CLOUD: bool = true; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "route_path", + "route_path_key", + "request_type", + "authentication_method", + "http_method", + "summary", + "description", + "static_asset_config", + "is_static_website", + "authentication_resource_path", + "workspaced_route", + "wrap_body", + "raw_string", + ]; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::HttpTrigger { path } + } + + fn additional_routes(&self) -> Router { + Router::new() + .route("/create_many", post(create_many_http_triggers)) + .route("/route_exists", post(exists_route)) + } + + async fn validate_new( + &self, + _db: &DB, + _workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + validate_authentication_method(new.authentication_method, new.raw_string)?; + + Ok(()) + } + + async fn validate_edit( + &self, + _db: &DB, + _workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + validate_authentication_method(edit.authentication_method, edit.raw_string)?; + + Ok(()) + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; + + insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + if authed.is_admin { + if trigger.config.route_path.is_empty() { + return Err(Error::BadRequest("route_path is required".to_string())); + }; + + let route_path = &trigger.config.route_path; + if !VALID_ROUTE_PATH_RE.is_match(route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + let route_path_key = + check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; + + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + route_path = $1, + route_path_key = $2, + workspaced_route = $3, + wrap_body = $4, + raw_string = $5, + authentication_resource_path = $6, + script_path = $7, + path = $8, + is_flow = $9, + mode = $10, + http_method = $11, + static_asset_config = $12, + edited_by = $13, + email = $14, + request_type = $15, + authentication_method = $16, + summary = $17, + description = $18, + edited_at = now(), + is_static_website = $19, + error_handler_path = $20, + error_handler_args = $21, + retry = $22 + WHERE + workspace_id = $23 AND + path = $24 + "#, + route_path, + &route_path_key, + trigger.config.workspaced_route, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.base.mode() as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } else { + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + wrap_body = $1, + raw_string = $2, + authentication_resource_path = $3, + script_path = $4, + path = $5, + is_flow = $6, + mode = $7, + http_method = $8, + static_asset_config = $9, + edited_by = $10, + email = $11, + request_type = $12, + authentication_method = $13, + summary = $14, + description = $15, + edited_at = now(), + is_static_website = $16, + error_handler_path = $17, + error_handler_args = $18, + retry = $19 + WHERE + workspace_id = $20 AND + path = $21 + "#, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.base.mode() as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn set_trigger_mode_extra_action(&self, tx: &mut PgConnection) -> Result<()> { + increase_trigger_version(tx).await + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + increase_trigger_version(tx).await?; + + Ok(deleted > 0) + } +} diff --git a/backend/windmill-trigger-http/src/http_trigger_auth.rs b/backend/windmill-trigger-http/src/http_trigger_auth.rs new file mode 100644 index 0000000000..13462f9156 --- /dev/null +++ b/backend/windmill-trigger-http/src/http_trigger_auth.rs @@ -0,0 +1,1774 @@ +use axum::response::{IntoResponse, Response}; +use base64::{ + prelude::{BASE64_STANDARD, BASE64_URL_SAFE}, + Engine, +}; +use hmac::{Hmac, Mac}; +use http::{header, HeaderMap, HeaderValue, StatusCode}; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sha1::Sha1; +use sha2::{Sha256, Sha512}; +use std::{borrow::Cow, collections::HashMap}; + +pub type HmacSha256 = Hmac; +pub type HmacSha512 = Hmac; +pub type HmacSha1 = Hmac; + +mod github { + use super::*; + pub struct Github; + + impl WebhookHandler for Github { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let github_secret_header = headers.try_get_webhook_header("X-Hub-Signature-256")?; + + let authentication_data = SignatureAuthenticationData::new( + Cow::Borrowed(raw_payload), + github_secret_header, + Some("sha256="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + ); + + Ok(authentication_data) + } + } +} + +mod slack { + use super::*; + pub struct Slack; + + impl WebhookHandler for Slack { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let slack_secret_signature = headers.try_get_webhook_header("X-Slack-Signature")?; + let slack_timestamp_header = + headers.try_get_webhook_header("X-Slack-Request-Timestamp")?; + let signed_payload = format!("v0:{}:{}", slack_timestamp_header, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(signed_payload), + slack_secret_signature, + Some("v0="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +mod stripe { + use super::*; + + pub struct Stripe; + + impl WebhookHandler for Stripe { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let stripe_signature_header = headers.try_get_webhook_header("STRIPE-SIGNATURE")?; + + let stripe_signature = parse_signature(stripe_signature_header, (",", "=")); + + let timestamp = *stripe_signature + .get("t") + .ok_or(AuthenticationError::InvalidTimestamp)?; + let v1 = *stripe_signature + .get("v1") + .ok_or(AuthenticationError::InvalidSignature)?; + + let signed_payload = format!("{}.{}", timestamp, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(signed_payload), + v1, + None, + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +mod tiktok { + use super::*; + + pub struct TikTok; + + impl WebhookHandler for TikTok { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let tiktok_secret_signature = headers.try_get_webhook_header("TikTok-Signature")?; + + let stripe_signature = parse_signature(tiktok_secret_signature, (",", "=")); + + let timestamp = *stripe_signature + .get("t") + .ok_or(AuthenticationError::InvalidTimestamp)?; + let s = *stripe_signature + .get("s") + .ok_or(AuthenticationError::InvalidSignature)?; + + let signed_payload = format!("{}.{}", timestamp, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(signed_payload), + s, + None, + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +mod twitch { + use super::*; + use http::header; + use serde_json::value::RawValue; + #[derive(Debug, Deserialize)] + struct TwitchCrcBody { + challenge: String, + #[allow(unused)] + subscription: Box, + } + + pub struct Twitch; + + impl WebhookHandler for Twitch { + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let twitch_secret_signature = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Signature")?; + let twitch_message_id_header = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Id")?; + let twitch_timestamp_header = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Timestamp")?; + + let message = format!( + "{}{}{}", + twitch_message_id_header, twitch_timestamp_header, raw_payload + ); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(message), + twitch_secret_signature, + Some("sha256="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + + fn handle_challenge_request<'header>( + &self, + headers: &'header HeaderMap, + signature_config_data: &SignatureConfigData, + raw_payload: &str, + ) -> Result, AuthenticationError> { + let authentication_data = self.get_hmac_authentication_data(headers, raw_payload)?; + verify_hmac_signature(authentication_data, &signature_config_data.secret_key)?; + + let twitch_eventsub_message_type = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Type")?; + + if twitch_eventsub_message_type != "webhook_callback_verification" { + return Ok(None); + } + let twitch_crc_body = + serde_json::from_str::(raw_payload).map_err(|e| { + AuthenticationError::InvalidChallengeResponse(format!( + "Twitch :{}", + e.to_string() + )) + })?; + + let response = ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/plain")], + twitch_crc_body.challenge.to_string(), + ); + + Ok(Some(response.into_response())) + } + } +} + +mod zoom { + use axum::Json; + + use super::*; + + #[derive(Debug, Deserialize)] + struct ZoomPayload { + #[serde(rename = "plainToken")] + plain_token: String, + } + + #[derive(Debug, Deserialize)] + #[allow(unused)] + struct ZoomChallengeResponse { + payload: ZoomPayload, + event_ts: u64, + event: String, + } + + pub struct Zoom; + + impl WebhookHandler for Zoom { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + signature_config_data: &SignatureConfigData, + raw_payload: &str, + ) -> Result, AuthenticationError> { + let Ok(zoom_request_body) = serde_json::from_str::(raw_payload) + else { + return Ok(None); + }; + + if zoom_request_body.event != "endpoint.url_validation" { + return Ok(None); + } + + let hmac_signature = calculate_hmac_signature( + HmacAlgorithm::Sha256, + &signature_config_data.secret_key, + &zoom_request_body.payload.plain_token, + ); + + let encoded_hmac_signature = encode_hmac_signature(Encoding::Hex, &hmac_signature); + + let response = ( + StatusCode::OK, + Json(json!({ + "plainToken": zoom_request_body.payload.plain_token, + "encryptedToken": encoded_hmac_signature + })), + ); + + Ok(Some(response.into_response())) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let zoom_signature_header = headers.try_get_webhook_header("x-zm-signature")?; + let zoom_timestamp_header = headers.try_get_webhook_header("x-zm-request-timestamp")?; + + let message = format!("v0:{}:{}", zoom_timestamp_header, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(message), + zoom_signature_header, + Some("v0="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +use constant_time_eq::constant_time_eq; +use github::Github; +use slack::Slack; +use stripe::Stripe; +use tiktok::TikTok; +use twitch::Twitch; +use zoom::Zoom; + +#[derive(Debug)] +pub struct SignatureAuthenticationDetails { + pub algorithm_to_use: HmacAlgorithm, + pub header_key_encoding: Encoding, +} + +impl SignatureAuthenticationDetails { + #[inline] + fn new(algorithm_to_use: HmacAlgorithm, header_key_encoding: Encoding) -> Self { + Self { algorithm_to_use, header_key_encoding } + } +} + +fn parse_signature<'header>( + signature: &'header str, + splitters: (&str, &str), +) -> HashMap<&'header str, &'header str> { + let headers: HashMap<&str, &str> = signature + .split(splitters.0) + .map(|header| { + let mut key_and_value = header.split(splitters.1); + let key = key_and_value.next(); + let value = key_and_value.next(); + (key, value) + }) + .filter_map(|(key, value)| match (key, value) { + (Some(key), Some(value)) => Some((key, value)), + _ => None, + }) + .collect(); + headers +} + +#[derive(Debug)] +pub struct SignatureAuthenticationData<'payload, 'header, 'prefix> { + pub signed_payload: Cow<'payload, str>, + pub header_key_value: &'header str, + pub signature_prefix: Option<&'prefix str>, + pub config: SignatureAuthenticationDetails, +} + +impl<'payload, 'header, 'prefix> SignatureAuthenticationData<'payload, 'header, 'prefix> { + pub fn new( + signed_payload: Cow<'payload, str>, + header_key_value: &'header str, + signature_prefix: Option<&'prefix str>, + config: SignatureAuthenticationDetails, + ) -> Self { + Self { signed_payload, header_key_value, signature_prefix, config } + } +} + +pub trait WebhookHandler { + fn handle_challenge_request<'header>( + &self, + headers: &'header HeaderMap, + signature_config_data: &SignatureConfigData, + raw_payload: &str, + ) -> Result, AuthenticationError>; + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError>; +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HmacAlgorithm { + Sha1, + Sha256, + Sha512, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Encoding { + Base64, + Base64Uri, + Hex, +} +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SignatureAuthenticationMethod { + algorithm: HmacAlgorithm, + encoding: Encoding, + signature_header_name: String, + signature_prefix: Option, +} + +pub struct SignatureConfigData<'config> { + secret_key: &'config str, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SignatureAuthentication { + signature_provider: WebhookType, + secret_key: String, + authentication_config: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct BasicAuthAuthentication { + pub username: String, + pub password: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ApiKeyAuthentication { + pub api_key_header: String, + pub api_key_secret: String, +} + +#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[non_exhaustive] +pub enum WebhookType { + Github, + Slack, + Stripe, + TikTok, + Twitch, + Zoom, + Custom, +} + +impl WebhookType { + pub fn get_webhook_handler(&self) -> Option<&'static dyn WebhookHandler> { + let handler: &'static dyn WebhookHandler = match *self { + WebhookType::Github => &Github, + WebhookType::Slack => &Slack, + WebhookType::Stripe => &Stripe, + WebhookType::TikTok => &TikTok, + WebhookType::Twitch => &Twitch, + WebhookType::Zoom => &Zoom, + WebhookType::Custom => return None, + }; + Some(handler) + } +} + +trait TryGetWebhookHeader { + fn try_get_webhook_header<'header>( + &'header self, + header_name: &str, + ) -> Result<&'header str, AuthenticationError>; +} + +impl TryGetWebhookHeader for HeaderMap { + fn try_get_webhook_header<'header>( + &'header self, + header_name: &str, + ) -> Result<&'header str, AuthenticationError> { + let Some(signature_header) = self.get(header_name) else { + return Err(AuthenticationError::MissingHeader(header_name.to_string())); + }; + let Some(signature_header) = signature_header.to_str().ok() else { + return Err(AuthenticationError::InvalidHeader(header_name.to_string())); + }; + + Ok(signature_header) + } +} + +pub fn calculate_hmac_signature(algorithm: HmacAlgorithm, secret: &str, payload: &str) -> Vec { + match algorithm { + HmacAlgorithm::Sha1 => { + let mut mac = + HmacSha1::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size"); + mac.update(payload.as_bytes()); + mac.finalize().into_bytes().to_vec() + } + HmacAlgorithm::Sha256 => { + let mut mac = HmacSha256::new_from_slice(secret.as_bytes()) + .expect("HMAC can take key of any size"); + mac.update(payload.as_bytes()); + mac.finalize().into_bytes().to_vec() + } + HmacAlgorithm::Sha512 => { + let mut mac = HmacSha512::new_from_slice(secret.as_bytes()) + .expect("HMAC can take key of any size"); + mac.update(payload.as_bytes()); + mac.finalize().into_bytes().to_vec() + } + } +} + +pub fn encode_hmac_signature(encoding: Encoding, hmac_signature: &[u8]) -> String { + match encoding { + Encoding::Hex => hex::encode(hmac_signature), + Encoding::Base64 => BASE64_STANDARD.encode(hmac_signature), + Encoding::Base64Uri => BASE64_URL_SAFE.encode(hmac_signature), + } +} + +pub fn verify_hmac_signature( + authentication_data: SignatureAuthenticationData, + webhook_signing_secret: &str, +) -> Result<(), AuthenticationError> { + let hmac_signature = calculate_hmac_signature( + authentication_data.config.algorithm_to_use, + &webhook_signing_secret, + &authentication_data.signed_payload, + ); + + let encoded_signature = encode_hmac_signature( + authentication_data.config.header_key_encoding, + &hmac_signature, + ); + + let final_expected_signature = + if let Some(signature_prefix) = authentication_data.signature_prefix { + format!("{}{}", signature_prefix, encoded_signature) + } else { + encoded_signature + }; + + if !constant_time_eq( + final_expected_signature.as_bytes(), + authentication_data.header_key_value.as_bytes(), + ) { + return Err(AuthenticationError::InvalidSignature); + } + + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(untagged)] +pub enum AuthenticationMethod { + Signature(SignatureAuthentication), + BasicAuth(BasicAuthAuthentication), + ApiKey(ApiKeyAuthentication), +} + +impl AuthenticationMethod { + pub fn authenticate_http_request( + &self, + headers: &HeaderMap, + raw_payload: Option<&String>, + ) -> Result, AuthenticationError> { + match self { + AuthenticationMethod::Signature(SignatureAuthentication { + secret_key, + authentication_config, + signature_provider, + }) => { + let raw_payload = raw_payload.ok_or(AuthenticationError::InvalidPayload)?; + let config_data = SignatureConfigData { secret_key: &secret_key }; + let handler = signature_provider.get_webhook_handler(); + let challenge_response = handler + .map(|handler| { + handler.handle_challenge_request(headers, &config_data, raw_payload) + }) + .transpose()? + .flatten(); + + if let Some(challenge_response) = challenge_response { + return Ok(Some(challenge_response)); + } + + let authentication_data = match handler { + Some(handler) => handler.get_hmac_authentication_data(headers, raw_payload)?, + None => { + let authentication_config = authentication_config + .as_ref() + .ok_or(AuthenticationError::InvalidCustomConfig)?; + let signature_header_value = headers + .try_get_webhook_header(&authentication_config.signature_header_name)?; + SignatureAuthenticationData::new( + Cow::Borrowed(raw_payload), + signature_header_value, + authentication_config.signature_prefix.as_deref(), + SignatureAuthenticationDetails::new( + authentication_config.algorithm, + authentication_config.encoding, + ), + ) + } + }; + + verify_hmac_signature(authentication_data, &secret_key)?; + } + AuthenticationMethod::ApiKey(ApiKeyAuthentication { + api_key_header, + api_key_secret, + }) => { + let api_key_to_cmp = headers + .try_get_webhook_header(&api_key_header) + .map_err(|_| AuthenticationError::InvalidApiKey)?; + if api_key_to_cmp != api_key_secret { + return Err(AuthenticationError::InvalidApiKey); + } + } + AuthenticationMethod::BasicAuth(BasicAuthAuthentication { username, password }) => { + let mut credentials_store = headers + .try_get_webhook_header("Authorization") + .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)? + .split(' '); + + let _ = credentials_store + .next() + .filter(|r#type| *r#type == "Basic") + .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials_as_base64 = credentials_store + .next() + .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials_from_base64_as_bytes = BASE64_STANDARD + .decode(credentials_as_base64.as_bytes()) + .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials_separated_with_colon = + String::from_utf8(credentials_from_base64_as_bytes) + .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials = credentials_separated_with_colon.split(':').collect_vec(); + + if credentials.len() != 2 { + return Err(AuthenticationError::UnauthorizedBasicHttpAuth); + } + + if credentials.get(0).unwrap() != username + || credentials.get(1).unwrap() != password + { + return Err(AuthenticationError::UnauthorizedBasicHttpAuth); + } + } + } + + Ok(None) + } +} + +#[derive(thiserror::Error, Debug)] +#[allow(unused)] +pub enum AuthenticationError { + #[error("failed to parse timestamp")] + InvalidTimestamp, + + #[error("invalid secret")] + InvalidSecret(#[from] base64::DecodeError), + + #[error("invalid header `{0}`")] + InvalidHeader(String), + + #[error("signature timestamp too old")] + TimestampTooOldError, + + #[error("signature timestamp too far in future")] + FutureTimestampError, + + #[error("missing header {0}")] + MissingHeader(String), + + #[error("signature invalid")] + InvalidSignature, + + #[error("payload invalid")] + InvalidPayload, + + #[error("invalid custom config")] + InvalidCustomConfig, + + #[error("invalid auth header: {0}")] + InvalidAuthHeader(String), + + #[error("invalid api key")] + InvalidApiKey, + + #[error("invalid challenge response: {0}")] + InvalidChallengeResponse(String), + + #[error("")] + UnauthorizedBasicHttpAuth, +} + +impl IntoResponse for AuthenticationError { + fn into_response(self) -> Response { + let (status, error_message) = match &self { + AuthenticationError::InvalidTimestamp + | AuthenticationError::InvalidPayload + | AuthenticationError::InvalidHeader(_) + | AuthenticationError::MissingHeader(_) + | AuthenticationError::TimestampTooOldError + | AuthenticationError::FutureTimestampError + | AuthenticationError::InvalidCustomConfig + | AuthenticationError::InvalidChallengeResponse(_) => { + (StatusCode::BAD_REQUEST, self.to_string()) + } + + AuthenticationError::InvalidSecret(_) + | AuthenticationError::InvalidSignature + | AuthenticationError::InvalidAuthHeader(_) => { + (StatusCode::UNAUTHORIZED, self.to_string()) + } + AuthenticationError::UnauthorizedBasicHttpAuth => { + return ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, r#"Basic realm="Restricted Area""#)], + "Unauthorized", + ) + .into_response() + } + AuthenticationError::InvalidApiKey => { + return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response() + } + }; + + let body = json!({ "error": error_message }); + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", HeaderValue::from_static("application/json")); + + (status, headers, body.to_string()).into_response() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // --- calculate_hmac_signature --- + + #[test] + fn test_hmac_sha256_deterministic() { + let sig1 = calculate_hmac_signature(HmacAlgorithm::Sha256, "secret", "payload"); + let sig2 = calculate_hmac_signature(HmacAlgorithm::Sha256, "secret", "payload"); + assert_eq!(sig1, sig2); + } + + #[test] + fn test_hmac_sha256_different_keys() { + let sig1 = calculate_hmac_signature(HmacAlgorithm::Sha256, "key1", "payload"); + let sig2 = calculate_hmac_signature(HmacAlgorithm::Sha256, "key2", "payload"); + assert_ne!(sig1, sig2); + } + + #[test] + fn test_hmac_sha256_different_payloads() { + let sig1 = calculate_hmac_signature(HmacAlgorithm::Sha256, "secret", "payload1"); + let sig2 = calculate_hmac_signature(HmacAlgorithm::Sha256, "secret", "payload2"); + assert_ne!(sig1, sig2); + } + + #[test] + fn test_hmac_sha1_length() { + let sig = calculate_hmac_signature(HmacAlgorithm::Sha1, "secret", "payload"); + assert_eq!(sig.len(), 20); // SHA1 = 160 bits = 20 bytes + } + + #[test] + fn test_hmac_sha256_length() { + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, "secret", "payload"); + assert_eq!(sig.len(), 32); // SHA256 = 256 bits = 32 bytes + } + + #[test] + fn test_hmac_sha512_length() { + let sig = calculate_hmac_signature(HmacAlgorithm::Sha512, "secret", "payload"); + assert_eq!(sig.len(), 64); // SHA512 = 512 bits = 64 bytes + } + + #[test] + fn test_hmac_sha256_empty_payload() { + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, "secret", ""); + assert_eq!(sig.len(), 32); + } + + #[test] + fn test_hmac_sha256_empty_key() { + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, "", "payload"); + assert_eq!(sig.len(), 32); + } + + // --- encode_hmac_signature --- + + #[test] + fn test_encode_hex() { + let bytes = vec![0xde, 0xad, 0xbe, 0xef]; + assert_eq!(encode_hmac_signature(Encoding::Hex, &bytes), "deadbeef"); + } + + #[test] + fn test_encode_base64() { + let bytes = vec![0xde, 0xad, 0xbe, 0xef]; + let encoded = encode_hmac_signature(Encoding::Base64, &bytes); + assert_eq!(BASE64_STANDARD.decode(&encoded).unwrap(), bytes); + } + + #[test] + fn test_encode_base64uri() { + let bytes = vec![0xde, 0xad, 0xbe, 0xef]; + let encoded = encode_hmac_signature(Encoding::Base64Uri, &bytes); + assert_eq!(BASE64_URL_SAFE.decode(&encoded).unwrap(), bytes); + } + + #[test] + fn test_encode_hex_empty() { + assert_eq!(encode_hmac_signature(Encoding::Hex, &[]), ""); + } + + // --- verify_hmac_signature round-trip --- + + fn make_auth_data<'a>( + payload: &'a str, + header_value: &'a str, + prefix: Option<&'a str>, + algorithm: HmacAlgorithm, + encoding: Encoding, + ) -> SignatureAuthenticationData<'a, 'a, 'a> { + SignatureAuthenticationData::new( + Cow::Borrowed(payload), + header_value, + prefix, + SignatureAuthenticationDetails::new(algorithm, encoding), + ) + } + + #[test] + fn test_verify_hmac_sha256_hex_roundtrip() { + let secret = "my_webhook_secret"; + let payload = r#"{"event":"push"}"#; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + + let data = make_auth_data(payload, &encoded, None, HmacAlgorithm::Sha256, Encoding::Hex); + assert!(verify_hmac_signature(data, secret).is_ok()); + } + + #[test] + fn test_verify_hmac_sha256_base64_roundtrip() { + let secret = "my_secret"; + let payload = "test body"; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, payload); + let encoded = encode_hmac_signature(Encoding::Base64, &sig); + + let data = make_auth_data( + payload, + &encoded, + None, + HmacAlgorithm::Sha256, + Encoding::Base64, + ); + assert!(verify_hmac_signature(data, secret).is_ok()); + } + + #[test] + fn test_verify_hmac_sha512_hex_roundtrip() { + let secret = "long_secret_key"; + let payload = "some data"; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha512, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + + let data = make_auth_data(payload, &encoded, None, HmacAlgorithm::Sha512, Encoding::Hex); + assert!(verify_hmac_signature(data, secret).is_ok()); + } + + #[test] + fn test_verify_hmac_sha1_hex_roundtrip() { + let secret = "sha1_key"; + let payload = "data"; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha1, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + + let data = make_auth_data(payload, &encoded, None, HmacAlgorithm::Sha1, Encoding::Hex); + assert!(verify_hmac_signature(data, secret).is_ok()); + } + + #[test] + fn test_verify_with_prefix() { + let secret = "key"; + let payload = "body"; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let header_value = format!("sha256={}", encoded); + + let data = make_auth_data( + payload, + &header_value, + Some("sha256="), + HmacAlgorithm::Sha256, + Encoding::Hex, + ); + assert!(verify_hmac_signature(data, secret).is_ok()); + } + + #[test] + fn test_verify_wrong_signature() { + let data = make_auth_data( + "payload", + "wrong_signature_value", + None, + HmacAlgorithm::Sha256, + Encoding::Hex, + ); + let result = verify_hmac_signature(data, "secret"); + assert!(matches!(result, Err(AuthenticationError::InvalidSignature))); + } + + #[test] + fn test_verify_wrong_key() { + let secret = "correct_key"; + let payload = "body"; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + + let data = make_auth_data(payload, &encoded, None, HmacAlgorithm::Sha256, Encoding::Hex); + let result = verify_hmac_signature(data, "wrong_key"); + assert!(matches!(result, Err(AuthenticationError::InvalidSignature))); + } + + #[test] + fn test_verify_wrong_prefix() { + let secret = "key"; + let payload = "body"; + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let header_value = format!("v0={}", encoded); + + let data = make_auth_data( + payload, + &header_value, + Some("sha256="), + HmacAlgorithm::Sha256, + Encoding::Hex, + ); + assert!(matches!( + verify_hmac_signature(data, secret), + Err(AuthenticationError::InvalidSignature) + )); + } + + // --- parse_signature --- + + #[test] + fn test_parse_signature_stripe_format() { + let header = "t=1234567890,v1=abc123def456"; + let parsed = parse_signature(header, (",", "=")); + assert_eq!(parsed.get("t"), Some(&"1234567890")); + assert_eq!(parsed.get("v1"), Some(&"abc123def456")); + } + + #[test] + fn test_parse_signature_tiktok_format() { + let header = "t=1234567890,s=signaturevalue"; + let parsed = parse_signature(header, (",", "=")); + assert_eq!(parsed.get("t"), Some(&"1234567890")); + assert_eq!(parsed.get("s"), Some(&"signaturevalue")); + } + + #[test] + fn test_parse_signature_single_entry() { + let header = "key=value"; + let parsed = parse_signature(header, (",", "=")); + assert_eq!(parsed.get("key"), Some(&"value")); + assert_eq!(parsed.len(), 1); + } + + #[test] + fn test_parse_signature_empty_string() { + let parsed = parse_signature("", (",", "=")); + assert!(parsed.is_empty() || parsed.len() == 1); + } + + #[test] + fn test_parse_signature_multiple_entries() { + let header = "a=1,b=2,c=3"; + let parsed = parse_signature(header, (",", "=")); + assert_eq!(parsed.len(), 3); + assert_eq!(parsed.get("a"), Some(&"1")); + assert_eq!(parsed.get("b"), Some(&"2")); + assert_eq!(parsed.get("c"), Some(&"3")); + } + + // --- WebhookType serde --- + + #[test] + fn test_webhook_type_serde_roundtrip() { + for wt in [ + WebhookType::Github, + WebhookType::Slack, + WebhookType::Stripe, + WebhookType::TikTok, + WebhookType::Twitch, + WebhookType::Zoom, + WebhookType::Custom, + ] { + let json = serde_json::to_value(wt).unwrap(); + let deserialized: WebhookType = serde_json::from_value(json).unwrap(); + assert_eq!(wt, deserialized); + } + } + + #[test] + fn test_webhook_type_handler_known_providers() { + assert!(WebhookType::Github.get_webhook_handler().is_some()); + assert!(WebhookType::Slack.get_webhook_handler().is_some()); + assert!(WebhookType::Stripe.get_webhook_handler().is_some()); + assert!(WebhookType::TikTok.get_webhook_handler().is_some()); + assert!(WebhookType::Twitch.get_webhook_handler().is_some()); + assert!(WebhookType::Zoom.get_webhook_handler().is_some()); + } + + #[test] + fn test_webhook_type_custom_has_no_handler() { + assert!(WebhookType::Custom.get_webhook_handler().is_none()); + } + + // --- HmacAlgorithm / Encoding serde --- + + #[test] + fn test_hmac_algorithm_serde() { + assert_eq!( + serde_json::to_value(HmacAlgorithm::Sha1).unwrap(), + "sha1" + ); + assert_eq!( + serde_json::to_value(HmacAlgorithm::Sha256).unwrap(), + "sha256" + ); + assert_eq!( + serde_json::to_value(HmacAlgorithm::Sha512).unwrap(), + "sha512" + ); + } + + #[test] + fn test_encoding_serde() { + assert_eq!(serde_json::to_value(Encoding::Hex).unwrap(), "hex"); + assert_eq!(serde_json::to_value(Encoding::Base64).unwrap(), "base64"); + assert_eq!( + serde_json::to_value(Encoding::Base64Uri).unwrap(), + "base64uri" + ); + } + + // --- TryGetWebhookHeader --- + + #[test] + fn test_try_get_header_present() { + let mut headers = HeaderMap::new(); + headers.insert("X-Custom-Header", HeaderValue::from_static("value123")); + assert_eq!( + headers.try_get_webhook_header("X-Custom-Header").unwrap(), + "value123" + ); + } + + #[test] + fn test_try_get_header_missing() { + let headers = HeaderMap::new(); + let result = headers.try_get_webhook_header("X-Missing"); + assert!(matches!(result, Err(AuthenticationError::MissingHeader(_)))); + } + + #[test] + fn test_try_get_header_case_insensitive() { + let mut headers = HeaderMap::new(); + headers.insert("x-hub-signature-256", HeaderValue::from_static("sig")); + assert_eq!( + headers + .try_get_webhook_header("X-Hub-Signature-256") + .unwrap(), + "sig" + ); + } + + // --- GitHub webhook end-to-end --- + + fn github_headers(secret: &str, payload: &str) -> HeaderMap { + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let mut headers = HeaderMap::new(); + headers.insert( + "X-Hub-Signature-256", + HeaderValue::from_str(&format!("sha256={}", encoded)).unwrap(), + ); + headers + } + + #[test] + fn test_github_authenticate_valid() { + let secret = "github_webhook_secret"; + let payload = r#"{"action":"opened","number":1}"#.to_string(); + let headers = github_headers(secret, &payload); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Github, + secret_key: secret.to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_github_authenticate_wrong_secret() { + let payload = r#"{"action":"opened"}"#.to_string(); + let headers = github_headers("correct_secret", &payload); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Github, + secret_key: "wrong_secret".to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_err()); + } + + #[test] + fn test_github_authenticate_missing_header() { + let payload = r#"{"action":"opened"}"#.to_string(); + let headers = HeaderMap::new(); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Github, + secret_key: "secret".to_string(), + authentication_config: None, + }); + assert!(matches!( + method.authenticate_http_request(&headers, Some(&payload)), + Err(AuthenticationError::MissingHeader(_)) + )); + } + + #[test] + fn test_github_authenticate_no_payload() { + let headers = HeaderMap::new(); + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Github, + secret_key: "secret".to_string(), + authentication_config: None, + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::InvalidPayload) + )); + } + + // --- Slack webhook end-to-end --- + + fn slack_headers(secret: &str, payload: &str, timestamp: &str) -> HeaderMap { + let signed_payload = format!("v0:{}:{}", timestamp, payload); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, &signed_payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let mut headers = HeaderMap::new(); + headers.insert( + "X-Slack-Signature", + HeaderValue::from_str(&format!("v0={}", encoded)).unwrap(), + ); + headers.insert( + "X-Slack-Request-Timestamp", + HeaderValue::from_str(timestamp).unwrap(), + ); + headers + } + + #[test] + fn test_slack_authenticate_valid() { + let secret = "slack_signing_secret"; + let payload = "token=xxx&command=%2Ftest".to_string(); + let timestamp = "1531420618"; + let headers = slack_headers(secret, &payload, timestamp); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Slack, + secret_key: secret.to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_slack_authenticate_wrong_timestamp() { + let secret = "slack_secret"; + let payload = "data".to_string(); + let headers = slack_headers(secret, &payload, "1000000000"); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Slack, + secret_key: secret.to_string(), + authentication_config: None, + }); + // Constructed with timestamp "1000000000" but that's valid - it just needs to match + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + // --- Stripe webhook end-to-end --- + + fn stripe_headers(secret: &str, payload: &str, timestamp: &str) -> HeaderMap { + let signed_payload = format!("{}.{}", timestamp, payload); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, &signed_payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let mut headers = HeaderMap::new(); + headers.insert( + "STRIPE-SIGNATURE", + HeaderValue::from_str(&format!("t={},v1={}", timestamp, encoded)).unwrap(), + ); + headers + } + + #[test] + fn test_stripe_authenticate_valid() { + let secret = "whsec_stripe_secret"; + let payload = r#"{"id":"evt_123"}"#.to_string(); + let timestamp = "1614556800"; + let headers = stripe_headers(secret, &payload, timestamp); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Stripe, + secret_key: secret.to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_stripe_authenticate_wrong_secret() { + let payload = r#"{"id":"evt_123"}"#.to_string(); + let headers = stripe_headers("correct", &payload, "12345"); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Stripe, + secret_key: "wrong".to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_err()); + } + + // --- TikTok webhook end-to-end --- + + fn tiktok_headers(secret: &str, payload: &str, timestamp: &str) -> HeaderMap { + let signed_payload = format!("{}.{}", timestamp, payload); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, &signed_payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let mut headers = HeaderMap::new(); + headers.insert( + "TikTok-Signature", + HeaderValue::from_str(&format!("t={},s={}", timestamp, encoded)).unwrap(), + ); + headers + } + + #[test] + fn test_tiktok_authenticate_valid() { + let secret = "tiktok_secret"; + let payload = r#"{"event":"video.upload"}"#.to_string(); + let timestamp = "1700000000"; + let headers = tiktok_headers(secret, &payload, timestamp); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::TikTok, + secret_key: secret.to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + // --- Twitch webhook end-to-end --- + + fn twitch_headers( + secret: &str, + payload: &str, + message_id: &str, + timestamp: &str, + message_type: &str, + ) -> HeaderMap { + let message = format!("{}{}{}", message_id, timestamp, payload); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, &message); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let mut headers = HeaderMap::new(); + headers.insert( + "Twitch-Eventsub-Message-Signature", + HeaderValue::from_str(&format!("sha256={}", encoded)).unwrap(), + ); + headers.insert( + "Twitch-Eventsub-Message-Id", + HeaderValue::from_str(message_id).unwrap(), + ); + headers.insert( + "Twitch-Eventsub-Message-Timestamp", + HeaderValue::from_str(timestamp).unwrap(), + ); + headers.insert( + "Twitch-Eventsub-Message-Type", + HeaderValue::from_str(message_type).unwrap(), + ); + headers + } + + #[test] + fn test_twitch_authenticate_valid_notification() { + let secret = "twitch_secret"; + let payload = r#"{"subscription":{},"event":{"user_id":"123"}}"#.to_string(); + let headers = twitch_headers(secret, &payload, "msg-123", "2024-01-01T00:00:00Z", "notification"); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Twitch, + secret_key: secret.to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_twitch_challenge_response() { + let secret = "twitch_secret"; + let payload = r#"{"challenge":"test_challenge_string","subscription":{"id":"sub-123"}}"#; + let headers = twitch_headers( + secret, + payload, + "msg-456", + "2024-01-01T00:00:00Z", + "webhook_callback_verification", + ); + + let handler = WebhookType::Twitch.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: secret }; + let response = handler + .handle_challenge_request(&headers, &config_data, payload) + .unwrap(); + assert!(response.is_some()); + } + + #[test] + fn test_twitch_non_challenge_returns_none() { + let secret = "twitch_secret"; + let payload = r#"{"subscription":{},"event":{}}"#; + let headers = twitch_headers( + secret, + payload, + "msg-789", + "2024-01-01T00:00:00Z", + "notification", + ); + + let handler = WebhookType::Twitch.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: secret }; + let response = handler + .handle_challenge_request(&headers, &config_data, payload) + .unwrap(); + assert!(response.is_none()); + } + + // --- Zoom webhook end-to-end --- + + fn zoom_headers(secret: &str, payload: &str, timestamp: &str) -> HeaderMap { + let message = format!("v0:{}:{}", timestamp, payload); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, &message); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + let mut headers = HeaderMap::new(); + headers.insert( + "x-zm-signature", + HeaderValue::from_str(&format!("v0={}", encoded)).unwrap(), + ); + headers.insert( + "x-zm-request-timestamp", + HeaderValue::from_str(timestamp).unwrap(), + ); + headers + } + + #[test] + fn test_zoom_authenticate_valid() { + let secret = "zoom_secret"; + let payload = r#"{"event":"meeting.started"}"#.to_string(); + let timestamp = "1700000000"; + let headers = zoom_headers(secret, &payload, timestamp); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Zoom, + secret_key: secret.to_string(), + authentication_config: None, + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_zoom_challenge_response() { + let secret = "zoom_secret"; + let payload = r#"{"event":"endpoint.url_validation","event_ts":1234567890,"payload":{"plainToken":"abc123"}}"#; + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: secret }; + let response = handler + .handle_challenge_request(&HeaderMap::new(), &config_data, payload) + .unwrap(); + assert!(response.is_some()); + } + + #[test] + fn test_zoom_non_challenge_returns_none() { + let payload = r#"{"event":"meeting.started","event_ts":1234567890,"payload":{"plainToken":"abc"}}"#; + + let handler = WebhookType::Zoom.get_webhook_handler().unwrap(); + let config_data = SignatureConfigData { secret_key: "secret" }; + let response = handler + .handle_challenge_request(&HeaderMap::new(), &config_data, payload) + .unwrap(); + assert!(response.is_none()); + } + + // --- Custom webhook end-to-end --- + + #[test] + fn test_custom_signature_authenticate_valid() { + let secret = "custom_key"; + let payload = "custom body".to_string(); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha256, secret, &payload); + let encoded = encode_hmac_signature(Encoding::Hex, &sig); + + let mut headers = HeaderMap::new(); + headers.insert( + "X-My-Signature", + HeaderValue::from_str(&encoded).unwrap(), + ); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Custom, + secret_key: secret.to_string(), + authentication_config: Some(SignatureAuthenticationMethod { + algorithm: HmacAlgorithm::Sha256, + encoding: Encoding::Hex, + signature_header_name: "X-My-Signature".to_string(), + signature_prefix: None, + }), + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_custom_signature_with_prefix() { + let secret = "key"; + let payload = "body".to_string(); + let sig = calculate_hmac_signature(HmacAlgorithm::Sha512, secret, &payload); + let encoded = encode_hmac_signature(Encoding::Base64, &sig); + + let mut headers = HeaderMap::new(); + headers.insert( + "X-Sig", + HeaderValue::from_str(&format!("hmac={}", encoded)).unwrap(), + ); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Custom, + secret_key: secret.to_string(), + authentication_config: Some(SignatureAuthenticationMethod { + algorithm: HmacAlgorithm::Sha512, + encoding: Encoding::Base64, + signature_header_name: "X-Sig".to_string(), + signature_prefix: Some("hmac=".to_string()), + }), + }); + assert!(method + .authenticate_http_request(&headers, Some(&payload)) + .is_ok()); + } + + #[test] + fn test_custom_signature_missing_config() { + let payload = "body".to_string(); + let headers = HeaderMap::new(); + + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Custom, + secret_key: "secret".to_string(), + authentication_config: None, + }); + assert!(matches!( + method.authenticate_http_request(&headers, Some(&payload)), + Err(AuthenticationError::InvalidCustomConfig) + )); + } + + // --- API key authentication --- + + #[test] + fn test_api_key_authenticate_valid() { + let mut headers = HeaderMap::new(); + headers.insert("X-API-Key", HeaderValue::from_static("my_secret_key_123")); + + let method = AuthenticationMethod::ApiKey(ApiKeyAuthentication { + api_key_header: "X-API-Key".to_string(), + api_key_secret: "my_secret_key_123".to_string(), + }); + assert!(method.authenticate_http_request(&headers, None).is_ok()); + } + + #[test] + fn test_api_key_authenticate_wrong_key() { + let mut headers = HeaderMap::new(); + headers.insert("X-API-Key", HeaderValue::from_static("wrong_key")); + + let method = AuthenticationMethod::ApiKey(ApiKeyAuthentication { + api_key_header: "X-API-Key".to_string(), + api_key_secret: "correct_key".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::InvalidApiKey) + )); + } + + #[test] + fn test_api_key_authenticate_missing_header() { + let headers = HeaderMap::new(); + + let method = AuthenticationMethod::ApiKey(ApiKeyAuthentication { + api_key_header: "X-API-Key".to_string(), + api_key_secret: "secret".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::InvalidApiKey) + )); + } + + // --- Basic auth --- + + fn basic_auth_header(username: &str, password: &str) -> HeaderMap { + let credentials = BASE64_STANDARD.encode(format!("{}:{}", username, password)); + let mut headers = HeaderMap::new(); + headers.insert( + "Authorization", + HeaderValue::from_str(&format!("Basic {}", credentials)).unwrap(), + ); + headers + } + + #[test] + fn test_basic_auth_valid() { + let headers = basic_auth_header("admin", "password123"); + + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "admin".to_string(), + password: "password123".to_string(), + }); + assert!(method.authenticate_http_request(&headers, None).is_ok()); + } + + #[test] + fn test_basic_auth_wrong_password() { + let headers = basic_auth_header("admin", "wrong"); + + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "admin".to_string(), + password: "correct".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::UnauthorizedBasicHttpAuth) + )); + } + + #[test] + fn test_basic_auth_wrong_username() { + let headers = basic_auth_header("wrong_user", "password"); + + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "admin".to_string(), + password: "password".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::UnauthorizedBasicHttpAuth) + )); + } + + #[test] + fn test_basic_auth_missing_header() { + let headers = HeaderMap::new(); + + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "admin".to_string(), + password: "password".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::UnauthorizedBasicHttpAuth) + )); + } + + #[test] + fn test_basic_auth_bearer_instead_of_basic() { + let mut headers = HeaderMap::new(); + headers.insert( + "Authorization", + HeaderValue::from_static("Bearer sometoken"), + ); + + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "admin".to_string(), + password: "password".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::UnauthorizedBasicHttpAuth) + )); + } + + #[test] + fn test_basic_auth_invalid_base64() { + let mut headers = HeaderMap::new(); + headers.insert( + "Authorization", + HeaderValue::from_static("Basic !!!invalid!!!"), + ); + + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "admin".to_string(), + password: "password".to_string(), + }); + assert!(matches!( + method.authenticate_http_request(&headers, None), + Err(AuthenticationError::UnauthorizedBasicHttpAuth) + )); + } + + // --- AuthenticationMethod serde (untagged enum) --- + + #[test] + fn test_authentication_method_signature_serde() { + let method = AuthenticationMethod::Signature(SignatureAuthentication { + signature_provider: WebhookType::Github, + secret_key: "secret".to_string(), + authentication_config: None, + }); + let json = serde_json::to_value(&method).unwrap(); + assert_eq!(json["signature_provider"], "Github"); + assert_eq!(json["secret_key"], "secret"); + + let deserialized: AuthenticationMethod = serde_json::from_value(json).unwrap(); + match deserialized { + AuthenticationMethod::Signature(sig) => { + assert_eq!(sig.signature_provider, WebhookType::Github); + assert_eq!(sig.secret_key, "secret"); + } + _ => panic!("expected Signature variant"), + } + } + + #[test] + fn test_authentication_method_api_key_serde() { + let method = AuthenticationMethod::ApiKey(ApiKeyAuthentication { + api_key_header: "X-Key".to_string(), + api_key_secret: "val".to_string(), + }); + let json = serde_json::to_value(&method).unwrap(); + let deserialized: AuthenticationMethod = serde_json::from_value(json).unwrap(); + match deserialized { + AuthenticationMethod::ApiKey(ak) => { + assert_eq!(ak.api_key_header, "X-Key"); + assert_eq!(ak.api_key_secret, "val"); + } + _ => panic!("expected ApiKey variant"), + } + } + + #[test] + fn test_authentication_method_basic_auth_serde() { + let method = AuthenticationMethod::BasicAuth(BasicAuthAuthentication { + username: "user".to_string(), + password: "pass".to_string(), + }); + let json = serde_json::to_value(&method).unwrap(); + let deserialized: AuthenticationMethod = serde_json::from_value(json).unwrap(); + match deserialized { + AuthenticationMethod::BasicAuth(ba) => { + assert_eq!(ba.username, "user"); + assert_eq!(ba.password, "pass"); + } + _ => panic!("expected BasicAuth variant"), + } + } + + // --- AuthenticationError into_response --- + + #[test] + fn test_error_invalid_signature_is_401() { + let response = AuthenticationError::InvalidSignature.into_response(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[test] + fn test_error_missing_header_is_400() { + let response = + AuthenticationError::MissingHeader("X-Sig".to_string()).into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn test_error_invalid_payload_is_400() { + let response = AuthenticationError::InvalidPayload.into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn test_error_invalid_api_key_is_401() { + let response = AuthenticationError::InvalidApiKey.into_response(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[test] + fn test_error_basic_auth_has_www_authenticate() { + let response = AuthenticationError::UnauthorizedBasicHttpAuth.into_response(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!(response.headers().contains_key("www-authenticate")); + } + + #[test] + fn test_error_invalid_custom_config_is_400() { + let response = AuthenticationError::InvalidCustomConfig.into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[test] + fn test_error_invalid_timestamp_is_400() { + let response = AuthenticationError::InvalidTimestamp.into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } +} diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs new file mode 100644 index 0000000000..2c109ff183 --- /dev/null +++ b/backend/windmill-trigger-http/src/lib.rs @@ -0,0 +1,595 @@ +use std::collections::HashMap; + +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use tokio::sync::{RwLock, RwLockReadGuard}; +use windmill_common::{ + error::{Error, Result}, + flows::Retry, + s3_helpers::S3Object, + utils::ExpiringCacheEntry, + worker::CLOUD_HOSTED, + DB, +}; + +use windmill_api_auth::ApiAuthed; +use windmill_trigger::TriggerMode; + +pub mod handler; +pub mod http_trigger_auth; + +lazy_static::lazy_static! { + pub static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); + pub static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); + + pub static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { + routers: HashMap::new(), + version: 0, + }); +} + +#[derive(Debug, Deserialize, Clone)] +pub struct TriggerRoute { + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub route_path: String, + pub workspace_id: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub edited_by: String, + pub email: String, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + pub raw_string: bool, + pub error_handler_path: Option, + pub error_handler_args: Option>>, + pub retry: Option>, + pub mode: TriggerMode, +} + +pub struct RoutersCache { + pub routers: HashMap>, + pub version: i64, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] +#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum RequestType { + Sync, + Async, + SyncSse, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] +#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] +pub enum AuthenticationMethod { + None, + Windmill, + ApiKey, + BasicHttp, + CustomScript, + Signature, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct HttpConfig { + pub route_path: String, + pub route_path_key: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + pub raw_string: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct HttpConfigRequest { + #[serde(default)] + pub route_path: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: Option, + pub wrap_body: Option, + pub raw_string: Option, +} + +#[derive(Deserialize)] +struct HttpConfigRequestHelper { + #[serde(default)] + route_path: String, + request_type: Option, + is_async: Option, + authentication_method: AuthenticationMethod, + http_method: HttpMethod, + summary: Option, + description: Option, + static_asset_config: Option>, + is_static_website: bool, + authentication_resource_path: Option, + workspaced_route: Option, + wrap_body: Option, + raw_string: Option, +} + +impl<'de> Deserialize<'de> for HttpConfigRequest { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let helper = HttpConfigRequestHelper::deserialize(deserializer)?; + + // Determine request_type with backward compatibility + let request_type = if let Some(mode) = helper.request_type { + mode + } else if let Some(is_async) = helper.is_async { + if is_async { + RequestType::Async + } else { + RequestType::Sync + } + } else { + RequestType::Sync + }; + + Ok(HttpConfigRequest { + route_path: helper.route_path, + request_type, + authentication_method: helper.authentication_method, + http_method: helper.http_method, + summary: helper.summary, + description: helper.description, + static_asset_config: helper.static_asset_config, + is_static_website: helper.is_static_website, + authentication_resource_path: helper.authentication_resource_path, + workspaced_route: helper.workspaced_route, + wrap_body: helper.wrap_body, + raw_string: helper.raw_string, + }) + } +} + +// Regex patterns for route validation +lazy_static::lazy_static! { + // Matches named params like :id or wildcards like :* or * + pub static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); + pub static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); +} + +#[derive(Deserialize)] +pub struct RouteExists { + pub route_path: String, + pub http_method: HttpMethod, + pub trigger_path: Option, + pub workspaced_route: Option, +} + +pub fn validate_authentication_method( + authentication_method: AuthenticationMethod, + raw_string: Option, +) -> Result<()> { + match (authentication_method, raw_string) { + (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { + Err(Error::BadRequest( + "To use custom script authentication, please enable the raw body option." + .to_string(), + )) + } + _ => Ok(()), + } +} + +pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { + let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) + .fetch_one(db) + .await?; + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + if routers_cache.version == 0 || version > routers_cache.version { + drop(routers_cache); + let mut routers = HashMap::new(); + + for http_method in [ + HttpMethod::Get, + HttpMethod::Post, + HttpMethod::Put, + HttpMethod::Patch, + HttpMethod::Delete, + ] { + let triggers = sqlx::query_as!( + TriggerRoute, + r#" + SELECT + path, + script_path, + is_flow, + route_path, + authentication_resource_path, + workspace_id, + request_type AS "request_type: _", + authentication_method AS "authentication_method: _", + edited_by, + email, + static_asset_config AS "static_asset_config: _", + wrap_body, + raw_string, + workspaced_route, + is_static_website, + error_handler_path, + error_handler_args as "error_handler_args: _", + retry as "retry: _", + mode as "mode: _" + FROM + http_trigger + WHERE + http_method = $1 AND + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + "#, + &http_method as &HttpMethod + ) + .fetch_all(db) + .await?; + + let mut router = matchit::Router::new(); + + for trigger in triggers { + let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { + format!("/{}/{}", trigger.workspace_id, trigger.route_path) + } else { + format!("/{}", trigger.route_path) + }; + + if trigger.is_static_website { + router + .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider HTTP route {}/*wm_subpath: {:?}", + full_path, + e, + ); + }); + } + router + .insert(full_path.clone(), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); + }); + } + + routers.insert(http_method, router); + } + + let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; + *routers_cache = RoutersCache { routers, version }; + + Ok((true, routers_cache.downgrade())) + } else { + tracing::debug!("No HTTP routers refresh needed"); + Ok((false, routers_cache)) + } +} + +pub async fn refresh_routers_loop( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + match refresh_routers(db).await { + Ok(_) => { + tracing::info!("Loaded HTTP routers"); + } + Err(err) => { + tracing::error!("Error loading HTTP routers: {err:#}"); + } + }; + let db = db.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = killpill_rx.recv() => { + break; + } + _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { + match refresh_routers(&db).await { + Ok((true, _)) => { + tracing::info!("Refreshed HTTP routers"); + } + Err(err) => { + tracing::error!("Error refreshing HTTP routers: {err:#}"); + } + _ => {} + } + } + } + } + }); +} + +pub struct HttpTrigger; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_request_type_backward_compatibility() { + // Test with new request_type field + let json_new = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + + // Test with legacy is_async = true + let json_legacy_async = r#"{ + "route_path": "/test", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); + assert_eq!(config.request_type, RequestType::Async); + + // Test with legacy is_async = false + let json_legacy_sync = r#"{ + "route_path": "/test", + "is_async": false, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test with neither field (default to sync) + let json_default = r#"{ + "route_path": "/test", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test that request_type takes precedence over is_async + let json_both = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + } + + // --- HttpMethod --- + + #[test] + fn test_http_method_from_http_get() { + let method = HttpMethod::try_from(&http::Method::GET).unwrap(); + assert_eq!(method, HttpMethod::Get); + } + + #[test] + fn test_http_method_from_http_post() { + let method = HttpMethod::try_from(&http::Method::POST).unwrap(); + assert_eq!(method, HttpMethod::Post); + } + + #[test] + fn test_http_method_from_http_put() { + let method = HttpMethod::try_from(&http::Method::PUT).unwrap(); + assert_eq!(method, HttpMethod::Put); + } + + #[test] + fn test_http_method_from_http_delete() { + let method = HttpMethod::try_from(&http::Method::DELETE).unwrap(); + assert_eq!(method, HttpMethod::Delete); + } + + #[test] + fn test_http_method_from_http_patch() { + let method = HttpMethod::try_from(&http::Method::PATCH).unwrap(); + assert_eq!(method, HttpMethod::Patch); + } + + #[test] + fn test_http_method_unsupported() { + let result = HttpMethod::try_from(&http::Method::HEAD); + assert!(result.is_err()); + } + + #[test] + fn test_http_method_options_unsupported() { + let result = HttpMethod::try_from(&http::Method::OPTIONS); + assert!(result.is_err()); + } + + // --- HttpMethod serde --- + + #[test] + fn test_http_method_serde_roundtrip() { + for method in [HttpMethod::Get, HttpMethod::Post, HttpMethod::Put, HttpMethod::Delete, HttpMethod::Patch] { + let json = serde_json::to_value(method).unwrap(); + let deserialized: HttpMethod = serde_json::from_value(json).unwrap(); + assert_eq!(method, deserialized); + } + } + + #[test] + fn test_http_method_serialize_lowercase() { + assert_eq!(serde_json::to_value(HttpMethod::Get).unwrap(), "get"); + assert_eq!(serde_json::to_value(HttpMethod::Post).unwrap(), "post"); + } + + // --- RequestType serde --- + + #[test] + fn test_request_type_serde_roundtrip() { + for rt in [RequestType::Sync, RequestType::Async, RequestType::SyncSse] { + let json = serde_json::to_value(rt).unwrap(); + let deserialized: RequestType = serde_json::from_value(json).unwrap(); + assert_eq!(rt, deserialized); + } + } + + #[test] + fn test_request_type_serialize_values() { + assert_eq!(serde_json::to_value(RequestType::Sync).unwrap(), "sync"); + assert_eq!(serde_json::to_value(RequestType::Async).unwrap(), "async"); + assert_eq!(serde_json::to_value(RequestType::SyncSse).unwrap(), "sync_sse"); + } + + // --- AuthenticationMethod serde --- + + #[test] + fn test_authentication_method_serde_roundtrip() { + for method in [ + AuthenticationMethod::None, + AuthenticationMethod::Windmill, + AuthenticationMethod::ApiKey, + AuthenticationMethod::BasicHttp, + AuthenticationMethod::CustomScript, + AuthenticationMethod::Signature, + ] { + let json = serde_json::to_value(method).unwrap(); + let deserialized: AuthenticationMethod = serde_json::from_value(json).unwrap(); + assert_eq!(method, deserialized); + } + } + + // --- validate_authentication_method --- + + #[test] + fn test_validate_auth_none_ok() { + assert!(validate_authentication_method(AuthenticationMethod::None, None).is_ok()); + } + + #[test] + fn test_validate_auth_windmill_ok() { + assert!(validate_authentication_method(AuthenticationMethod::Windmill, None).is_ok()); + } + + #[test] + fn test_validate_auth_custom_script_requires_raw() { + assert!(validate_authentication_method(AuthenticationMethod::CustomScript, None).is_err()); + assert!(validate_authentication_method(AuthenticationMethod::CustomScript, Some(false)).is_err()); + assert!(validate_authentication_method(AuthenticationMethod::CustomScript, Some(true)).is_ok()); + } + + #[test] + fn test_validate_auth_signature_without_raw_ok() { + assert!(validate_authentication_method(AuthenticationMethod::Signature, None).is_ok()); + } + + // --- Route path regex --- + + #[test] + fn test_valid_route_path() { + assert!(VALID_ROUTE_PATH_RE.is_match("users")); + assert!(VALID_ROUTE_PATH_RE.is_match("users/:id")); + assert!(VALID_ROUTE_PATH_RE.is_match("api/v1/users")); + assert!(VALID_ROUTE_PATH_RE.is_match("api/v1/:id")); + assert!(VALID_ROUTE_PATH_RE.is_match("files/*path")); + } + + #[test] + fn test_invalid_route_path() { + assert!(!VALID_ROUTE_PATH_RE.is_match("")); + assert!(!VALID_ROUTE_PATH_RE.is_match("/leading-slash")); + } + + #[test] + fn test_route_path_key_regex() { + assert!(ROUTE_PATH_KEY_RE.is_match("/:id")); + assert!(ROUTE_PATH_KEY_RE.is_match("/*path")); + assert!(ROUTE_PATH_KEY_RE.is_match("/users/:userId/posts/:postId")); + } + + // --- HttpConfig deserialization --- + + #[test] + fn test_http_config_request_full() { + let json = r#"{ + "route_path": "api/v1/users", + "request_type": "async", + "authentication_method": "api_key", + "http_method": "post", + "is_static_website": false, + "workspaced_route": true, + "wrap_body": true, + "raw_string": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json).unwrap(); + assert_eq!(config.route_path, "api/v1/users"); + assert_eq!(config.request_type, RequestType::Async); + assert_eq!(config.authentication_method, AuthenticationMethod::ApiKey); + assert_eq!(config.http_method, HttpMethod::Post); + assert_eq!(config.workspaced_route, Some(true)); + assert_eq!(config.wrap_body, Some(true)); + } + + #[test] + fn test_http_config_request_minimal() { + let json = r#"{ + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json).unwrap(); + assert_eq!(config.route_path, ""); + assert_eq!(config.request_type, RequestType::Sync); + assert!(config.workspaced_route.is_none()); + assert!(config.summary.is_none()); + } +} diff --git a/backend/windmill-trigger-kafka/Cargo.toml b/backend/windmill-trigger-kafka/Cargo.toml new file mode 100644 index 0000000000..9887dfc04a --- /dev/null +++ b/backend/windmill-trigger-kafka/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-trigger-kafka" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_kafka" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] +kafka-gssapi = ["rdkafka/gssapi"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +rdkafka.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +base64.workspace = true +sha2.workspace = true +anyhow.workspace = true diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-trigger-kafka/src/handler_oss.rs similarity index 90% rename from backend/windmill-api/src/triggers/kafka/handler_oss.rs rename to backend/windmill-trigger-kafka/src/handler_oss.rs index 9fda83916f..d85a6caba6 100644 --- a/backend/windmill-api/src/triggers/kafka/handler_oss.rs +++ b/backend/windmill-trigger-kafka/src/handler_oss.rs @@ -1,5 +1,5 @@ #[cfg(not(feature = "private"))] -use crate::triggers::TriggerData; +use windmill_trigger::TriggerData; #[allow(unused)] #[cfg(feature = "private")] @@ -8,14 +8,15 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::KafkaTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::TriggerCrud, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + db::DB, + error::{Error, Result}, + }, windmill_git_sync::DeployedObject, + windmill_trigger::TriggerCrud, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-kafka/src/lib.rs b/backend/windmill-trigger-kafka/src/lib.rs new file mode 100644 index 0000000000..0a2733cc5c --- /dev/null +++ b/backend/windmill-trigger-kafka/src/lib.rs @@ -0,0 +1,23 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct KafkaTrigger; diff --git a/backend/windmill-api/src/triggers/kafka/listener_oss.rs b/backend/windmill-trigger-kafka/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/kafka/listener_oss.rs rename to backend/windmill-trigger-kafka/src/listener_oss.rs index c5dca4ab37..057844efaf 100644 --- a/backend/windmill-api/src/triggers/kafka/listener_oss.rs +++ b/backend/windmill-trigger-kafka/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::KafkaTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for KafkaTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Kafka; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for KafkaTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for KafkaTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-mqtt/Cargo.toml b/backend/windmill-trigger-mqtt/Cargo.toml new file mode 100644 index 0000000000..305880b176 --- /dev/null +++ b/backend/windmill-trigger-mqtt/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-trigger-mqtt" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_mqtt" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["mqtt_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +rumqttc.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +base64.workspace = true +anyhow.workspace = true +bytes.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-api/src/triggers/mqtt/handler.rs b/backend/windmill-trigger-mqtt/src/handler.rs similarity index 92% rename from backend/windmill-api/src/triggers/mqtt/handler.rs rename to backend/windmill-trigger-mqtt/src/handler.rs index ac4c027fff..7b12a23c5f 100644 --- a/backend/windmill-api/src/triggers/mqtt/handler.rs +++ b/backend/windmill-trigger-mqtt/src/handler.rs @@ -1,16 +1,15 @@ -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - triggers::{Trigger, TriggerCrud, TriggerData}, -}; use axum::async_trait; use itertools::Itertools; use sqlx::{types::Json as SqlxJson, PgConnection}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; use windmill_common::{ db::UserDB, error::{Error, Result}, }; use windmill_git_sync::DeployedObject; +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ MqttClientBuilder, MqttClientVersion, MqttConfig, MqttConfigRequest, MqttResource, MqttTrigger, @@ -92,16 +91,16 @@ impl TriggerCrud for MqttTrigger { v3_config, v5_config, workspace_id, - path, - script_path, - is_flow, - email, - mode, + path, + script_path, + is_flow, + email, + mode, edited_by, error_handler_path, error_handler_args, retry - ) + ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 )"#, @@ -149,8 +148,8 @@ impl TriggerCrud for MqttTrigger { // Important to set server_id to NULL to stop current mqtt listener sqlx::query!( r#" - UPDATE - mqtt_trigger + UPDATE + mqtt_trigger SET mqtt_resource_path = $1, subscribe_topics = $2, @@ -158,19 +157,19 @@ impl TriggerCrud for MqttTrigger { client_id = $4, v3_config = $5, v5_config = $6, - is_flow = $7, - edited_by = $8, + is_flow = $7, + edited_by = $8, email = $9, script_path = $10, path = $11, - edited_at = now(), + edited_at = now(), error = NULL, server_id = NULL, error_handler_path = $14, error_handler_args = $15, retry = $16 - WHERE - workspace_id = $12 AND + WHERE + workspace_id = $12 AND path = $13 "#, trigger.config.mqtt_resource_path, diff --git a/backend/windmill-trigger-mqtt/src/lib.rs b/backend/windmill-trigger-mqtt/src/lib.rs new file mode 100644 index 0000000000..62e5093f7f --- /dev/null +++ b/backend/windmill-trigger-mqtt/src/lib.rs @@ -0,0 +1,347 @@ +use base64::{engine, prelude::*}; +use itertools::Itertools; +use rumqttc::{ + v5::{ + mqttbytes::{ + v5::{ConnectProperties, Filter}, + QoS as V5QoS, + }, + AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, + }, + AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, + QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow, Type}; +use std::{collections::HashMap, time::Duration}; +use windmill_common::{ + error::{to_anyhow, Error}, + triggers::TriggerKind, + worker::to_raw_value, +}; + +use crate::listener::EventLoop; +use windmill_trigger::trigger_helpers::TriggerJobArgs; + +pub mod handler; +pub mod listener; + +#[derive(Clone, Copy)] +pub struct MqttTrigger; + +impl TriggerJobArgs for MqttTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[serde(rename_all = "lowercase")] +pub enum QualityOfService { + Qos0, + Qos1, + Qos2, +} + +impl From for V3QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V3QoS::AtMostOnce, + QualityOfService::Qos1 => V3QoS::AtLeastOnce, + QualityOfService::Qos2 => V3QoS::ExactlyOnce, + } + } +} + +impl From for V5QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V5QoS::AtMostOnce, + QualityOfService::Qos1 => V5QoS::AtLeastOnce, + QualityOfService::Qos2 => V5QoS::ExactlyOnce, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV3Config { + clean_session: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV5Config { + clean_start: Option, + session_expiry_interval: Option, + topic_alias_maximum: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[sqlx(type_name = "MQTT_CLIENT_VERSION")] +#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MqttClientVersion { + V3, + V5, +} + +#[derive(Debug, Deserialize)] +pub struct Tls { + enabled: bool, + ca_certificate: String, + pkcs12_client_certificate: Option, + pkcs12_certificate_password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Credentials { + username: Option, + password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct MqttResource { + broker: String, + port: u16, + credentials: Option, + tls: Option, +} + +#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] +pub struct SubscribeTopic { + qos: QualityOfService, + topic: String, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct MqttConfig { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec>, + pub v3_config: Option>, + pub v5_config: Option>, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttConfigRequest { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec, + pub v3_config: Option, + pub v5_config: Option, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestMqttConfig { + pub mqtt_resource_path: String, + pub client_version: Option, + pub v3_config: Option, + pub v5_config: Option, +} + +// Constants +pub const KEEP_ALIVE: u64 = 60; +pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; +pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; +pub const TIMEOUT_DURATION: u64 = 10; +pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +pub struct V3MqttHandler; +pub struct V5MqttHandler; + +pub enum MqttClientResult { + V3((V3MqttHandler, V3EventLoop)), + V5((V5MqttHandler, V5EventLoop)), +} + +#[derive(Debug, thiserror::Error)] +pub enum MqttError { + #[error("{0}")] + Common(#[from] Error), + #[error("{0}")] + V5RumqttClient(#[from] rumqttc::v5::ClientError), + #[error("{0}")] + V5ConnectionError(#[from] rumqttc::v5::ConnectionError), + #[error("{0}")] + V3RumqttClient(#[from] rumqttc::ClientError), + #[error("{0}")] + V3ConnectionError(#[from] rumqttc::ConnectionError), + #[error("{0}")] + Base64Decode(#[from] base64::DecodeError), +} + +pub struct MqttClientBuilder<'client> { + mqtt_resource: MqttResource, + client_id: &'client str, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, +} + +impl<'client> MqttClientBuilder<'client> { + pub fn new( + mqtt_resource: MqttResource, + client_id: Option<&'client str>, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, + ) -> Self { + Self { + mqtt_resource, + client_id: client_id.unwrap_or(""), + subscribe_topics, + v3_config, + v5_config, + mqtt_client_version, + } + } + + pub async fn build_client(&self) -> Result { + match self.mqtt_client_version { + Some(MqttClientVersion::V5) | None => self.build_v5_client().await, + Some(MqttClientVersion::V3) => self.build_v3_client().await, + } + } + + fn get_tls_configuration(&self) -> Result, Error> { + let transport = match self.mqtt_resource.tls { + Some(ref tls) if tls.enabled => { + let transport = match tls.ca_certificate.trim().is_empty() { + true => rumqttc::Transport::Tls(TlsConfiguration::Native), + false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { + ca: tls.ca_certificate.as_bytes().to_vec(), + client_auth: { + match tls.pkcs12_client_certificate.as_ref() { + Some(client_certificate) + if !client_certificate.trim().is_empty() => + { + let client_certificate = BASE64_STANDARD + .decode(client_certificate) + .map_err(to_anyhow)?; + let password = tls + .pkcs12_certificate_password + .clone() + .unwrap_or("".to_string()); + Some((client_certificate, password)) + } + _ => None, + } + }, + }), + }; + + Some(transport) + } + _ => None, + }; + + Ok(transport) + } + + async fn build_v5_client(&self) -> Result { + let mut mqtt_options = V5MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + + mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); + + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + + if let Some(v5_config) = self.v5_config { + mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); + mqtt_options.set_connect_properties(ConnectProperties { + session_expiry_interval: v5_config.session_expiry_interval, + receive_maximum: None, + max_packet_size: None, + topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), + request_response_info: None, + request_problem_info: None, + user_properties: vec![], + authentication_method: None, + authentication_data: None, + }); + } + + let (async_client, mut event_loop) = + V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) + } + + async fn build_v3_client(&self) -> Result { + let mut mqtt_options = V3MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + if let Some(v3_config) = self.v3_config { + mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); + } + + let (async_client, mut event_loop) = + V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) + } +} diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-trigger-mqtt/src/listener.rs similarity index 97% rename from backend/windmill-api/src/triggers/mqtt/listener.rs rename to backend/windmill-trigger-mqtt/src/listener.rs index af2d94b113..2098a3c20a 100644 --- a/backend/windmill-api/src/triggers/mqtt/listener.rs +++ b/backend/windmill-trigger-mqtt/src/listener.rs @@ -19,17 +19,14 @@ use windmill_common::{ DB, }; -use crate::{ - resources::try_get_resource_from_db_as, - triggers::{ - listener::ListeningTrigger, - mqtt::{ - MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, - V3MqttHandler, V5MqttHandler, - }, - trigger_helpers::TriggerJobArgs, - Listener, - }, +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::trigger_helpers::TriggerJobArgs; +use windmill_trigger::Listener; + +use super::{ + MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, V3MqttHandler, + V5MqttHandler, }; #[async_trait] diff --git a/backend/windmill-trigger-nats/Cargo.toml b/backend/windmill-trigger-nats/Cargo.toml new file mode 100644 index 0000000000..a0830097bb --- /dev/null +++ b/backend/windmill-trigger-nats/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-trigger-nats" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_nats" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["nats"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +async-nats.workspace = true +nkeys.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +base64.workspace = true diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-trigger-nats/src/handler_oss.rs similarity index 88% rename from backend/windmill-api/src/triggers/nats/handler_oss.rs rename to backend/windmill-trigger-nats/src/handler_oss.rs index 3ab3430e5f..9c011041b5 100644 --- a/backend/windmill-api/src/triggers/nats/handler_oss.rs +++ b/backend/windmill-trigger-nats/src/handler_oss.rs @@ -2,17 +2,21 @@ #[cfg(feature = "private")] pub use super::handler_ee::*; +#[cfg(not(feature = "private"))] +use windmill_trigger::TriggerData; + #[cfg(not(feature = "private"))] use { super::NatsTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{TriggerCrud, TriggerData}, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + db::DB, + error::{Error, Result}, + }, windmill_git_sync::DeployedObject, + windmill_trigger::TriggerCrud, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-nats/src/lib.rs b/backend/windmill-trigger-nats/src/lib.rs new file mode 100644 index 0000000000..668df073c9 --- /dev/null +++ b/backend/windmill-trigger-nats/src/lib.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct NatsTrigger; diff --git a/backend/windmill-api/src/triggers/nats/listener_oss.rs b/backend/windmill-trigger-nats/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/nats/listener_oss.rs rename to backend/windmill-trigger-nats/src/listener_oss.rs index 80f1befded..588536f6bf 100644 --- a/backend/windmill-api/src/triggers/nats/listener_oss.rs +++ b/backend/windmill-trigger-nats/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::NatsTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for NatsTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Nats; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for NatsTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for NatsTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-postgres/Cargo.toml b/backend/windmill-trigger-postgres/Cargo.toml new file mode 100644 index 0000000000..d932274f8e --- /dev/null +++ b/backend/windmill-trigger-postgres/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "windmill-trigger-postgres" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_postgres" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["postgres_trigger"] } +anyhow.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +rust-postgres.workspace = true +rust-postgres-native-tls.workspace = true +pg_escape.workspace = true +native-tls.workspace = true +rust_decimal.workspace = true +byteorder.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +chrono.workspace = true +rand.workspace = true +uuid.workspace = true +bytes.workspace = true +futures.workspace = true +thiserror.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-trigger-postgres/src/bool.rs b/backend/windmill-trigger-postgres/src/bool.rs new file mode 100644 index 0000000000..2acf42b460 --- /dev/null +++ b/backend/windmill-trigger-postgres/src/bool.rs @@ -0,0 +1,57 @@ +use thiserror::Error; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs +* +*/ + +#[derive(Debug, Error)] +pub enum ParseBoolError { + #[error("invalid input value: {0}")] + InvalidInput(String), +} + +pub fn parse_bool(s: &str) -> Result { + match s { + "t" => Ok(true), + "f" => Ok(false), + _ => Err(ParseBoolError::InvalidInput(s.to_string())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_true() { + assert_eq!(parse_bool("t").unwrap(), true); + } + + #[test] + fn test_parse_false() { + assert_eq!(parse_bool("f").unwrap(), false); + } + + #[test] + fn test_invalid_true_string() { + assert!(matches!( + parse_bool("true"), + Err(ParseBoolError::InvalidInput(s)) if s == "true" + )); + } + + #[test] + fn test_invalid_empty() { + assert!(parse_bool("").is_err()); + } + + #[test] + fn test_invalid_uppercase() { + assert!(parse_bool("T").is_err()); + } +} diff --git a/backend/windmill-api/src/triggers/postgres/converter.rs b/backend/windmill-trigger-postgres/src/converter.rs similarity index 57% rename from backend/windmill-api/src/triggers/postgres/converter.rs rename to backend/windmill-trigger-postgres/src/converter.rs index f57268c4b8..172b62df77 100644 --- a/backend/windmill-api/src/triggers/postgres/converter.rs +++ b/backend/windmill-trigger-postgres/src/converter.rs @@ -18,10 +18,10 @@ use uuid::Uuid; /** * This implementation is inspired by Postgres replication functionality * from https://github.com/supabase/pg_replicate -* -* Original implementation: +* +* Original implementation: * - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/text.rs -* +* */ #[derive(Debug, Error)] @@ -253,3 +253,244 @@ impl Converter { Ok(arr) } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + // --- Scalar type conversions --- + + #[test] + fn test_bool_true() { + let result = Converter::try_from_str(Some(Type::BOOL), "t").unwrap(); + assert_eq!(result, Value::Bool(true)); + } + + #[test] + fn test_bool_false() { + let result = Converter::try_from_str(Some(Type::BOOL), "f").unwrap(); + assert_eq!(result, Value::Bool(false)); + } + + #[test] + fn test_text() { + let result = Converter::try_from_str(Some(Type::TEXT), "hello world").unwrap(); + assert_eq!(result, Value::String("hello world".to_string())); + } + + #[test] + fn test_varchar() { + let result = Converter::try_from_str(Some(Type::VARCHAR), "test").unwrap(); + assert_eq!(result, Value::String("test".to_string())); + } + + #[test] + fn test_int2() { + let result = Converter::try_from_str(Some(Type::INT2), "42").unwrap(); + assert_eq!(result, json!(42)); + } + + #[test] + fn test_int4() { + let result = Converter::try_from_str(Some(Type::INT4), "-100").unwrap(); + assert_eq!(result, json!(-100)); + } + + #[test] + fn test_int8() { + let result = Converter::try_from_str(Some(Type::INT8), "9999999999").unwrap(); + assert_eq!(result, json!(9999999999i64)); + } + + #[test] + fn test_float4() { + let result = Converter::try_from_str(Some(Type::FLOAT4), "3.14").unwrap(); + assert!(result.as_f64().unwrap() - 3.14 < 0.001); + } + + #[test] + fn test_float8() { + let result = Converter::try_from_str(Some(Type::FLOAT8), "2.718281828").unwrap(); + assert!(result.as_f64().unwrap() - 2.718281828 < 0.0001); + } + + #[test] + fn test_numeric() { + let result = Converter::try_from_str(Some(Type::NUMERIC), "123.456").unwrap(); + assert_eq!(result.to_string(), "123.456"); + } + + #[test] + fn test_uuid() { + let result = + Converter::try_from_str(Some(Type::UUID), "550e8400-e29b-41d4-a716-446655440000") + .unwrap(); + assert_eq!( + result, + Value::String("550e8400-e29b-41d4-a716-446655440000".to_string()) + ); + } + + #[test] + fn test_json() { + let result = + Converter::try_from_str(Some(Type::JSON), r#"{"key": "value", "n": 1}"#).unwrap(); + assert_eq!(result, json!({"key": "value", "n": 1})); + } + + #[test] + fn test_jsonb() { + let result = Converter::try_from_str(Some(Type::JSONB), r#"[1,2,3]"#).unwrap(); + assert_eq!(result, json!([1, 2, 3])); + } + + #[test] + fn test_date() { + let result = Converter::try_from_str(Some(Type::DATE), "2024-01-15").unwrap(); + assert_eq!(result, Value::String("2024-01-15".to_string())); + } + + #[test] + fn test_time() { + let result = Converter::try_from_str(Some(Type::TIME), "14:30:00.0").unwrap(); + assert_eq!(result, Value::String("14:30:00".to_string())); + } + + #[test] + fn test_timestamp() { + let result = + Converter::try_from_str(Some(Type::TIMESTAMP), "2024-01-15 14:30:00.0").unwrap(); + assert_eq!( + result, + Value::String("2024-01-15 14:30:00".to_string()) + ); + } + + #[test] + fn test_timestamptz() { + let result = Converter::try_from_str( + Some(Type::TIMESTAMPTZ), + "2024-01-15 14:30:00.0+00", + ) + .unwrap(); + assert!(result.as_str().unwrap().contains("2024-01-15")); + } + + #[test] + fn test_bytea() { + let result = Converter::try_from_str(Some(Type::BYTEA), "\\x48656c6c6f").unwrap(); + assert_eq!(result, json!([72, 101, 108, 108, 111])); + } + + #[test] + fn test_oid() { + let result = Converter::try_from_str(Some(Type::OID), "12345").unwrap(); + assert_eq!(result, json!(12345u32)); + } + + #[test] + fn test_none_type_defaults_to_text() { + let result = Converter::try_from_str(None, "anything").unwrap(); + assert_eq!(result, Value::String("anything".to_string())); + } + + // --- Array type conversions --- + + #[test] + fn test_int4_array() { + let result = Converter::try_from_str(Some(Type::INT4_ARRAY), "{1,2,3}").unwrap(); + assert_eq!(result, json!([1, 2, 3])); + } + + #[test] + fn test_text_array() { + let result = Converter::try_from_str(Some(Type::TEXT_ARRAY), "{hello,world}").unwrap(); + assert_eq!(result, json!(["hello", "world"])); + } + + #[test] + fn test_bool_array() { + let result = Converter::try_from_str(Some(Type::BOOL_ARRAY), "{t,f,t}").unwrap(); + assert_eq!(result, json!([true, false, true])); + } + + #[test] + fn test_array_with_null() { + let result = Converter::try_from_str(Some(Type::INT4_ARRAY), "{1,NULL,3}").unwrap(); + assert_eq!(result, json!([1, null, 3])); + } + + #[test] + fn test_array_with_quoted_strings() { + let result = + Converter::try_from_str(Some(Type::TEXT_ARRAY), r#"{"hello, world","test"}"#).unwrap(); + assert_eq!(result, json!(["hello, world", "test"])); + } + + #[test] + fn test_empty_array() { + let result = Converter::try_from_str(Some(Type::INT4_ARRAY), "{}").unwrap(); + assert_eq!(result, json!([])); + } + + #[test] + fn test_uuid_array() { + let result = Converter::try_from_str( + Some(Type::UUID_ARRAY), + "{550e8400-e29b-41d4-a716-446655440000,6ba7b810-9dad-11d1-80b4-00c04fd430c8}", + ) + .unwrap(); + assert_eq!( + result, + json!([ + "550e8400-e29b-41d4-a716-446655440000", + "6ba7b810-9dad-11d1-80b4-00c04fd430c8" + ]) + ); + } + + // --- Error cases --- + + #[test] + fn test_invalid_int() { + assert!(Converter::try_from_str(Some(Type::INT4), "not_a_number").is_err()); + } + + #[test] + fn test_invalid_bool() { + assert!(Converter::try_from_str(Some(Type::BOOL), "yes").is_err()); + } + + #[test] + fn test_invalid_uuid() { + assert!(Converter::try_from_str(Some(Type::UUID), "not-a-uuid").is_err()); + } + + #[test] + fn test_invalid_json() { + assert!(Converter::try_from_str(Some(Type::JSON), "not json").is_err()); + } + + #[test] + fn test_array_missing_braces() { + assert!(Converter::try_from_str(Some(Type::INT4_ARRAY), "1,2,3").is_err()); + } + + #[test] + fn test_array_too_short() { + assert!(Converter::try_from_str(Some(Type::INT4_ARRAY), "{").is_err()); + } + + #[test] + fn test_array_with_escaped_backslash() { + let result = + Converter::try_from_str(Some(Type::TEXT_ARRAY), r#"{"a\\b","c"}"#).unwrap(); + assert_eq!(result, json!(["a\\b", "c"])); + } + + #[test] + fn test_float_nan_rejected() { + assert!(Converter::try_from_str(Some(Type::FLOAT4), "NaN").is_err()); + } +} diff --git a/backend/windmill-api/src/triggers/postgres/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs similarity index 98% rename from backend/windmill-api/src/triggers/postgres/handler.rs rename to backend/windmill-trigger-postgres/src/handler.rs index c46441cd0c..dc0057ffa4 100644 --- a/backend/windmill-api/src/triggers/postgres/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -16,20 +16,19 @@ use uuid; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error, Result}, + DB, }; use windmill_git_sync::DeployedObject; -use crate::{ - db::{ApiAuthed, DB}, - triggers::{postgres::PostgresTrigger, Trigger, TriggerCrud, TriggerData}, -}; +use windmill_api_auth::ApiAuthed; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, mapper::{Mapper, MappingInfo}, - PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PublicationData, - Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, + PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PostgresTrigger, + PublicationData, Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, ERROR_PUBLICATION_NAME_NOT_EXISTS, }; @@ -211,8 +210,8 @@ impl TriggerCrud for PostgresTrigger { sqlx::query!( r#" - UPDATE postgres_trigger - SET + UPDATE postgres_trigger + SET postgres_resource_path = $1, replication_slot_name = $2, publication_name = $3, @@ -227,7 +226,7 @@ impl TriggerCrud for PostgresTrigger { error_handler_path = $11, error_handler_args = $12, retry = $13 - WHERE + WHERE workspace_id = $9 AND path = $10 "#, postgres_resource_path, @@ -419,12 +418,12 @@ pub async fn list_slot_name( let rows = pg_connection .query( r#" - SELECT + SELECT slot_name, active FROM - pg_replication_slots - WHERE + pg_replication_slots + WHERE plugin = 'pgoutput' AND slot_type = 'logical'; "#, @@ -467,11 +466,11 @@ pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &s let row = pg_connection .query_opt( r#" - SELECT - active_pid - FROM - pg_replication_slots - WHERE + SELECT + active_pid + FROM + pg_replication_slots + WHERE slot_name = $1 "#, &[&slot_name], diff --git a/backend/windmill-trigger-postgres/src/hex.rs b/backend/windmill-trigger-postgres/src/hex.rs new file mode 100644 index 0000000000..9c8c914f2c --- /dev/null +++ b/backend/windmill-trigger-postgres/src/hex.rs @@ -0,0 +1,126 @@ +use std::num::ParseIntError; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs +* +*/ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ByteaHexParseError { + #[error("missing prefix '\\x'")] + InvalidPrefix, + + #[error("invalid byte")] + OddNumerOfDigits, + + #[error("parse int result: {0}")] + ParseInt(#[from] ParseIntError), +} + +pub fn from_bytea_hex(s: &str) -> Result, ByteaHexParseError> { + if s.len() < 2 || &s[..2] != "\\x" { + return Err(ByteaHexParseError::InvalidPrefix); + } + + let mut result = Vec::with_capacity((s.len() - 2) / 2); + let s = &s[2..]; + + if s.len() % 2 != 0 { + return Err(ByteaHexParseError::OddNumerOfDigits); + } + + for i in (0..s.len()).step_by(2) { + let val = u8::from_str_radix(&s[i..i + 2], 16)?; + result.push(val); + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid_bytea_hex() { + assert_eq!(from_bytea_hex("\\x48656c6c6f").unwrap(), b"Hello"); + } + + #[test] + fn test_empty_bytea_hex() { + assert_eq!(from_bytea_hex("\\x").unwrap(), Vec::::new()); + } + + #[test] + fn test_single_byte() { + assert_eq!(from_bytea_hex("\\xff").unwrap(), vec![0xff]); + } + + #[test] + fn test_all_zeros() { + assert_eq!(from_bytea_hex("\\x000000").unwrap(), vec![0, 0, 0]); + } + + #[test] + fn test_missing_prefix() { + assert!(matches!( + from_bytea_hex("48656c6c6f"), + Err(ByteaHexParseError::InvalidPrefix) + )); + } + + #[test] + fn test_wrong_prefix() { + assert!(matches!( + from_bytea_hex("0x48656c6c6f"), + Err(ByteaHexParseError::InvalidPrefix) + )); + } + + #[test] + fn test_too_short() { + assert!(matches!( + from_bytea_hex("\\"), + Err(ByteaHexParseError::InvalidPrefix) + )); + } + + #[test] + fn test_empty_string() { + assert!(matches!( + from_bytea_hex(""), + Err(ByteaHexParseError::InvalidPrefix) + )); + } + + #[test] + fn test_odd_digits() { + assert!(matches!( + from_bytea_hex("\\xabc"), + Err(ByteaHexParseError::OddNumerOfDigits) + )); + } + + #[test] + fn test_invalid_hex_chars() { + assert!(matches!( + from_bytea_hex("\\xzz"), + Err(ByteaHexParseError::ParseInt(_)) + )); + } + + #[test] + fn test_uppercase_hex() { + assert_eq!(from_bytea_hex("\\xABCD").unwrap(), vec![0xab, 0xcd]); + } + + #[test] + fn test_mixed_case_hex() { + assert_eq!(from_bytea_hex("\\xAbCd").unwrap(), vec![0xab, 0xcd]); + } +} diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs new file mode 100644 index 0000000000..011453739c --- /dev/null +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -0,0 +1,712 @@ +use std::collections::HashMap; + +use windmill_api_auth::ApiAuthed; +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::trigger_helpers::TriggerJobArgs; +use chrono::Utc; +use itertools::Itertools; +use native_tls::{Certificate, TlsConnector}; +use pg_escape::quote_identifier; +use rand::Rng; +use rust_postgres::{config::SslMode, Client, Config, NoTls}; +use rust_postgres_native_tls::MakeTlsConnector; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::value::RawValue; +use sqlx::FromRow; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + utils::empty_as_none, + DB, +}; + +mod bool; +mod converter; +pub mod handler; +mod hex; +pub mod listener; +mod mapper; +mod relation; +mod replication_message; + +#[derive(Clone, Copy)] +pub struct PostgresTrigger; + +impl TriggerJobArgs for PostgresTrigger { + type Payload = HashMap>; + const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; + fn v1_payload_fn(payload: &HashMap>) -> HashMap> { + payload.to_owned() + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct PostgresConfig { + pub postgres_resource_path: String, + pub replication_slot_name: String, + pub publication_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub basic_mode: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PostgresConfigRequest { + postgres_resource_path: String, + #[serde(default)] + replication_slot_name: String, + #[serde(default)] + publication_name: String, + publication: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestPostgresConfig { + pub postgres_resource_path: String, +} + +fn check_if_valid_relation<'de, D>( + relations: D, +) -> std::result::Result>, D::Error> +where + D: Deserializer<'de>, +{ + let relations: Option> = Option::deserialize(relations)?; + let mut track_all_table_in_schema = false; + let mut track_specific_columns_in_table = false; + match relations { + Some(relations) => { + for relation in relations.iter() { + if relation.schema_name.is_empty() { + return Err(serde::de::Error::custom( + "Schema Name must not be empty".to_string(), + )); + } + + if !track_all_table_in_schema && relation.table_to_track.is_empty() { + track_all_table_in_schema = true; + continue; + } + + for table_to_track in relation.table_to_track.iter() { + if table_to_track.table_name.trim().is_empty() { + return Err(serde::de::Error::custom( + "Table name must not be empty".to_string(), + )); + } + + if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { + track_specific_columns_in_table = true; + } + } + + if track_all_table_in_schema && track_specific_columns_in_table { + return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); + } + } + + if !relations + .iter() + .map(|relation| relation.schema_name.as_str()) + .all_unique() + { + return Err(serde::de::Error::custom( + "You cannot choose a schema more than one time".to_string(), + )); + } + + Ok(Some(relations)) + } + None => Ok(None), + } +} + +fn check_if_valid_transaction_type<'de, D>( + transaction_type: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; + if transaction_type.len() > 3 { + return Err(serde::de::Error::custom( + "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" + .to_string(), + )); + } + transaction_type.sort_unstable(); + transaction_type.dedup(); + + for transaction in transaction_type.iter() { + match transaction.to_lowercase().as_ref() { + "insert" => {}, + "update" => {}, + "delete" => {}, + _ => { + return Err(serde::de::Error::custom( + "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" + .to_string(), + )) + } + } + } + + Ok(transaction_type) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PublicationData { + #[serde(default, deserialize_with = "check_if_valid_relation")] + pub table_to_track: Option>, + #[serde(deserialize_with = "check_if_valid_transaction_type")] + pub transaction_to_track: Vec, +} + +impl PublicationData { + pub fn new( + table_to_track: Option>, + transaction_to_track: Vec, + ) -> PublicationData { + PublicationData { table_to_track, transaction_to_track } + } +} + +// Slot list struct +#[derive(FromRow, Debug, Serialize)] +pub struct SlotList { + pub slot_name: Option, + pub active: Option, +} + +// Slot struct +#[derive(Debug, Serialize, Deserialize)] +pub struct Slot { + pub name: String, +} + +// Template script struct +#[derive(Debug, Deserialize)] +pub struct TemplateScript { + pub postgres_resource_path: String, + #[serde(deserialize_with = "check_if_valid_relation")] + pub relations: Option>, + pub language: Language, +} + +// Language enum +#[derive(Deserialize, Debug)] +pub enum Language { + #[serde(rename = "typescript", alias = "Typescript")] + Typescript, +} + +// Test postgres struct +#[derive(Serialize, Deserialize)] +pub struct TestPostgres { + pub postgres_resource_path: String, +} + +// PostgreSQL publication replication struct +#[derive(Serialize, Deserialize)] +pub struct PostgresPublicationReplication { + pub publication_name: String, + pub replication_slot_name: String, +} + +impl PostgresPublicationReplication { + pub fn new( + publication_name: String, + replication_slot_name: String, + ) -> PostgresPublicationReplication { + PostgresPublicationReplication { publication_name, replication_slot_name } + } +} + +pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +#[derive(FromRow, Serialize, Deserialize, Debug)] +pub struct Postgres { + pub user: String, + pub password: String, + pub host: String, + pub port: Option, + pub dbname: String, + #[serde(default)] + pub sslmode: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub root_certificate_pem: Option, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct TableToTrack { + pub table_name: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub where_clause: Option, + #[serde(default, deserialize_with = "empty_as_none")] + pub columns_name: Option>, +} + +impl TableToTrack { + pub fn new( + table_name: String, + where_clause: Option, + columns_name: Option>, + ) -> TableToTrack { + TableToTrack { table_name, where_clause, columns_name } + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct Relations { + pub schema_name: String, + pub table_to_track: Vec, +} + +impl Relations { + pub fn new(schema_name: String, table_to_track: Vec) -> Relations { + Relations { schema_name, table_to_track } + } + + pub fn add_new_table(&mut self, table_to_track: TableToTrack) { + self.table_to_track.push(table_to_track); + } +} + +fn build_tls_connector( + ssl_mode: SslMode, + root_certificate_pem: Option<&String>, +) -> Result> { + let get_tls_builder_for_verify = |root_certificate: Option<&String>| { + let mut builder = TlsConnector::builder(); + if let Some(root_certificate) = root_certificate { + let root_certificate_pem = + Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; + builder.add_root_certificate(root_certificate_pem); + } + Ok::<_, Error>(builder) + }; + let connector = match ssl_mode { + SslMode::Disable => return Ok(None), + SslMode::Require | SslMode::Prefer => { + let mut builder = TlsConnector::builder(); + builder.danger_accept_invalid_certs(true); + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyCa => { + let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyFull => { + let builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder + } + _ => unreachable!(), + }; + + Ok(Some(MakeTlsConnector::new( + connector.build().map_err(to_anyhow)?, + ))) +} + +pub async fn get_raw_postgres_connection( + database: &Postgres, + logical_mode: bool, +) -> Result { + let ssl_mode = match database.sslmode.as_ref() { + "disable" => SslMode::Disable, + "" | "prefer" | "allow" => SslMode::Prefer, + "require" => SslMode::Require, + "verify-ca" => SslMode::VerifyCa, + "verify-full" => SslMode::VerifyFull, + ssl_mode => { + return Err(Error::BadRequest( + format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), + )) + } + }; + + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .user(&database.user) + .ssl_mode(ssl_mode); + + if logical_mode { + config.replication_mode(rust_postgres::config::ReplicationMode::Logical); + } + + if let Some(port) = database.port { + config.port(port); + }; + + if !database.password.is_empty() { + config.password(&database.password); + } + + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; + let client = if let Some(connector) = connector { + let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; + tokio::spawn(async move { + tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); + if let Err(e) = connection.await { + tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); + }; + tracing::info!("PostgreSQL trigger connection closed"); + }); + client + } else { + let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + }; + + Ok(client) +} + +pub async fn get_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, + logical_mode: bool, +) -> Result { + let database = + try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) + .await?; + + Ok(get_raw_postgres_connection(&database, logical_mode).await?) +} + +pub async fn get_default_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await +} + +pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { + tx.execute( + &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn check_if_valid_publication_for_postgres_version( + pg_connection: &Client, + table_to_track: Option<&[Relations]>, +) -> Result { + use crate::handler::get_postgres_version_internal; + + let postgres_version = get_postgres_version_internal(pg_connection).await?; + + let pg_14 = postgres_version.starts_with("14"); + if pg_14 { + let unsupported_publication = table_to_track + .and_then(|relations| { + relations.iter().find(|relation| { + let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { + table_to_track.where_clause.is_some() + || table_to_track.columns_name.is_some() + }); + + relation.table_to_track.is_empty() || invalid_relation.is_some() + }) + }) + .is_some(); + + if unsupported_publication { + return Err(Error::BadRequest( + "Your PostgreSQL database is running version 14, which does not support the following publication features: \ + - WHERE clause filtering, \ + - selective column tracking, and \ + - tracking all tables within a schema.\n\ + These features are only available in PostgreSQL 15 and above.".to_string(), + )); + } + } + Ok(pg_14) +} + +pub async fn create_pg_publication( + pg_connection: &Client, + publication_name: &str, + table_to_track: Option<&[Relations]>, + transaction_to_track: &[String], +) -> Result<()> { + let pg_14 = + check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; + let mut query = String::from("CREATE PUBLICATION "); + + query.push_str("e_identifier(publication_name)); + + match table_to_track { + Some(database_component) if !database_component.is_empty() => { + query.push_str(" FOR"); + let mut first = true; + for (i, schema) in database_component.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push_str(" TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); + } else { + if pg_14 && first { + query.push_str(" TABLE ONLY "); + first = false + } else if !pg_14 { + query.push_str(" TABLE ONLY "); + } + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", &schema_name, &table_name); + query.push_str(&full_name); + if let Some(columns) = table.columns_name.as_ref() { + query.push_str(" ("); + let columns = columns + .iter() + .map(|column| quote_identifier(column)) + .join(", "); + query.push_str(&columns); + query.push_str(")"); + } + + if let Some(where_clause) = &table.where_clause { + query.push_str(" WHERE ("); + query.push_str(where_clause); + query.push(')'); + } + + if j + 1 != schema.table_to_track.len() { + query.push_str(", "); + } + } + } + if i < database_component.len() - 1 { + query.push_str(", "); + } + } + } + _ => { + query.push_str(" FOR ALL TABLES "); + } + }; + + if !transaction_to_track.is_empty() { + let transactions = || transaction_to_track.iter().join(", "); + query.push_str(" WITH (publish = '"); + query.push_str(&transactions()); + query.push_str("');"); + } + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { + let mut query = String::from("DROP PUBLICATION IF EXISTS "); + let quoted_publication_name = quote_identifier(publication_name); + query.push_str("ed_publication_name); + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + + Ok(()) +} + +pub fn generate_random_string() -> String { + let timestamp = Utc::now().timestamp_millis().to_string(); + let mut rng = rand::rng(); + let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; + + let random_part = (0..10) + .map(|_| { + charset + .chars() + .nth(rng.random_range(0..charset.len())) + .unwrap() + }) + .collect::(); + + format!("{}_{}", timestamp, random_part) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_valid_publication_data() { + let json = r#"{ + "table_to_track": [ + { + "schema_name": "public", + "table_to_track": [ + {"table_name": "users"} + ] + } + ], + "transaction_to_track": ["insert", "update"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_ok()); + } + + #[test] + fn test_publication_data_empty_schema_name() { + let json = r#"{ + "table_to_track": [ + { + "schema_name": "", + "table_to_track": [] + } + ], + "transaction_to_track": ["insert"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_publication_data_empty_table_name() { + let json = r#"{ + "table_to_track": [ + { + "schema_name": "public", + "table_to_track": [ + {"table_name": " "} + ] + } + ], + "transaction_to_track": ["insert"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_publication_data_invalid_transaction_type() { + let json = r#"{ + "transaction_to_track": ["insert", "truncate"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_publication_data_too_many_transaction_types() { + let json = r#"{ + "transaction_to_track": ["insert", "update", "delete", "insert"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_publication_data_duplicate_schema_names() { + let json = r#"{ + "table_to_track": [ + {"schema_name": "public", "table_to_track": [{"table_name": "a"}]}, + {"schema_name": "public", "table_to_track": [{"table_name": "b"}]} + ], + "transaction_to_track": ["insert"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_publication_data_all_tables_in_schema() { + let json = r#"{ + "table_to_track": [ + {"schema_name": "public", "table_to_track": []} + ], + "transaction_to_track": ["insert"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_ok()); + } + + #[test] + fn test_publication_data_incompatible_tracking() { + let json = r#"{ + "table_to_track": [ + {"schema_name": "schema1", "table_to_track": []}, + {"schema_name": "schema2", "table_to_track": [{"table_name": "t1", "columns_name": ["col1"]}]} + ], + "transaction_to_track": ["insert"] + }"#; + let result: std::result::Result = serde_json::from_str(json); + assert!(result.is_err()); + } + + #[test] + fn test_postgres_config_serialization() { + let config = PostgresConfig { + postgres_resource_path: "f/db/postgres".to_string(), + replication_slot_name: "slot_1".to_string(), + publication_name: "pub_1".to_string(), + basic_mode: Some(false), + }; + let json = serde_json::to_value(&config).unwrap(); + assert_eq!(json["postgres_resource_path"], "f/db/postgres"); + assert_eq!(json["replication_slot_name"], "slot_1"); + } + + #[test] + fn test_generate_random_string_format() { + let s = generate_random_string(); + assert!(s.contains('_')); + let parts: Vec<&str> = s.split('_').collect(); + assert_eq!(parts.len(), 2); + assert_eq!(parts[1].len(), 10); + } + + #[test] + fn test_relations_add_table() { + let mut rel = Relations::new("public".to_string(), vec![]); + rel.add_new_table(TableToTrack::new("users".to_string(), None, None)); + assert_eq!(rel.table_to_track.len(), 1); + assert_eq!(rel.table_to_track[0].table_name, "users"); + } + + #[test] + fn test_table_to_track_with_where_clause() { + let tt = TableToTrack::new( + "orders".to_string(), + Some("status = 'active'".to_string()), + None, + ); + assert_eq!(tt.where_clause, Some("status = 'active'".to_string())); + } + + #[test] + fn test_table_to_track_with_columns() { + let tt = TableToTrack::new( + "users".to_string(), + None, + Some(vec!["id".to_string(), "email".to_string()]), + ); + assert_eq!(tt.columns_name.as_ref().unwrap().len(), 2); + } +} diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-trigger-postgres/src/listener.rs similarity index 95% rename from backend/windmill-api/src/triggers/postgres/listener.rs rename to backend/windmill-trigger-postgres/src/listener.rs index bfb527e3e7..cc71cedd33 100644 --- a/backend/windmill-api/src/triggers/postgres/listener.rs +++ b/backend/windmill-trigger-postgres/src/listener.rs @@ -15,25 +15,18 @@ use windmill_common::{ DB, }; -use crate::{ - resources::try_get_resource_from_db_as, - triggers::{ - listener::ListeningTrigger, - postgres::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, - handler::drop_logical_replication_slot, - relation::RelationConverter, - replication_message::{ - LogicalReplicationMessage::{ - Begin, Commit, Delete, Insert, Relation, Type, Update, - }, - PrimaryKeepAliveBody, ReplicationMessage, - }, - Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, - }, - trigger_helpers::TriggerJobArgs, - Listener, +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}; + +use super::{ + drop_publication, get_default_pg_connection, get_raw_postgres_connection, + handler::drop_logical_replication_slot, + relation::RelationConverter, + replication_message::{ + LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update}, + PrimaryKeepAliveBody, ReplicationMessage, }, + Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, }; const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; diff --git a/backend/windmill-api/src/triggers/postgres/mapper.rs b/backend/windmill-trigger-postgres/src/mapper.rs similarity index 57% rename from backend/windmill-api/src/triggers/postgres/mapper.rs rename to backend/windmill-trigger-postgres/src/mapper.rs index a24fb65378..abd40a2838 100644 --- a/backend/windmill-api/src/triggers/postgres/mapper.rs +++ b/backend/windmill-trigger-postgres/src/mapper.rs @@ -129,8 +129,94 @@ export async function main( ) {{ }} "#, - &struct_definition, - &struct_definition + &struct_definition, &struct_definition ) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_postgres_to_typescript_bool() { + assert_eq!(postgres_to_typescript_type(Some(Type::BOOL)), "boolean"); + } + + #[test] + fn test_postgres_to_typescript_text_types() { + assert_eq!(postgres_to_typescript_type(Some(Type::TEXT)), "string"); + assert_eq!(postgres_to_typescript_type(Some(Type::VARCHAR)), "string"); + assert_eq!(postgres_to_typescript_type(Some(Type::CHAR)), "string"); + } + + #[test] + fn test_postgres_to_typescript_number_types() { + assert_eq!(postgres_to_typescript_type(Some(Type::INT2)), "number"); + assert_eq!(postgres_to_typescript_type(Some(Type::INT4)), "number"); + assert_eq!(postgres_to_typescript_type(Some(Type::INT8)), "number"); + assert_eq!(postgres_to_typescript_type(Some(Type::FLOAT4)), "number"); + assert_eq!(postgres_to_typescript_type(Some(Type::FLOAT8)), "number"); + assert_eq!(postgres_to_typescript_type(Some(Type::NUMERIC)), "number"); + } + + #[test] + fn test_postgres_to_typescript_array_types() { + assert_eq!( + postgres_to_typescript_type(Some(Type::INT4_ARRAY)), + "Array" + ); + assert_eq!( + postgres_to_typescript_type(Some(Type::TEXT_ARRAY)), + "Array" + ); + assert_eq!( + postgres_to_typescript_type(Some(Type::BOOL_ARRAY)), + "Array" + ); + } + + #[test] + fn test_postgres_to_typescript_date_types() { + assert_eq!(postgres_to_typescript_type(Some(Type::DATE)), "string"); + assert_eq!(postgres_to_typescript_type(Some(Type::TIMESTAMP)), "string"); + assert_eq!(postgres_to_typescript_type(Some(Type::TIMESTAMPTZ)), "string"); + assert_eq!(postgres_to_typescript_type(Some(Type::UUID)), "string"); + } + + #[test] + fn test_postgres_to_typescript_json() { + assert_eq!(postgres_to_typescript_type(Some(Type::JSON)), "unknown"); + assert_eq!(postgres_to_typescript_type(Some(Type::JSONB)), "unknown"); + } + + #[test] + fn test_postgres_to_typescript_none() { + assert_eq!(postgres_to_typescript_type(None), "string"); + } + + #[test] + fn test_into_body_struct_typescript() { + let fields = vec![ + MappingInfo::new("id".to_string(), Some(Type::INT4), false), + MappingInfo::new("name".to_string(), Some(Type::TEXT), true), + ]; + let result = into_body_struct(Language::Typescript, fields); + assert!(result.contains("id: number,")); + assert!(result.contains("name?: string,")); + } + + #[test] + fn test_empty_template() { + let mapper = Mapper::new(HashMap::new(), Language::Typescript); + let template = mapper.get_template(); + assert!(template.contains("row: any")); + } + + #[test] + fn test_mapping_info_nullable_field() { + let info = MappingInfo::new("email".to_string(), Some(Type::VARCHAR), true); + assert!(info.is_nullable); + assert_eq!(info.column_name, "email"); + } +} diff --git a/backend/windmill-trigger-postgres/src/relation.rs b/backend/windmill-trigger-postgres/src/relation.rs new file mode 100644 index 0000000000..0907c134cf --- /dev/null +++ b/backend/windmill-trigger-postgres/src/relation.rs @@ -0,0 +1,188 @@ +use core::str; + +use serde_json::{Map, Value}; +use std::{collections::HashMap, str::Utf8Error}; + +use super::{ + converter::{Converter, ConverterError}, + replication_message::{Columns, RelationBody, TupleData}, +}; +use rust_postgres::types::Oid; +#[derive(Debug, thiserror::Error)] +pub enum RelationConversionError { + #[error("Could not find matching table")] + FailToFindMatchingTable, + + #[error("Binary data not supported")] + BinaryFormatNotSupported, + + #[error("decode error: {0}")] + FromBytes(#[from] ConverterError), + + #[error("invalid string value")] + InvalidStr(#[from] Utf8Error), +} + +pub struct RelationConverter(HashMap); + +impl RelationConverter { + pub fn new() -> Self { + Self(HashMap::new()) + } + + pub fn add_relation(&mut self, relation: RelationBody) { + self.0.insert(relation.o_id, relation); + } + + pub fn get_columns(&self, o_id: Oid) -> Result<&Columns, RelationConversionError> { + self.0 + .get(&o_id) + .map(|relation_body| &relation_body.columns) + .ok_or(RelationConversionError::FailToFindMatchingTable) + } + + pub fn get_relation(&self, o_id: Oid) -> Result<&RelationBody, RelationConversionError> { + self.0 + .get(&o_id) + .ok_or(RelationConversionError::FailToFindMatchingTable) + } + + pub fn row_to_json( + &self, + to_decode: (Oid, Vec), + ) -> Result, RelationConversionError> { + let (o_id, tuple_data) = to_decode; + let mut object: Map = Map::new(); + let columns = self.get_columns(o_id)?; + + for (i, column) in columns.iter().enumerate() { + let value = match &tuple_data[i] { + TupleData::Null | TupleData::UnchangedToast => Value::Null, + TupleData::Binary(_) => { + return Err(RelationConversionError::BinaryFormatNotSupported) + } + TupleData::Text(bytes) => { + let str = str::from_utf8(&bytes[..])?; + Converter::try_from_str(column.type_o_id.clone(), str)? + } + }; + + object.insert(column.name.clone(), value); + } + Ok(object) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bytes::Bytes; + use rust_postgres::types::Type; + use serde_json::json; + + use super::super::replication_message::{Column, ReplicaIdentity, RelationBody}; + + fn make_relation(o_id: Oid, columns: Vec) -> RelationBody { + RelationBody::new(None, o_id, "public".to_string(), "test".to_string(), ReplicaIdentity::Default, columns) + } + + fn text_col(name: &str, typ: Option) -> Column { + Column::new(0, name.to_string(), typ, -1) + } + + #[test] + fn test_row_to_json_text_columns() { + let mut converter = RelationConverter::new(); + converter.add_relation(make_relation(1, vec![ + text_col("id", Some(Type::INT4)), + text_col("name", Some(Type::TEXT)), + ])); + + let tuple = vec![ + TupleData::Text(Bytes::from("42")), + TupleData::Text(Bytes::from("Alice")), + ]; + + let result = converter.row_to_json((1, tuple)).unwrap(); + assert_eq!(result["id"], json!(42)); + assert_eq!(result["name"], json!("Alice")); + } + + #[test] + fn test_row_to_json_with_null() { + let mut converter = RelationConverter::new(); + converter.add_relation(make_relation(1, vec![ + text_col("id", Some(Type::INT4)), + text_col("email", Some(Type::TEXT)), + ])); + + let tuple = vec![ + TupleData::Text(Bytes::from("1")), + TupleData::Null, + ]; + + let result = converter.row_to_json((1, tuple)).unwrap(); + assert_eq!(result["id"], json!(1)); + assert_eq!(result["email"], Value::Null); + } + + #[test] + fn test_row_to_json_with_unchanged_toast() { + let mut converter = RelationConverter::new(); + converter.add_relation(make_relation(1, vec![ + text_col("data", Some(Type::TEXT)), + ])); + + let tuple = vec![TupleData::UnchangedToast]; + let result = converter.row_to_json((1, tuple)).unwrap(); + assert_eq!(result["data"], Value::Null); + } + + #[test] + fn test_row_to_json_binary_rejected() { + let mut converter = RelationConverter::new(); + converter.add_relation(make_relation(1, vec![ + text_col("data", Some(Type::BYTEA)), + ])); + + let tuple = vec![TupleData::Binary(Bytes::from("data"))]; + assert!(matches!( + converter.row_to_json((1, tuple)), + Err(RelationConversionError::BinaryFormatNotSupported) + )); + } + + #[test] + fn test_missing_relation() { + let converter = RelationConverter::new(); + let tuple = vec![TupleData::Null]; + assert!(matches!( + converter.row_to_json((999, tuple)), + Err(RelationConversionError::FailToFindMatchingTable) + )); + } + + #[test] + fn test_multiple_relations() { + let mut converter = RelationConverter::new(); + converter.add_relation(make_relation(1, vec![text_col("a", Some(Type::TEXT))])); + converter.add_relation(make_relation(2, vec![text_col("b", Some(Type::INT4))])); + + let result1 = converter.row_to_json((1, vec![TupleData::Text(Bytes::from("hello"))])).unwrap(); + let result2 = converter.row_to_json((2, vec![TupleData::Text(Bytes::from("42"))])).unwrap(); + + assert_eq!(result1["a"], json!("hello")); + assert_eq!(result2["b"], json!(42)); + } + + #[test] + fn test_row_to_json_bool_column() { + let mut converter = RelationConverter::new(); + converter.add_relation(make_relation(1, vec![ + text_col("active", Some(Type::BOOL)), + ])); + + let result = converter.row_to_json((1, vec![TupleData::Text(Bytes::from("t"))])).unwrap(); + assert_eq!(result["active"], json!(true)); + } +} diff --git a/backend/windmill-api/src/triggers/postgres/replication_message.rs b/backend/windmill-trigger-postgres/src/replication_message.rs similarity index 59% rename from backend/windmill-api/src/triggers/postgres/replication_message.rs rename to backend/windmill-trigger-postgres/src/replication_message.rs index 5ae7f8cf2f..43dba75889 100644 --- a/backend/windmill-api/src/triggers/postgres/replication_message.rs +++ b/backend/windmill-trigger-postgres/src/replication_message.rs @@ -508,3 +508,309 @@ impl ReplicationMessage { Ok(replication_message) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn build_keepalive(wal_end: u64, timestamp: i64, reply: bool) -> Bytes { + let mut buf = Vec::new(); + buf.push(PRIMARY_KEEPALIVE_BYTE); + buf.extend_from_slice(&wal_end.to_be_bytes()); + buf.extend_from_slice(×tamp.to_be_bytes()); + buf.push(if reply { 1 } else { 0 }); + Bytes::from(buf) + } + + fn build_xlog_data(wal_start: u64, wal_end: u64, timestamp: i64, data: &[u8]) -> Bytes { + let mut buf = Vec::new(); + buf.push(X_LOG_DATA_BYTE); + buf.extend_from_slice(&wal_start.to_be_bytes()); + buf.extend_from_slice(&wal_end.to_be_bytes()); + buf.extend_from_slice(×tamp.to_be_bytes()); + buf.extend_from_slice(data); + Bytes::from(buf) + } + + #[test] + fn test_parse_keepalive_with_reply() { + let buf = build_keepalive(100, 200, true); + match ReplicationMessage::parse(buf).unwrap() { + ReplicationMessage::PrimaryKeepAlive(body) => { + assert_eq!(body.wal_end, 100); + assert_eq!(body.timestamp, 200); + assert!(body.reply); + } + _ => panic!("expected PrimaryKeepAlive"), + } + } + + #[test] + fn test_parse_keepalive_without_reply() { + let buf = build_keepalive(500, 1000, false); + match ReplicationMessage::parse(buf).unwrap() { + ReplicationMessage::PrimaryKeepAlive(body) => { + assert_eq!(body.wal_end, 500); + assert_eq!(body.timestamp, 1000); + assert!(!body.reply); + } + _ => panic!("expected PrimaryKeepAlive"), + } + } + + #[test] + fn test_parse_xlog_data() { + let payload = b"test payload"; + let buf = build_xlog_data(10, 20, 30, payload); + match ReplicationMessage::parse(buf).unwrap() { + ReplicationMessage::XLogData(body) => { + assert_eq!(body.wal_start, 10); + assert_eq!(body.wal_end, 20); + assert_eq!(body.timestamp, 30); + assert_eq!(&body.data[..], payload); + } + _ => panic!("expected XLogData"), + } + } + + #[test] + fn test_parse_unknown_byte() { + let buf = Bytes::from(vec![0xFF, 0, 0, 0, 0, 0, 0, 0, 0]); + assert!(ReplicationMessage::parse(buf).is_err()); + } + + fn build_begin_message() -> Vec { + let mut buf = Vec::new(); + buf.push(BEGIN_BYTE); + buf.extend_from_slice(&0i64.to_be_bytes()); // lsn + buf.extend_from_slice(&0i64.to_be_bytes()); // timestamp + buf.extend_from_slice(&0i32.to_be_bytes()); // xid + buf + } + + fn build_commit_message() -> Vec { + let mut buf = Vec::new(); + buf.push(COMMIT_BYTE); + buf.push(0); // flags + buf.extend_from_slice(&0u64.to_be_bytes()); // lsn + buf.extend_from_slice(&0u64.to_be_bytes()); // end_lsn + buf.extend_from_slice(&0i64.to_be_bytes()); // timestamp + buf + } + + fn build_insert_message(o_id: u32, tuple_data: &[(u8, &[u8])]) -> Vec { + let mut buf = Vec::new(); + buf.push(INSERT_BYTE); + buf.extend_from_slice(&o_id.to_be_bytes()); + buf.push(TUPLE_NEW_BYTE); + buf.extend_from_slice(&(tuple_data.len() as i16).to_be_bytes()); + for (tag, data) in tuple_data { + buf.push(*tag); + match *tag { + TUPLE_DATA_TEXT_BYTE | TUPLE_DATA_BINARY_BYTE => { + buf.extend_from_slice(&(data.len() as i32).to_be_bytes()); + buf.extend_from_slice(data); + } + _ => {} + } + } + buf + } + + fn build_delete_message(o_id: u32, tuple_data: &[(u8, &[u8])]) -> Vec { + let mut buf = Vec::new(); + buf.push(DELETE_BYTE); + buf.extend_from_slice(&o_id.to_be_bytes()); + buf.push(TUPLE_OLD_BYTE); + buf.extend_from_slice(&(tuple_data.len() as i16).to_be_bytes()); + for (tag, data) in tuple_data { + buf.push(*tag); + match *tag { + TUPLE_DATA_TEXT_BYTE | TUPLE_DATA_BINARY_BYTE => { + buf.extend_from_slice(&(data.len() as i32).to_be_bytes()); + buf.extend_from_slice(data); + } + _ => {} + } + } + buf + } + + fn settings(streaming: bool) -> LogicalReplicationSettings { + LogicalReplicationSettings { streaming } + } + + #[test] + fn test_parse_begin() { + let data = Bytes::from(build_begin_message()); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Begin => {} + other => panic!("expected Begin, got {:?}", other), + } + } + + #[test] + fn test_parse_commit() { + let data = Bytes::from(build_commit_message()); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Commit => {} + other => panic!("expected Commit, got {:?}", other), + } + } + + #[test] + fn test_parse_insert_with_text_tuple() { + let data = Bytes::from(build_insert_message( + 42, + &[(TUPLE_DATA_TEXT_BYTE, b"hello")], + )); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Insert(insert) => { + assert_eq!(insert.o_id, 42); + assert_eq!(insert.tuple.len(), 1); + match &insert.tuple[0] { + TupleData::Text(b) => assert_eq!(&b[..], b"hello"), + other => panic!("expected Text, got {:?}", other), + } + } + other => panic!("expected Insert, got {:?}", other), + } + } + + #[test] + fn test_parse_insert_with_null_tuple() { + let data = Bytes::from(build_insert_message( + 10, + &[(TUPLE_DATA_NULL_BYTE, &[])], + )); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Insert(insert) => { + assert_eq!(insert.tuple.len(), 1); + assert!(matches!(insert.tuple[0], TupleData::Null)); + } + other => panic!("expected Insert, got {:?}", other), + } + } + + #[test] + fn test_parse_insert_with_toast_tuple() { + let data = Bytes::from(build_insert_message( + 10, + &[(TUPLE_DATA_TOAST_BYTE, &[])], + )); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Insert(insert) => { + assert!(matches!(insert.tuple[0], TupleData::UnchangedToast)); + } + other => panic!("expected Insert, got {:?}", other), + } + } + + #[test] + fn test_parse_insert_multiple_columns() { + let data = Bytes::from(build_insert_message( + 1, + &[ + (TUPLE_DATA_TEXT_BYTE, b"col1"), + (TUPLE_DATA_NULL_BYTE, &[]), + (TUPLE_DATA_TEXT_BYTE, b"col3"), + ], + )); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Insert(insert) => { + assert_eq!(insert.tuple.len(), 3); + assert!(matches!(&insert.tuple[0], TupleData::Text(_))); + assert!(matches!(insert.tuple[1], TupleData::Null)); + assert!(matches!(&insert.tuple[2], TupleData::Text(_))); + } + other => panic!("expected Insert, got {:?}", other), + } + } + + #[test] + fn test_parse_delete_with_old_tuple() { + let data = Bytes::from(build_delete_message( + 99, + &[(TUPLE_DATA_TEXT_BYTE, b"old_val")], + )); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Delete(delete) => { + assert_eq!(delete.o_id, 99); + assert!(delete.old_tuple.is_some()); + assert!(delete.key_tuple.is_none()); + } + other => panic!("expected Delete, got {:?}", other), + } + } + + #[test] + fn test_parse_relation() { + let mut buf = Vec::new(); + buf.push(RELATION_BYTE); + buf.extend_from_slice(&100u32.to_be_bytes()); // o_id + buf.extend_from_slice(b"public\0"); // namespace + buf.extend_from_slice(b"users\0"); // name + buf.push(REPLICA_IDENTITY_DEFAULT_BYTE as u8); // replica identity + buf.extend_from_slice(&1i16.to_be_bytes()); // num columns + // column: flags=0, name="id", type_oid=23 (INT4), type_modifier=-1 + buf.push(0); // flags + buf.extend_from_slice(b"id\0"); // name + buf.extend_from_slice(&23u32.to_be_bytes()); // type_oid (INT4) + buf.extend_from_slice(&(-1i32).to_be_bytes()); // type_modifier + + let data = Bytes::from(buf); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(false)).unwrap() { + LogicalReplicationMessage::Relation(rel) => { + assert_eq!(rel.o_id, 100); + assert_eq!(rel.namespace, "public"); + assert_eq!(rel.name, "users"); + assert_eq!(rel.columns.len(), 1); + assert_eq!(rel.columns[0].name, "id"); + assert!(matches!(rel.columns[0].type_o_id, Some(Type::INT4))); + } + other => panic!("expected Relation, got {:?}", other), + } + } + + #[test] + fn test_parse_insert_with_streaming_transaction_id() { + let mut buf = Vec::new(); + buf.push(INSERT_BYTE); + buf.extend_from_slice(&42i32.to_be_bytes()); // transaction_id + buf.extend_from_slice(&10u32.to_be_bytes()); // o_id + buf.push(TUPLE_NEW_BYTE); + buf.extend_from_slice(&0i16.to_be_bytes()); // 0 columns + + let data = Bytes::from(buf); + let body = XLogDataBody::new(0, 0, 0, data); + match body.parse(&settings(true)).unwrap() { + LogicalReplicationMessage::Insert(insert) => { + assert_eq!(insert.transaction_id, Some(42)); + assert_eq!(insert.o_id, 10); + } + other => panic!("expected Insert, got {:?}", other), + } + } + + #[test] + fn test_unknown_tuple_data_byte() { + let mut buf = Vec::new(); + buf.push(INSERT_BYTE); + buf.extend_from_slice(&1u32.to_be_bytes()); // o_id + buf.push(TUPLE_NEW_BYTE); + buf.extend_from_slice(&1i16.to_be_bytes()); // 1 column + buf.push(0xFF); // invalid tuple data byte + + let data = Bytes::from(buf); + let body = XLogDataBody::new(0, 0, 0, data); + assert!(body.parse(&settings(false)).is_err()); + } +} diff --git a/backend/windmill-trigger-sqs/Cargo.toml b/backend/windmill-trigger-sqs/Cargo.toml new file mode 100644 index 0000000000..ac46dc5935 --- /dev/null +++ b/backend/windmill-trigger-sqs/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "windmill-trigger-sqs" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_sqs" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-common/aws_auth", "windmill-common/openidconnect", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["sqs_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +aws-sdk-sqs.workspace = true +aws-sdk-sts.workspace = true +aws-config.workspace = true +aws-credential-types.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +chrono.workspace = true +backon.workspace = true +thiserror.workspace = true +aws-smithy-types.workspace = true \ No newline at end of file diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-trigger-sqs/src/handler_oss.rs similarity index 90% rename from backend/windmill-api/src/triggers/sqs/handler_oss.rs rename to backend/windmill-trigger-sqs/src/handler_oss.rs index 21200e8e38..6340e95cd1 100644 --- a/backend/windmill-api/src/triggers/sqs/handler_oss.rs +++ b/backend/windmill-trigger-sqs/src/handler_oss.rs @@ -5,19 +5,19 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::SqsTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{Trigger, TriggerCrud, TriggerData}, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + error::{Error, Result}, + DB, + }, windmill_git_sync::DeployedObject, + windmill_trigger::{Trigger, TriggerCrud, TriggerData}, }; #[cfg(not(feature = "private"))] #[async_trait] -#[cfg(not(feature = "private"))] impl TriggerCrud for SqsTrigger { type Trigger = Trigger; type TriggerConfig = (); diff --git a/backend/windmill-trigger-sqs/src/lib.rs b/backend/windmill-trigger-sqs/src/lib.rs new file mode 100644 index 0000000000..8c7217725c --- /dev/null +++ b/backend/windmill-trigger-sqs/src/lib.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct SqsTrigger; diff --git a/backend/windmill-api/src/triggers/sqs/listener_oss.rs b/backend/windmill-trigger-sqs/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/sqs/listener_oss.rs rename to backend/windmill-trigger-sqs/src/listener_oss.rs index 0aaf5360c3..7081d67b0f 100644 --- a/backend/windmill-api/src/triggers/sqs/listener_oss.rs +++ b/backend/windmill-trigger-sqs/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::SqsTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for SqsTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Sqs; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for SqsTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for SqsTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-websocket/Cargo.toml b/backend/windmill-trigger-websocket/Cargo.toml new file mode 100644 index 0000000000..6b239d237c --- /dev/null +++ b/backend/windmill-trigger-websocket/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "windmill-trigger-websocket" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_websocket" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +windmill-queue.workspace = true +tokio-tungstenite.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +futures.workspace = true +http.workspace = true diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-trigger-websocket/src/handler.rs similarity index 98% rename from backend/windmill-api/src/triggers/websocket/handler.rs rename to backend/windmill-trigger-websocket/src/handler.rs index bea4f195b2..c50b2dffec 100644 --- a/backend/windmill-api/src/triggers/websocket/handler.rs +++ b/backend/windmill-trigger-websocket/src/handler.rs @@ -1,20 +1,19 @@ use std::borrow::Cow; -use crate::{ - db::{ApiAuthed, DB}, - triggers::{Trigger, TriggerCrud, TriggerData}, -}; use axum::async_trait; use itertools::Itertools; use serde_json::value::RawValue; use sqlx::{types::Json as SqlxJson, PgConnection}; use tokio_tungstenite::connect_async; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; use windmill_common::{ db::UserDB, error::{Error, Result}, worker::to_raw_value, }; use windmill_git_sync::DeployedObject; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ get_url_from_runnable_value, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, @@ -167,7 +166,7 @@ impl TriggerCrud for WebsocketTrigger { // important to update server_id to NULL to stop current websocket listener sqlx::query!( " - UPDATE + UPDATE websocket_trigger SET url = $1, diff --git a/backend/windmill-trigger-websocket/src/lib.rs b/backend/windmill-trigger-websocket/src/lib.rs new file mode 100644 index 0000000000..ca754d8349 --- /dev/null +++ b/backend/windmill-trigger-websocket/src/lib.rs @@ -0,0 +1,126 @@ +use std::collections::HashMap; + +use windmill_api_auth::ApiAuthed; +use windmill_trigger::trigger_helpers::{ + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerKind, TriggerMetadata}, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +pub mod handler; +pub mod listener; + +#[derive(Copy, Clone)] +pub struct WebsocketTrigger; + +impl TriggerJobArgs for WebsocketTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("msg".to_string(), to_raw_value(&payload))]) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WebsocketConfig { + pub url: String, + #[serde(default)] + pub filters: Vec>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_messages: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub url_runnable_args: Option>>, + #[serde(default)] + pub can_return_message: bool, + #[serde(default)] + pub can_return_error_result: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebsocketConfigRequest { + url: String, + filters: Vec, + initial_messages: Option>, + url_runnable_args: Option, + can_return_message: bool, + can_return_error_result: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestWebsocketConfig { + url: String, + url_runnable_args: Option, +} + +pub fn value_to_args_hashmap( + args: Option<&Box>, +) -> Result>> { + let args = if let Some(args) = args { + let args_map: Option> = serde_json::from_str(args.get()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; + + args_map + .unwrap_or_else(HashMap::new) + .into_iter() + .map(|(k, v)| { + let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { + Error::BadRequest(format!("failed to convert to raw value: {}", e)) + })?; + Ok((k, raw_value)) + }) + .collect::>>>() + } else { + Ok(HashMap::new()) + }?; + Ok(args) +} + +pub async fn get_url_from_runnable_value( + path: &str, + is_flow: bool, + db: &DB, + authed: ApiAuthed, + args: Option<&Box>, + workspace_id: &str, +) -> Result { + tracing::info!( + "Running {} {} to get WebSocket URL", + if is_flow { "flow" } else { "script" }, + path + ); + + let args = value_to_args_hashmap(args)?; + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed, + workspace_id, + path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket), + ) + .await?; + + serde_json::from_str::(result.get()).map_err(|_| { + Error::BadConfig(format!( + "{} {} did not return a string", + if is_flow { "Flow" } else { "Script" }, + path, + )) + }) +} diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs similarity index 78% rename from backend/windmill-api/src/triggers/websocket/listener.rs rename to backend/windmill-trigger-websocket/src/listener.rs index 0f22dffdc3..f6fba882a8 100644 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -1,14 +1,4 @@ -use super::WebsocketTrigger; -use crate::triggers::{ - filter::{is_value_superset, Filter, JsonFilter}, - listener::ListeningTrigger, - trigger_helpers::{ - trigger_runnable, trigger_runnable_and_wait_for_raw_result, - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, - }, - websocket::{get_url_from_runnable_value, WebsocketConfig}, - Listener, -}; +use super::{get_url_from_runnable_value, WebsocketConfig, WebsocketTrigger}; use anyhow::Context; use async_trait::async_trait; use futures::{stream::SplitSink, SinkExt, StreamExt}; @@ -28,99 +18,109 @@ use windmill_common::{ DB, }; use windmill_queue::PushArgsOwned; +use windmill_trigger::filter::{is_value_superset, Filter, JsonFilter}; +use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::trigger_helpers::{ + trigger_runnable, trigger_runnable_and_wait_for_raw_result, + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, +}; +use windmill_trigger::Listener; -impl ListeningTrigger { - async fn send_initial_messages( - &self, - writer: &mut SplitSink>, Message>, - db: &DB, - ) -> Result<()> { - let initial_messages: Vec = self - .trigger_config - .initial_messages - .as_deref() - .unwrap_or_default() - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); +async fn send_initial_messages( + listening_trigger: &ListeningTrigger, + writer: &mut SplitSink>, Message>, + db: &DB, +) -> Result<()> { + let initial_messages: Vec = listening_trigger + .trigger_config + .initial_messages + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(); - let WebsocketConfig { ref url, .. } = self.trigger_config; - let runnable_kind = if self.is_flow { "flow" } else { "script" }; - let mut authed_o = None; - for start_message in initial_messages { - match start_message { - InitialMessage::RawMessage(msg) => { - let msg = if msg.starts_with("\"") && msg.ends_with("\"") { - msg[1..msg.len() - 1].to_string() - } else { - msg - }; - tracing::info!( - "Sending raw message initial message to WebSocket {}: {}", - url, - msg - ); - writer - .send(tokio_tungstenite::tungstenite::Message::Text(msg)) - .await - .map_err(to_anyhow) - .with_context(|| "failed to send raw message")?; - } - InitialMessage::RunnableResult { path, is_flow, args } => { - tracing::info!( - "Running {} {} for initial message to WebSocket {}", - runnable_kind, - path, - url, - ); - - let args = raw_value_to_args_hashmap(Some(&args))?; - - if authed_o.is_none() { - authed_o = Some(self.authed(db, "ws").await?); - } - let authed = authed_o.clone().unwrap(); - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed.clone(), - &self.workspace_id, - &path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - TriggerMetadata::new(Some(self.path.to_owned()), JobTriggerKind::Websocket), - ) + let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; + let runnable_kind = if listening_trigger.is_flow { + "flow" + } else { + "script" + }; + let mut authed_o = None; + for start_message in initial_messages { + match start_message { + InitialMessage::RawMessage(msg) => { + let msg = if msg.starts_with("\"") && msg.ends_with("\"") { + msg[1..msg.len() - 1].to_string() + } else { + msg + }; + tracing::info!( + "Sending raw message initial message to WebSocket {}: {}", + url, + msg + ); + writer + .send(tokio_tungstenite::tungstenite::Message::Text(msg)) .await - .map(|r| r.get().to_owned())?; + .map_err(to_anyhow) + .with_context(|| "failed to send raw message")?; + } + InitialMessage::RunnableResult { path, is_flow, args } => { + tracing::info!( + "Running {} {} for initial message to WebSocket {}", + runnable_kind, + path, + url, + ); - tracing::info!( - "Sending {} {} result to WebSocket {}", - runnable_kind, - path, - url - ); + let args = raw_value_to_args_hashmap(Some(&args))?; - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - - writer - .send(tokio_tungstenite::tungstenite::Message::Text(result)) - .await - .map_err(to_anyhow) - .with_context(|| { - format!("Failed to send {} {} result", runnable_kind, path) - })?; + if authed_o.is_none() { + authed_o = Some(listening_trigger.authed(db, "ws").await?); } + let authed = authed_o.clone().unwrap(); + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed.clone(), + &listening_trigger.workspace_id, + &path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + TriggerMetadata::new( + Some(listening_trigger.path.to_owned()), + JobTriggerKind::Websocket, + ), + ) + .await + .map(|r| r.get().to_owned())?; + + tracing::info!( + "Sending {} {} result to WebSocket {}", + runnable_kind, + path, + url + ); + + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + + writer + .send(tokio_tungstenite::tungstenite::Message::Text(result)) + .await + .map_err(to_anyhow) + .with_context(|| format!("Failed to send {} {} result", runnable_kind, path))?; } } - Ok(()) } + Ok(()) } #[async_trait] @@ -205,7 +205,7 @@ impl Listener for WebsocketTrigger { _ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => { return; }, - result = listening_trigger.send_initial_messages(&mut writer, &db) => { + result = send_initial_messages(listening_trigger, &mut writer, &db) => { if let Err(err) = result { self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await; return diff --git a/backend/windmill-trigger/Cargo.toml b/backend/windmill-trigger/Cargo.toml new file mode 100644 index 0000000000..8132f8b62a --- /dev/null +++ b/backend/windmill-trigger/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "windmill-trigger" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-api-jobs/enterprise"] +cloud = ["windmill-common/cloud"] +python = ["dep:windmill-parser-py"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-api-jobs.workspace = true +windmill-api-sse.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +uuid.workspace = true +chrono.workspace = true +http.workspace = true +anyhow.workspace = true +tracing.workspace = true +tokio.workspace = true +async-trait.workspace = true +sql-builder.workspace = true +reqwest.workspace = true +lazy_static.workspace = true +hyper.workspace = true +rand.workspace = true +itertools.workspace = true +windmill-parser.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-py = { workspace = true, optional = true } diff --git a/backend/windmill-trigger/src/capture.rs b/backend/windmill-trigger/src/capture.rs new file mode 100644 index 0000000000..7831adef55 --- /dev/null +++ b/backend/windmill-trigger/src/capture.rs @@ -0,0 +1,127 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use sqlx::types::Json as SqlxJson; +use windmill_common::{ + db::DB, error::Result, triggers::TriggerKind, utils::not_found_if_none, worker::CLOUD_HOSTED, +}; +use windmill_queue::{PushArgs, PushArgsOwned}; + +const KEEP_LAST: i64 = 20; + +struct ActiveCaptureOwner { + owner: String, + email: String, +} + +pub async fn get_active_capture_owner_and_email( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + kind: &TriggerKind, +) -> Result<(String, String)> { + let capture_config = sqlx::query_as!( + ActiveCaptureOwner, + r#" + SELECT + owner, + email + FROM + capture_config + WHERE + workspace_id = $1 + AND path = $2 + AND is_flow = $3 + AND trigger_kind = $4 + AND last_client_ping > NOW() - INTERVAL '10 seconds' + "#, + &w_id, + &path, + is_flow, + kind as &TriggerKind, + ) + .fetch_optional(db) + .await?; + + let capture_config = not_found_if_none( + capture_config, + &format!("capture config for {} trigger", kind), + path, + )?; + + Ok((capture_config.owner, capture_config.email)) +} + +async fn clear_captures_history(db: &DB, w_id: &str) -> Result<()> { + if *CLOUD_HOSTED { + /* Retain only KEEP_LAST most recent captures in this workspace. */ + sqlx::query!( + r#" + DELETE FROM + capture + WHERE + workspace_id = $1 + AND created_at <= ( + SELECT + created_at + FROM + capture + WHERE + workspace_id = $1 + ORDER BY + created_at DESC + OFFSET $2 + LIMIT 1 + ) + "#, + &w_id, + KEEP_LAST, + ) + .execute(db) + .await?; + } + Ok(()) +} + +pub async fn insert_capture_payload( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + trigger_kind: &TriggerKind, + main_args: PushArgsOwned, + preprocessor_args: PushArgsOwned, + owner: &str, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO + capture ( + workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7 + ) + "#, + &w_id, + path, + is_flow, + trigger_kind as &TriggerKind, + SqlxJson(PushArgs { args: &main_args.args, extra: main_args.extra }) as SqlxJson, + SqlxJson(PushArgs { args: &preprocessor_args.args, extra: preprocessor_args.extra }) + as SqlxJson, + owner, + ) + .execute(db) + .await?; + + clear_captures_history(db, &w_id).await?; + + Ok(()) +} diff --git a/backend/windmill-api/src/triggers/filter.rs b/backend/windmill-trigger/src/filter.rs similarity index 62% rename from backend/windmill-api/src/triggers/filter.rs rename to backend/windmill-trigger/src/filter.rs index 553e0a70f5..a1dd59f848 100644 --- a/backend/windmill-api/src/triggers/filter.rs +++ b/backend/windmill-trigger/src/filter.rs @@ -53,13 +53,11 @@ impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { match (json_value, value_to_check) { - (Value::Object(json_map), Value::Object(check_map)) => { - check_map.iter().all(|(k, v)| { - json_map - .get(k) - .map_or(false, |json_val| is_superset(json_val, v)) - }) - } + (Value::Object(json_map), Value::Object(check_map)) => check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }), (Value::Array(json_array), Value::Array(check_array)) => { check_array.iter().all(|check_item| { json_array @@ -141,4 +139,82 @@ mod tests { let result = is_value_superset(&mut deserializer, key, &value).unwrap(); assert!(!result, "Should not match when key doesn't exist"); } + + // --- is_superset unit tests --- + + #[test] + fn test_superset_equal_scalars() { + assert!(is_superset(&json!(42), &json!(42))); + assert!(is_superset(&json!("hello"), &json!("hello"))); + assert!(is_superset(&json!(true), &json!(true))); + assert!(is_superset(&json!(null), &json!(null))); + } + + #[test] + fn test_superset_unequal_scalars() { + assert!(!is_superset(&json!(42), &json!(43))); + assert!(!is_superset(&json!("hello"), &json!("world"))); + assert!(!is_superset(&json!(true), &json!(false))); + } + + #[test] + fn test_superset_object_subset() { + let full = json!({"a": 1, "b": 2, "c": 3}); + let subset = json!({"a": 1, "b": 2}); + assert!(is_superset(&full, &subset)); + } + + #[test] + fn test_superset_object_not_subset() { + let full = json!({"a": 1, "b": 2}); + let check = json!({"a": 1, "b": 3}); + assert!(!is_superset(&full, &check)); + } + + #[test] + fn test_superset_object_missing_key() { + let full = json!({"a": 1}); + let check = json!({"a": 1, "b": 2}); + assert!(!is_superset(&full, &check)); + } + + #[test] + fn test_superset_nested_objects() { + let full = json!({"a": {"b": {"c": 1, "d": 2}, "e": 3}}); + let check = json!({"a": {"b": {"c": 1}}}); + assert!(is_superset(&full, &check)); + } + + #[test] + fn test_superset_array_subset() { + let full = json!([1, 2, 3, 4]); + let check = json!([2, 4]); + assert!(is_superset(&full, &check)); + } + + #[test] + fn test_superset_array_not_subset() { + let full = json!([1, 2, 3]); + let check = json!([4]); + assert!(!is_superset(&full, &check)); + } + + #[test] + fn test_superset_empty_check() { + assert!(is_superset(&json!({"a": 1}), &json!({}))); + assert!(is_superset(&json!([1, 2]), &json!([]))); + } + + #[test] + fn test_superset_array_of_objects() { + let full = json!([{"id": 1, "name": "a"}, {"id": 2, "name": "b"}]); + let check = json!([{"id": 1}]); + assert!(is_superset(&full, &check)); + } + + #[test] + fn test_superset_type_mismatch() { + assert!(!is_superset(&json!(42), &json!("42"))); + assert!(!is_superset(&json!([1]), &json!(1))); + } } diff --git a/backend/windmill-api/src/triggers/global_handler.rs b/backend/windmill-trigger/src/global_handler.rs similarity index 97% rename from backend/windmill-api/src/triggers/global_handler.rs rename to backend/windmill-trigger/src/global_handler.rs index 3143d6806f..8c8ab70d5c 100644 --- a/backend/windmill-api/src/triggers/global_handler.rs +++ b/backend/windmill-trigger/src/global_handler.rs @@ -1,8 +1,4 @@ -use crate::{ - db::{ApiAuthed, DB}, - jobs::cancel_jobs, - triggers::trigger_helpers::trigger_runnable_inner, -}; +use crate::trigger_helpers::trigger_runnable_inner; use axum::{ extract::{Extension, Path}, response::Json, @@ -13,8 +9,10 @@ use serde_json::value::RawValue; use sqlx::PgConnection; use std::collections::HashMap; use uuid::Uuid; +use windmill_api_auth::ApiAuthed; +use windmill_api_jobs::execution::cancel_jobs; use windmill_common::{ - db::UserDB, + db::{UserDB, DB}, error::{self, Error, Result}, jobs::JobTriggerKind, triggers::TriggerMetadata, @@ -62,12 +60,12 @@ async fn get_suspended_trigger( ]; let sql = format!( - r#"SELECT - {} - FROM - {} - WHERE - workspace_id = $1 AND + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND path = $2 "#, fields.join(", "), diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs new file mode 100644 index 0000000000..be9e92228d --- /dev/null +++ b/backend/windmill-trigger/src/handler.rs @@ -0,0 +1,693 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::types::{StandardTriggerQuery, TriggerData, TriggerMode}; +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::{FromRow, PgConnection}; +use std::fmt::Debug; +use windmill_api_auth::{check_scopes, ApiAuthed}; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{paginate, Pagination, StripPath}, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_git_sync::DeployedObject; + +use axum::{ + extract::{Path, Query}, + http::StatusCode, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use std::sync::Arc; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_git_sync::handle_deployment_metadata; + +#[async_trait] +pub trait TriggerCrud: Send + Sync + 'static { + type Trigger: Serialize + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Send + + Sync + + Unpin; + + type TriggerConfig: Debug + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Serialize + + Send + + Sync + + Unpin; + + type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; + type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + + const TABLE_NAME: &'static str; + const TRIGGER_TYPE: &'static str; + const SUPPORTS_SERVER_STATE: bool; + const SUPPORTS_TEST_CONNECTION: bool; + const ROUTE_PREFIX: &'static str; + const DEPLOYMENT_NAME: &'static str; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; + const IS_ALLOWED_ON_CLOUD: bool; + + fn get_deployed_object(path: String) -> DeployedObject; + + async fn validate_new( + &self, + db: &DB, + workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + self.validate_config(db, new, workspace_id).await + } + + async fn validate_edit( + &self, + db: &DB, + workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + self.validate_config(db, edit, workspace_id).await + } + + async fn validate_config( + &self, + _db: &DB, + _config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + Ok(()) + } + + fn scope_domain_name() -> &'static str { + &Self::ROUTE_PREFIX[1..] + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn test_connection( + &self, + _db: &DB, + _authed: &ApiAuthed, + _user_db: &UserDB, + _workspace_id: &str, + _config: Self::TestConnectionConfig, + ) -> Result<()> { + Err( + anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) + .into(), + ) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn get_trigger_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let sql = format!( + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND + path = $2 + "#, + fields.join(", "), + Self::TABLE_NAME + ); + + sqlx::query_as(&sql) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) + } + + async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { + let exists = sqlx::query_scalar(&format!( + "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .fetch_one(db) + .await?; + + Ok(exists) + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) + } + + async fn set_trigger_mode_extra_action(&self, _: &mut PgConnection) -> Result<()> { + Ok(()) + } + + async fn set_trigger_mode( + &self, + authed: &ApiAuthed, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + mode: &TriggerMode, + ) -> Result { + let updated = if Self::SUPPORTS_SERVER_STATE { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = $1, + email = $2, + edited_by = $3, + edited_at = now(), + server_id = NULL, + error = NULL + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(mode) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + } else { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = $1, + email = $2, + edited_by = $3, + edited_at = now() + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(mode) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + }; + + self.set_trigger_mode_extra_action(&mut *tx).await?; + + Ok(updated > 0) + } + + #[allow(unused)] + async fn trigger_count( + &self, + tx: &mut PgConnection, + workspace_id: &str, + is_flow: bool, + script_path: &str, + ) -> i64 { + let count = sqlx::query_scalar(&format!( + r#" + SELECT + COUNT(*) + FROM + {} + WHERE + workspace_id = $1 AND + is_flow = $2 AND + script_path = $3 + "#, + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(is_flow) + .bind(script_path) + .fetch_one(&mut *tx) + .await + .unwrap_or(0); + + count + } + + async fn list_triggers( + &self, + tx: &mut PgConnection, + workspace_id: &str, + query: Option<&StandardTriggerQuery>, + ) -> Result> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&workspace_id)); + + if let Some(query) = query { + let (per_page, offset) = + paginate(Pagination { per_page: query.per_page, page: query.page }); + if let Some(path) = &query.path { + sqlb.and_where_eq("script_path", "?".bind(path)); + } + + if let Some(is_flow) = query.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + + if let Some(path_start) = &query.path_start { + sqlb.and_where_like_left("path", path_start); + } + + sqlb.offset(offset).limit(per_page); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; + + Ok(triggers) + } +} + +pub fn trigger_routes() -> Router { + let mut router = Router::new() + .route("/create", post(create_trigger::)) + .route("/list", get(list_triggers::)) + .route("/get/*path", get(get_trigger::)) + .route("/update/*path", post(update_trigger::)) + .route("/delete/*path", delete(delete_trigger::)) + .route("/exists/*path", get(exists_trigger::)) + .route("/setmode/*path", post(set_trigger_mode::)); + + if T::SUPPORTS_TEST_CONNECTION { + router = router.route("/test", post(test_connection::)); + } + + router +} + +async fn create_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(new_trigger): Json>, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &new_trigger.base.path + ) + })?; + + if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { + return Err(Error::BadRequest(format!( + "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", + T::TRIGGER_TYPE + ))); + } + + handler + .validate_new(&db, &workspace_id, &new_trigger.config) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = new_trigger.base.path.clone(); + + handler + .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.create", T::TRIGGER_TYPE), + ActionKind::Create, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), + true, + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, new_path)) +} + +async fn list_triggers( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path(workspace_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let triggers = handler + .list_triggers(&mut *tx, &workspace_id, Some(&query)) + .await?; + tx.commit().await?; + + Ok(Json(triggers)) +} + +async fn get_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let trigger = handler + .get_trigger_by_path(&mut *tx, &workspace_id, path) + .await?; + + tx.commit().await?; + + Ok(Json(trigger)) +} + +async fn update_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(edit_trigger): Json>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &edit_trigger.base.path + ) + })?; + + handler + .validate_edit(&db, &workspace_id, &edit_trigger.config, path) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = edit_trigger.base.path.to_string(); + + handler + .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.update", T::TRIGGER_TYPE), + ActionKind::Update, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), + true, + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' updated", path)) +} + +async fn delete_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:write:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let deleted = handler + .delete_by_path(&mut *tx, &workspace_id, path) + .await?; + + if !deleted { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.delete", T::TRIGGER_TYPE), + ActionKind::Delete, + &workspace_id, + Some(&path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' deleted", path)) +} + +async fn exists_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), path) + })?; + let exists = handler.exists(&db, &workspace_id, path).await?; + + Ok(Json(exists)) +} + +#[derive(serde::Deserialize)] +struct SetTriggerModePayload { + mode: TriggerMode, +} + +async fn set_trigger_mode( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; + + let mut tx = user_db.begin(&authed).await?; + let updated = handler + .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) + .await?; + + if !updated { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(path.to_owned()), + Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), + true, + None, + ) + .await?; + + Ok(format!( + "Trigger '{}' {}", + path, + if payload.mode == TriggerMode::Enabled { + "enabled" + } else if payload.mode == TriggerMode::Disabled { + "disabled" + } else { + "suspended" + } + )) +} + +async fn test_connection( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(config): Json, +) -> Result<()> { + let connect_f = async move { + handler + .test_connection(&db, &authed, &user_db, &workspace_id, config) + .await + }; + + tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) + .await + .map_err(|_| { + Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) + })??; + Ok(()) +} + +#[allow(unused)] +pub fn complete_trigger_routes(handler: T) -> Router { + let standard_routes = trigger_routes::(); + + let additional_routes = handler.additional_routes(); + + standard_routes + .merge(additional_routes) + .layer(Extension(Arc::new(handler))) +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggerPrimarySchedule { + pub schedule: String, +} + +// generate_trigger_routers(), get_triggers_count_internal(), TriggersCount stay in windmill-api diff --git a/backend/windmill-trigger/src/lib.rs b/backend/windmill-trigger/src/lib.rs new file mode 100644 index 0000000000..690d9cc022 --- /dev/null +++ b/backend/windmill-trigger/src/lib.rs @@ -0,0 +1,19 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod capture; +pub mod filter; +pub mod global_handler; +pub mod handler; +pub mod listener; +pub mod trigger_helpers; +pub mod types; + +pub use handler::TriggerCrud; +pub use listener::Listener; +pub use types::*; diff --git a/backend/windmill-trigger/src/listener.rs b/backend/windmill-trigger/src/listener.rs new file mode 100644 index 0000000000..d09bb5e55a --- /dev/null +++ b/backend/windmill-trigger/src/listener.rs @@ -0,0 +1,864 @@ +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +use crate::{ + capture::insert_capture_payload, + handler::TriggerCrud, + trigger_helpers::{trigger_runnable, TriggerJobArgs}, + types::{Trigger, TriggerErrorHandling, TriggerMode}, +}; +use async_trait::async_trait; +use itertools::Itertools; +use rand::seq::SliceRandom; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sql_builder::SqlBuilder; +use sqlx::{FromRow, Row}; +use tokio::sync::RwLock; +use windmill_api_auth::{fetch_api_authed, ApiAuthed}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerKind, TriggerMetadata}, + utils::report_critical_error, + DB, INSTANCE_NAME, +}; + +#[allow(unused)] +#[async_trait] +pub trait Listener: TriggerCrud + TriggerJobArgs { + type Consumer: Send; + type Extra: Send + Sync; + type ExtraState: Send + Sync; + + const JOB_TRIGGER_KIND: JobTriggerKind; + const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; + const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result>; + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + extra: Option<&Self::ExtraState>, + ); + async fn fetch_enabled_unlistened_triggers( + &self, + db: &DB, + ) -> Result>> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + "error_handler_path", + "error_handler_args", + "retry", + ]; + + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + ); + + for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let triggers = triggers + .into_iter() + .map(|trigger| ListeningTrigger { + path: trigger.base.path, + workspace_id: trigger.base.workspace_id, + is_flow: trigger.base.is_flow, + username: trigger.base.edited_by, + email: trigger.base.email, + script_path: trigger.base.script_path, + trigger_config: trigger.config, + error_handling: Some(trigger.error_handling), + trigger_mode: true, + suspended_mode: trigger.base.mode == TriggerMode::Suspended, + }) + .collect_vec(); + + Ok(triggers) + } + + async fn fetch_unlistened_captures( + &self, + db: &DB, + ) -> Result>> { + let fields = vec![ + "path", + "is_flow", + "workspace_id", + "owner AS username", + "email", + "trigger_config", + ]; + + let mut sqlb = SqlBuilder::select_from("capture_config"); + sqlb.fields(&fields) + .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) + .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") + .and_where("trigger_config IS NOT NULL") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", + ); + + for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb.sql().expect("failed to build SQL"); + + let captures: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let captures = captures + .into_iter() + .map(|capture| ListeningTrigger { + username: capture.username, + path: capture.path, + workspace_id: capture.workspace_id, + script_path: "".to_string(), + email: capture.email, + trigger_config: capture.trigger_config, + trigger_mode: false, + is_flow: capture.is_flow, + error_handling: None, + suspended_mode: false, + }) + .collect_vec(); + + Ok(captures) + } + + async fn get_extra_state(&self) -> Option { + None + } + + async fn cleanup( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _extra: Option<&Self::ExtraState>, + ) -> Result<()> { + Ok(()) + } + + async fn loop_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + status: Arc>>, + error_message: Option, + ) { + update_rw_lock(status.clone(), error_message).await; + loop { + if let None = self + .update_ping(db, listening_trigger, status.read().await.as_deref()) + .await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + + async fn update_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + if listening_trigger.trigger_mode { + self.update_trigger_ping(db, listening_trigger, error).await + } else { + self.update_capture_ping(db, listening_trigger, error).await + } + } + + async fn update_ping_and_loop_ping_status( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + loop_ping_status: Arc>>, + error: Option, + ) -> Option<()> { + // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status + update_rw_lock(loop_ping_status.clone(), error.clone()).await; + if let None = self + .update_ping(db, listening_trigger, error.as_deref()) + .await + { + return None; + } + Some(()) + } + + async fn update_trigger_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar::<_, i32>(&format!( + r#" + UPDATE + {} + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + server_id = $4 AND + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + RETURNING 1 + "#, + Self::TABLE_NAME + )) + .bind(error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .bind(&*INSTANCE_NAME) + .fetch_optional(db) + .await; + + self.handle_ping_result(updated, db, listening_trigger, "trigger") + .await + } + + async fn update_capture_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + server_id = $6 AND + last_client_ping > NOW() - INTERVAL '10 seconds' + RETURNING 1 + "#, + error, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind, + &*INSTANCE_NAME + ) + .fetch_optional(db) + .await + .map(|result| result.flatten()); + + self.handle_ping_result(updated, db, listening_trigger, "capture") + .await + } + + async fn handle_ping_result( + &self, + result: sqlx::Result>, + db: &DB, + listening_trigger: &ListeningTrigger, + entity_type: &str, + ) -> Option<()> { + match result { + Ok(updated) => { + if updated.is_none() { + self.reset_ping_for_restart(db, listening_trigger).await; + tracing::info!( + "{} {} {} changed, disabled, or deleted, stopping...", + Self::TRIGGER_KIND, + entity_type, + listening_trigger.path + ); + return None; + } + } + Err(error) => { + tracing::warn!( + "Error updating ping of {} {} {}: {:?}", + Self::TRIGGER_KIND, + entity_type, + &listening_trigger.path, + error + ); + } + } + + Some(()) + } + + async fn reset_ping_for_restart( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) { + if listening_trigger.trigger_mode { + let _ = sqlx::query(&format!( + r#" + UPDATE + {} + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + server_id IS NULL + "#, + Self::TABLE_NAME + )) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + } else { + let _ = sqlx::query!( + r#" + UPDATE + capture_config + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + is_flow = $3 AND + trigger_kind = $4 AND + server_id IS NULL + "#, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + } + } + + async fn disable_with_error( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: String, + ) { + if listening_trigger.trigger_mode { + let report_status = sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = 'disabled'::TRIGGER_MODE, + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 + "#, + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + + match report_status { + Ok(_) => { + report_critical_error( + format!( + "Disabling {} trigger {} because of error: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + error + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + } + Err(disable_err) => { + report_critical_error( + format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ).await; + } + } + return; + } + + let report_status = sqlx::query!( + r#" + UPDATE + capture_config + SET + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 + "#, + error, + listening_trigger.workspace_id, + listening_trigger.path, + listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + + if let Err(disable_err) = report_status { + tracing::error!( + "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", + Self::TRIGGER_KIND, + listening_trigger.path, + listening_trigger.workspace_id, + disable_err, + error + ) + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + _extra: Option, + ) -> Result<()> { + let args = Self::build_job_args( + &listening_trigger.script_path, + listening_trigger.is_flow, + &listening_trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let (retry, error_handler_path, error_handler_args) = + match listening_trigger.error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + + tracing::debug!( + "Triggering job from {} event {} with args {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + args + ); + + trigger_runnable( + db, + None, + authed, + &listening_trigger.workspace_id, + &listening_trigger.script_path, + listening_trigger.is_flow, + args, + retry, + error_handler_path.as_deref(), + error_handler_args, + format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), + None, + listening_trigger.suspended_mode, + TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND), + ) + .await?; + + Ok(()) + } + + async fn handle_event( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + if listening_trigger.trigger_mode { + if let Err(err) = self + .handle_trigger(db, listening_trigger, payload, trigger_info, extra) + .await + { + report_critical_error( + format!( + "Failed to trigger job from {} event {}: {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + return Err(err); + }; + return Ok(()); + } + + let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); + if let Err(err) = insert_capture_payload( + db, + &listening_trigger.workspace_id, + &listening_trigger.path, + listening_trigger.is_flow, + &Self::TRIGGER_KIND, + main_args, + preprocessor_args, + &listening_trigger.username, + ) + .await + { + tracing::error!("Error inserting capture payload: {:?}", err); + return Err(err); + } + Ok(()) + } +} + +#[allow(unused)] +pub async fn listening( + db: DB, + listener: T, + listening_trigger: ListeningTrigger, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let killpill_rx_consumer = killpill_rx.resubscribe(); + let killpill_rx_get_consumer = killpill_rx.resubscribe(); + + let loop_ping_status = Arc::new(RwLock::new(None)); + let extra_state = listener.get_extra_state().await; + let path = listening_trigger.path.clone(); + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + consumer = { + tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path); + listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) + } => { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path); + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { + tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path); + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = async { + match consumer { + Ok(Some(consumer)) => { + listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; + tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path); + listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; + tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path); + } + Err(error) => { + tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error); + listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; + } + Ok(None) => { + tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path); + } + } + } => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + } + } + } +} + +#[allow(unused)] +pub async fn listen_to_unlistened_events( + listener: T, + db: DB, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) { + let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; + + match unlistend_enabled_triggers { + Ok(mut unlistend_enabled_triggers) => { + unlistend_enabled_triggers.shuffle(&mut rand::rng()); + for trigger in unlistend_enabled_triggers { + let has_lock = sqlx::query_scalar(&format!( + r#" + UPDATE + {} + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + AND workspace_id = $2 + AND path = $3 + AND (last_server_ping IS NULL + OR last_server_ping < now() - INTERVAL '15 seconds' + ) + RETURNING true + "#, + T::TABLE_NAME, + )) + .bind(&*INSTANCE_NAME) + .bind(&trigger.workspace_id) + .bind(&trigger.path) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tracing::info!( + "Spawning new task to listen for {} event", + T::TABLE_NAME + ); + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, trigger, killpill_rx).await } + }); + } else { + tracing::info!( + "{} trigger {} already being listened to", + T::TRIGGER_KIND, + trigger.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for {} trigger {}: {:?}", + T::TRIGGER_KIND, + trigger.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); + } + } + + let unlisted_captures = listener.fetch_unlistened_captures(&db).await; + + match unlisted_captures { + Ok(unlistened_captures) => { + for capture in unlistened_captures { + let has_lock = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + last_client_ping > NOW() - INTERVAL '10 seconds' AND + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') + RETURNING true + "#, + *INSTANCE_NAME, + &capture.workspace_id, + &capture.path, + &capture.is_flow, + T::TRIGGER_KIND as TriggerKind + ) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, capture, killpill_rx).await } + }); + } else { + tracing::info!( + "{} capture {} already being listened to", + T::TRIGGER_KIND.to_string(), + capture.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for capture {} {}: {:?}", + T::TRIGGER_KIND, + capture.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!( + "Error fetching captures {} triggers: {:?}", + T::TRIGGER_KIND, + err + ); + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + path: String, + is_flow: bool, + workspace_id: String, + username: String, + email: String, + #[serde(flatten)] + trigger_config: T, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let trigger_config_value = row.try_get("trigger_config")?; + let trigger_config: T = serde_json::from_value(trigger_config_value) + .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; + Ok(Capture { + path: row.try_get("path")?, + is_flow: row.try_get("is_flow")?, + workspace_id: row.try_get("workspace_id")?, + username: row.try_get("username")?, + email: row.try_get("email")?, + trigger_config, + }) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ListeningTrigger { + pub path: String, + pub is_flow: bool, + pub workspace_id: String, + pub username: String, + pub email: String, + pub trigger_config: T, + pub script_path: String, + pub trigger_mode: bool, + pub error_handling: Option, + pub suspended_mode: bool, +} + +impl ListeningTrigger { + pub async fn authed(&self, db: &DB, username: &str) -> Result { + fetch_api_authed( + self.username.clone(), + self.email.clone(), + &self.workspace_id, + db, + Some(format!("{}-{}", username, self.path)), + ) + .await + } +} + +#[allow(unused)] +pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { + let mut w = lock.write().await; + *w = value; +} + +#[allow(unused)] +fn listen_to( + trigger: T, + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tokio::spawn(async move { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await + } + } + } + }); +} + +// start_all_listeners() stays in windmill-api since it references concrete trigger types diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-trigger/src/trigger_helpers.rs similarity index 98% rename from backend/windmill-api/src/triggers/trigger_helpers.rs rename to backend/windmill-trigger/src/trigger_helpers.rs index c533dcaaeb..b50cdb69a3 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-trigger/src/trigger_helpers.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::future::Future; use uuid::Uuid; use windmill_common::{ - db::{UserDB, UserDbWithAuthed}, + db::{UserDB, UserDbWithAuthed, DB}, error::Result, flows::{FlowModuleValue, Retry}, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, @@ -19,22 +19,21 @@ use windmill_common::{ RUNNABLE_FORMAT_VERSION_CACHE, }, users::username_to_permissioned_as, - utils::StripPath, + utils::{StripPath, HTTP_CLIENT}, worker::to_raw_value, }; use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; +use windmill_api_auth::{check_scopes, ApiAuthed}; #[cfg(feature = "enterprise")] -use crate::jobs::check_license_key_valid; -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ +use windmill_api_jobs::execution::check_license_key_valid; +use windmill_api_jobs::{ + execution::{ check_tag_available_for_workspace, delete_job_metadata_after_use, push_flow_job_by_path_into_queue, push_script_job_by_path_into_queue, result_to_response, - run_wait_result_internal, RunJobQuery, + run_wait_result_internal, }, - utils::check_scopes, - HTTP_CLIENT, + types::RunJobQuery, }; struct ScriptInfo { @@ -211,6 +210,7 @@ fn get_preprocessor_args_from_content_and_language( )?; Some(args.args) } + #[cfg(feature = "python")] ScriptLang::Python3 => { let args = windmill_parser_py::parse_python_signature( &content, diff --git a/backend/windmill-trigger/src/types.rs b/backend/windmill-trigger/src/types.rs new file mode 100644 index 0000000000..1b471ab30c --- /dev/null +++ b/backend/windmill-trigger/src/types.rs @@ -0,0 +1,338 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use std::{collections::HashMap, fmt::Debug}; +use windmill_common::jobs::JobTriggerKind; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HandlerAction { + Trigger { path: String, trigger_kind: JobTriggerKind }, + // Future variants can be added here (e.g., Script, Flow, etc.) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StandardTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct BaseTrigger { + pub workspace_id: String, + pub path: String, + pub script_path: String, + pub mode: TriggerMode, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: DateTime, + pub extra_perms: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct ServerState { + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct TriggerErrorHandling { + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option>, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + #[serde(flatten)] + pub base: BaseTrigger, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub server_state: Option, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let base = BaseTrigger::from_row(row)?; + + Ok(Trigger { + base, + config: T::from_row(row)?, + server_state: ServerState::from_row(row).ok(), + error_handling: TriggerErrorHandling::from_row(row)?, + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BaseTriggerData { + pub path: String, + pub script_path: String, + pub is_flow: bool, + #[deprecated(note = "Use mode instead")] + enabled: Option, // Kept for backwards compatibility, use mode instead + mode: Option, +} + +impl BaseTriggerData { + pub fn mode(&self) -> &TriggerMode { + self.mode.as_ref().unwrap_or( + #[allow(deprecated)] + if self.enabled.unwrap_or(true) { + &TriggerMode::Enabled + } else { + &TriggerMode::Disabled + }, + ) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TriggerData { + #[serde(flatten)] + pub base: BaseTriggerData, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl StandardTriggerQuery { + pub fn offset(&self) -> i64 { + let page = self.page.unwrap_or(0); + let per_page = self.per_page.unwrap_or(100); + (page * per_page) as i64 + } + + pub fn limit(&self) -> i64 { + self.per_page.unwrap_or(100) as i64 + } +} + +impl Default for StandardTriggerQuery { + fn default() -> Self { + Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] +#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum TriggerMode { + Enabled, + Disabled, + Suspended, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + // --- TriggerMode serde --- + + #[test] + fn test_trigger_mode_serialize() { + assert_eq!(serde_json::to_value(TriggerMode::Enabled).unwrap(), json!("enabled")); + assert_eq!(serde_json::to_value(TriggerMode::Disabled).unwrap(), json!("disabled")); + assert_eq!(serde_json::to_value(TriggerMode::Suspended).unwrap(), json!("suspended")); + } + + #[test] + fn test_trigger_mode_deserialize() { + let enabled: TriggerMode = serde_json::from_value(json!("enabled")).unwrap(); + assert_eq!(enabled, TriggerMode::Enabled); + let disabled: TriggerMode = serde_json::from_value(json!("disabled")).unwrap(); + assert_eq!(disabled, TriggerMode::Disabled); + } + + #[test] + fn test_trigger_mode_invalid() { + let result: Result = serde_json::from_value(json!("paused")); + assert!(result.is_err()); + } + + // --- StandardTriggerQuery --- + + #[test] + fn test_query_default() { + let q = StandardTriggerQuery::default(); + assert_eq!(q.offset(), 0); + assert_eq!(q.limit(), 100); + } + + #[test] + fn test_query_offset_calculation() { + let q = StandardTriggerQuery { + page: Some(2), + per_page: Some(50), + path: None, + is_flow: None, + path_start: None, + }; + assert_eq!(q.offset(), 100); + assert_eq!(q.limit(), 50); + } + + #[test] + fn test_query_offset_defaults() { + let q = StandardTriggerQuery { + page: None, + per_page: None, + path: None, + is_flow: None, + path_start: None, + }; + assert_eq!(q.offset(), 0); + assert_eq!(q.limit(), 100); + } + + // --- BaseTriggerData backward compatibility --- + + #[test] + fn test_base_trigger_data_mode_field() { + let json = r#"{ + "path": "test", + "script_path": "f/test/script", + "is_flow": false, + "mode": "enabled" + }"#; + let data: BaseTriggerData = serde_json::from_str(json).unwrap(); + assert_eq!(data.mode(), &TriggerMode::Enabled); + } + + #[test] + fn test_base_trigger_data_legacy_enabled_true() { + let json = r#"{ + "path": "test", + "script_path": "f/test/script", + "is_flow": false, + "enabled": true + }"#; + let data: BaseTriggerData = serde_json::from_str(json).unwrap(); + assert_eq!(data.mode(), &TriggerMode::Enabled); + } + + #[test] + fn test_base_trigger_data_legacy_enabled_false() { + let json = r#"{ + "path": "test", + "script_path": "f/test/script", + "is_flow": false, + "enabled": false + }"#; + let data: BaseTriggerData = serde_json::from_str(json).unwrap(); + assert_eq!(data.mode(), &TriggerMode::Disabled); + } + + #[test] + fn test_base_trigger_data_mode_takes_precedence() { + let json = r#"{ + "path": "test", + "script_path": "f/test/script", + "is_flow": false, + "mode": "suspended", + "enabled": true + }"#; + let data: BaseTriggerData = serde_json::from_str(json).unwrap(); + assert_eq!(data.mode(), &TriggerMode::Suspended); + } + + #[test] + fn test_base_trigger_data_neither_field() { + let json = r#"{ + "path": "test", + "script_path": "f/test/script", + "is_flow": false + }"#; + let data: BaseTriggerData = serde_json::from_str(json).unwrap(); + assert_eq!(data.mode(), &TriggerMode::Enabled); + } + + // --- HandlerAction --- + + #[test] + fn test_handler_action_serialization() { + let action = HandlerAction::Trigger { + path: "f/test/trigger".to_string(), + trigger_kind: JobTriggerKind::Webhook, + }; + let json = serde_json::to_value(&action).unwrap(); + assert_eq!(json["type"], "trigger"); + assert_eq!(json["path"], "f/test/trigger"); + } + + #[test] + fn test_handler_action_deserialization() { + let json = r#"{"type": "trigger", "path": "f/test/trigger", "trigger_kind": "webhook"}"#; + let action: HandlerAction = serde_json::from_str(json).unwrap(); + match action { + HandlerAction::Trigger { path, trigger_kind } => { + assert_eq!(path, "f/test/trigger"); + assert_eq!( + serde_json::to_value(&trigger_kind).unwrap(), + serde_json::to_value(&JobTriggerKind::Webhook).unwrap() + ); + } + } + } + + // --- ServerState --- + + #[test] + fn test_server_state_skip_none_fields() { + let state = ServerState { + server_id: None, + last_server_ping: None, + error: None, + }; + let json = serde_json::to_value(&state).unwrap(); + assert!(!json.as_object().unwrap().contains_key("server_id")); + assert!(!json.as_object().unwrap().contains_key("error")); + } + + #[test] + fn test_server_state_with_error() { + let state = ServerState { + server_id: Some("srv-1".to_string()), + last_server_ping: None, + error: Some("connection timeout".to_string()), + }; + let json = serde_json::to_value(&state).unwrap(); + assert_eq!(json["server_id"], "srv-1"); + assert_eq!(json["error"], "connection timeout"); + } +} diff --git a/backend/windmill-types/Cargo.toml b/backend/windmill-types/Cargo.toml new file mode 100644 index 0000000000..4eb18fe88e --- /dev/null +++ b/backend/windmill-types/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "windmill-types" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_types" +path = "src/lib.rs" + +[dependencies] +serde.workspace = true +serde_json.workspace = true +chrono.workspace = true +uuid.workspace = true +sqlx = { workspace = true, features = ["postgres"] } +rand.workspace = true +hex.workspace = true +anyhow.workspace = true +tracing.workspace = true +itertools.workspace = true +strum.workspace = true diff --git a/backend/windmill-types/src/apps.rs b/backend/windmill-types/src/apps.rs new file mode 100644 index 0000000000..1008396bb4 --- /dev/null +++ b/backend/windmill-types/src/apps.rs @@ -0,0 +1,30 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::scripts::ScriptLang; + +/// Id in the `app_script` table. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] +#[serde(transparent)] +pub struct AppScriptId(pub i64); + +#[derive(Deserialize)] +pub struct ListAppQuery { + pub starred_only: Option, + pub path_exact: Option, + pub path_start: Option, + pub include_draft_only: Option, + pub with_deployment_msg: Option, +} + +#[derive(Deserialize)] +pub struct RawAppValue { + pub files: HashMap, +} + +pub struct AppInlineScript { + pub language: Option, + pub content: String, + pub lock: Option, +} diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs new file mode 100644 index 0000000000..5f75388410 --- /dev/null +++ b/backend/windmill-types/src/assets.rs @@ -0,0 +1,83 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive( + Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, +)] +#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetKind { + S3Object, + Resource, + // Avoid unnexpected crashes when deserializing old assets + Variable, // Deprecated + Ducklake, + DataTable, +} + +#[derive( + Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, +)] +#[sqlx(type_name = "ASSET_USAGE_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageKind { + Script, + Flow, + Job, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_ACCESS_TYPE", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageAccessType { + R, + W, + RW, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +pub struct AssetWithAltAccessType { + pub path: String, + pub kind: AssetKind, + pub access_type: Option, + pub alt_access_type: Option, + /// Map of column name to access type for column-level access tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub columns: Option>, +} + +pub fn merge_asset_usage_access_types( + a: Option, + b: Option, +) -> Option { + use AssetUsageAccessType::*; + match (a, b) { + (None, _) | (_, None) => None, + (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), + (Some(RW), _) | (_, Some(RW)) => Some(RW), + (Some(R), Some(R)) => Some(R), + (Some(W), Some(W)) => Some(W), + } +} + +pub fn merge_asset_columns( + a: &Option>, + b: &Option>, +) -> Option> { + match (a, b) { + (None, None) => None, + (Some(cols), None) | (None, Some(cols)) => Some(cols.clone()), + (Some(cols_a), Some(cols_b)) => { + let mut merged = cols_a.clone(); + for (col, access_b) in cols_b { + let access_a = merged.get(col); + let merged_access = + merge_asset_usage_access_types(access_a.cloned(), Some(*access_b)); + if let Some(access) = merged_access { + merged.insert(col.clone(), access); + } + } + Some(merged) + } + } +} diff --git a/backend/windmill-types/src/flow_status.rs b/backend/windmill-types/src/flow_status.rs new file mode 100644 index 0000000000..6306141ab9 --- /dev/null +++ b/backend/windmill-types/src/flow_status.rs @@ -0,0 +1,531 @@ +use std::collections::HashMap; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::flows::FlowValue; + +const MINUTES: Duration = Duration::from_secs(60); +const HOURS: Duration = MINUTES.saturating_mul(60); + +pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX; +pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); + +pub fn is_retry_default(v: &RetryStatus) -> bool { + v.fail_count == 0 && v.failed_jobs.is_empty() +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlowStatus { + pub step: i32, + pub modules: Vec, + pub failure_module: Box, + pub preprocessor_module: Option, + + #[serde(skip_serializing_if = "HashMap::is_empty")] + #[serde(default)] + pub user_states: HashMap, + #[serde(default)] + pub cleanup_module: FlowCleanupModule, + #[serde(default)] + #[serde(skip_serializing_if = "is_retry_default")] + pub retry: RetryStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub approval_conditions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub restarted_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_input_enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_id: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct RetryStatus { + pub fail_count: u32, + pub failed_jobs: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct ApprovalConditions { + pub user_auth_required: bool, + pub user_groups_required: Vec, + pub self_approval_disabled: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[serde(default)] +pub struct RestartedFrom { + pub flow_job_id: Uuid, + pub step_id: String, + pub branch_or_iteration_n: Option, + pub flow_version: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Iterator { + pub index: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub itered: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub itered_len: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BranchAllStatus { + pub branch: usize, + pub len: usize, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum BranchChosen { + Default, + Branch { branch: usize }, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Approval { + pub resume_id: u16, + pub approver: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlowStatusModuleWParent { + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_module: Option, + #[serde(flatten)] + pub module_status: FlowStatusModule, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct FlowCleanupModule { + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub flow_jobs_to_clean: Vec, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowJobsDuration { + pub started_at: Vec>>, + pub duration_ms: Vec>, +} + +impl FlowJobsDuration { + pub fn set(&mut self, position: Option, value: &Option) { + if let Some(position) = position { + if position >= self.started_at.len() + || position >= self.duration_ms.len() + || value.is_none() + { + return; + } + let value = value.clone().unwrap(); + self.started_at[position] = Some(value.started_at); + self.duration_ms[position] = Some(value.duration_ms); + } + } + + pub fn push(&mut self, value: &Option) { + self.started_at.push(value.as_ref().map(|x| x.started_at)); + self.duration_ms.push(value.as_ref().map(|x| x.duration_ms)); + } + + pub fn new(n: usize) -> Self { + Self { started_at: vec![None; n], duration_ms: vec![None; n] } + } + + pub fn truncate(&mut self, n: usize) { + self.started_at.truncate(n); + self.duration_ms.truncate(n); + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FlowJobDuration { + pub started_at: chrono::DateTime, + pub duration_ms: i64, +} + +#[derive(Deserialize)] +struct UntaggedFlowStatusModule { + #[serde(rename = "type")] + type_: String, + id: Option, + count: Option, + progress: Option, + job: Option, + iterator: Option, + flow_jobs: Option>, + flow_jobs_success: Option>>, + flow_jobs_duration: Option, + branch_chosen: Option, + branchall: Option, + parallel: Option, + while_loop: Option, + approvers: Option>, + failed_retries: Option>, + skipped: Option, + agent_actions: Option>, + agent_actions_success: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AgentAction { + ToolCall { + job_id: uuid::Uuid, + function_name: String, + module_id: String, + }, + McpToolCall { + call_id: uuid::Uuid, + function_name: String, + resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + arguments: Option, + }, + Message {}, + WebSearch {}, +} + +#[derive(Serialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum FlowStatusModule { + WaitingForPriorSteps { + id: String, + }, + WaitingForEvents { + id: String, + count: u16, + job: Uuid, + }, + WaitingForExecutor { + id: String, + job: Uuid, + }, + InProgress { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + progress: Option, + #[serde(skip_serializing_if = "Option::is_none")] + iterator: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_success: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branchall: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + parallel: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + while_loop: bool, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions_success: Option>, + }, + Success { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_success: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + approvers: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + failed_retries: Vec, + skipped: bool, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions_success: Option>, + }, + Failure { + id: String, + job: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_success: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + flow_jobs_duration: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch_chosen: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + failed_retries: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + agent_actions_success: Option>, + }, +} + +impl<'de> Deserialize<'de> for FlowStatusModule { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let untagged: UntaggedFlowStatusModule = + UntaggedFlowStatusModule::deserialize(deserializer)?; + + match untagged.type_.as_str() { + "WaitingForPriorSteps" => Ok(FlowStatusModule::WaitingForPriorSteps { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + }), + "WaitingForEvents" => Ok(FlowStatusModule::WaitingForEvents { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + count: untagged + .count + .ok_or_else(|| serde::de::Error::missing_field("count"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + }), + "WaitingForExecutor" => Ok(FlowStatusModule::WaitingForExecutor { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + }), + "InProgress" => Ok(FlowStatusModule::InProgress { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + iterator: untagged.iterator, + flow_jobs: untagged.flow_jobs, + flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, + branch_chosen: untagged.branch_chosen, + branchall: untagged.branchall, + parallel: untagged.parallel.unwrap_or(false), + while_loop: untagged.while_loop.unwrap_or(false), + progress: untagged.progress, + agent_actions: untagged.agent_actions, + agent_actions_success: untagged.agent_actions_success, + }), + "Success" => Ok(FlowStatusModule::Success { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + flow_jobs: untagged.flow_jobs, + flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, + branch_chosen: untagged.branch_chosen, + approvers: untagged.approvers.unwrap_or_default(), + failed_retries: untagged.failed_retries.unwrap_or_default(), + skipped: untagged.skipped.unwrap_or(false), + agent_actions: untagged.agent_actions, + agent_actions_success: untagged.agent_actions_success, + }), + "Failure" => Ok(FlowStatusModule::Failure { + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + job: untagged + .job + .ok_or_else(|| serde::de::Error::missing_field("job"))?, + flow_jobs: untagged.flow_jobs, + flow_jobs_success: untagged.flow_jobs_success, + flow_jobs_duration: untagged.flow_jobs_duration, + branch_chosen: untagged.branch_chosen, + failed_retries: untagged.failed_retries.unwrap_or_default(), + agent_actions: untagged.agent_actions, + agent_actions_success: untagged.agent_actions_success, + }), + other => Err(serde::de::Error::unknown_variant( + other, + &[ + "WaitingForPriorSteps", + "WaitingForEvents", + "WaitingForExecutor", + "InProgress", + "Success", + "Failure", + ], + )), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum JobResult { + SingleJob(Uuid), + ListJob(Vec), +} + +impl FlowStatusModule { + pub fn job(&self) -> Option { + match self { + FlowStatusModule::WaitingForPriorSteps { .. } => None, + FlowStatusModule::WaitingForEvents { job, .. } => Some(*job), + FlowStatusModule::WaitingForExecutor { job, .. } => Some(*job), + FlowStatusModule::InProgress { job, .. } => Some(*job), + FlowStatusModule::Success { job, .. } => Some(*job), + FlowStatusModule::Failure { job, .. } => Some(*job), + } + } + + pub fn flow_jobs(&self) -> Option> { + match self { + FlowStatusModule::InProgress { flow_jobs, .. } => flow_jobs.clone(), + FlowStatusModule::Success { flow_jobs, .. } => flow_jobs.clone(), + FlowStatusModule::Failure { flow_jobs, .. } => flow_jobs.clone(), + _ => None, + } + } + + pub fn branch_chosen(&self) -> Option { + match self { + FlowStatusModule::InProgress { branch_chosen, .. } => branch_chosen.clone(), + FlowStatusModule::Success { branch_chosen, .. } => branch_chosen.clone(), + FlowStatusModule::Failure { branch_chosen, .. } => branch_chosen.clone(), + _ => None, + } + } + + pub fn flow_jobs_success(&self) -> Option>> { + match self { + FlowStatusModule::InProgress { flow_jobs_success, .. } => flow_jobs_success.clone(), + FlowStatusModule::Success { flow_jobs_success, .. } => flow_jobs_success.clone(), + FlowStatusModule::Failure { flow_jobs_success, .. } => flow_jobs_success.clone(), + _ => None, + } + } + + pub fn flow_jobs_duration(&self) -> Option { + match self { + FlowStatusModule::InProgress { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + FlowStatusModule::Success { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + FlowStatusModule::Failure { flow_jobs_duration, .. } => flow_jobs_duration.clone(), + _ => None, + } + } + + pub fn job_result(&self) -> Option { + self.flow_jobs() + .map(JobResult::ListJob) + .or_else(|| self.job().map(JobResult::SingleJob)) + } + + pub fn id(&self) -> String { + match self { + FlowStatusModule::WaitingForPriorSteps { id, .. } => id.clone(), + FlowStatusModule::WaitingForEvents { id, .. } => id.clone(), + FlowStatusModule::WaitingForExecutor { id, .. } => id.clone(), + FlowStatusModule::InProgress { id, .. } => id.clone(), + FlowStatusModule::Success { id, .. } => id.clone(), + FlowStatusModule::Failure { id, .. } => id.clone(), + } + } + + pub fn is_failure(&self) -> bool { + match self { + FlowStatusModule::Failure { .. } => true, + _ => false, + } + } + + pub fn agent_actions(&self) -> Option> { + match self { + FlowStatusModule::InProgress { agent_actions, .. } => agent_actions.clone(), + FlowStatusModule::Success { agent_actions, .. } => agent_actions.clone(), + FlowStatusModule::Failure { agent_actions, .. } => agent_actions.clone(), + _ => None, + } + } + + pub fn agent_actions_success(&self) -> Option> { + match self { + FlowStatusModule::InProgress { agent_actions_success, .. } => { + agent_actions_success.clone() + } + FlowStatusModule::Success { agent_actions_success, .. } => { + agent_actions_success.clone() + } + FlowStatusModule::Failure { agent_actions_success, .. } => { + agent_actions_success.clone() + } + _ => None, + } + } +} + +impl FlowStatus { + pub fn new(f: &FlowValue) -> Self { + Self { + step: if f.preprocessor_module.is_some() { + -1 + } else { + 0 + }, + approval_conditions: None, + modules: f + .modules + .iter() + .map(|m| FlowStatusModule::WaitingForPriorSteps { id: m.id.clone() }) + .collect(), + failure_module: Box::new(FlowStatusModuleWParent { + parent_module: None, + module_status: FlowStatusModule::WaitingForPriorSteps { + id: f + .failure_module + .as_ref() + .map(|x| x.id.clone()) + .unwrap_or_else(|| "failure".to_string()), + }, + }), + preprocessor_module: if f.preprocessor_module.is_some() { + Some(FlowStatusModule::WaitingForPriorSteps { + id: f.preprocessor_module.as_ref().unwrap().id.clone(), + }) + } else { + None + }, + cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, + retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, + restarted_from: None, + user_states: HashMap::new(), + stream_job: None, + chat_input_enabled: f.chat_input_enabled, + memory_id: None, + } + } + + /// current module status ... excluding failure_module + pub fn current_step(&self) -> Option<&FlowStatusModule> { + let i = usize::try_from(self.step).ok()?; + self.modules.get(i) + } +} diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs new file mode 100644 index 0000000000..a1a2038452 --- /dev/null +++ b/backend/windmill-types/src/flows.rs @@ -0,0 +1,1123 @@ +use std::{ + collections::{BTreeMap, HashMap}, + time::Duration, +}; + +use rand::Rng; +use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use sqlx::types::JsonRawValue; + +use crate::{ + assets::AssetWithAltAccessType, + more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, + runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, + scripts::{Schema, ScriptHash, ScriptLang}, + to_raw_value, +}; + +#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] +pub struct Flow { + pub workspace_id: String, + pub path: String, + pub summary: String, + pub description: String, + pub value: Json>, + pub edited_by: String, + pub edited_at: chrono::DateTime, + pub archived: bool, + pub schema: Option, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dedicated_worker: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tag: Option, + #[serde(skip_serializing_if = "is_none_or_false")] + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "is_none_or_false")] + pub visible_to_runner_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_behalf_of_email: Option, +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct FlowWithStarred { + #[sqlx(flatten)] + #[serde(flatten)] + pub flow: Flow, + #[serde(skip_serializing_if = "Option::is_none")] + pub starred: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub lock_error_logs: Option, + pub version_id: i64, +} + +pub fn is_none_or_false(b: &Option) -> bool { + b.is_none() || !b.unwrap() +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct ListableFlow { + pub workspace_id: String, + pub path: String, + pub summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub edited_by: Option, + pub edited_at: Option>, + pub archived: bool, + pub extra_perms: serde_json::Value, + pub starred: bool, + pub has_draft: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_error_handler_muted: Option, + #[sqlx(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, +} + +#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] +pub struct NewFlow { + pub path: String, + pub summary: String, + pub description: Option, + #[serde(deserialize_with = "validate_flow_value")] + pub value: Box, + pub schema: Option, + pub draft_only: Option, + pub tag: Option, + pub dedicated_worker: Option, + pub timeout: Option, + pub deployment_message: Option, + pub visible_to_runner_only: Option, + pub on_behalf_of_email: Option, + pub ws_error_handler_muted: Option, +} + +impl NewFlow { + pub fn parse_flow_value(&self) -> anyhow::Result { + serde_json::from_str(self.value.get()) + .map_err(|e| anyhow::anyhow!("Failed to parse flow value: {}", e)) + } +} + +fn validate_retry(retry: &Retry, module_id: &str) -> anyhow::Result<()> { + if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { + return Err(anyhow::anyhow!( + "Module '{}': Exponential backoff base (seconds) must be greater than 0. A base of 0 would cause immediate retries.", + module_id + )); + } + Ok(()) +} + +fn validate_flow_value<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let raw_value = Box::::deserialize(deserializer)?; + + let flow_value: FlowValue = serde_json::from_str(raw_value.get()) + .map_err(|e| serde::de::Error::custom(format!("Invalid flow value: {}", e)))?; + + FlowModule::traverse_modules(&flow_value.modules, &mut |module| { + if let Some(ref retry) = module.retry { + validate_retry(retry, &module.id)?; + } + return Ok(()); + }) + .map_err(|e| serde::de::Error::custom(e.to_string()))?; + + if let Some(ref _failure_module) = flow_value.failure_module { + //add validation logic here for failure module + } + + if let Some(ref _preprocessor_module) = flow_value.preprocessor_module { + //add validation logic here for preprocessor module + } + + Ok(raw_value) +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct FlowValue { + pub modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub failure_module: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub preprocessor_module: Option>, + #[serde(default)] + #[serde(skip_serializing_if = "is_default")] + pub same_worker: bool, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_expr: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub early_return: Option, + #[serde(skip_serializing_if = "Option::is_none")] + // Priority at the flow level + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_input_enabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_env: Option>>, +} + +impl FlowValue { + pub fn get_flow_module_at_step(&self, step: Step) -> anyhow::Result<&FlowModule> { + use anyhow::Context; + let flow_module = match step { + Step::PreprocessorStep => self + .preprocessor_module + .as_deref() + .with_context(|| format!("no preprocessor module")), + Step::Step { idx, .. } => self + .modules + .get(idx) + .with_context(|| format!("no module found at index: {idx}")), + Step::FailureStep => self + .failure_module + .as_deref() + .with_context(|| format!("no failure module")), + }; + + flow_module + } + + pub fn traverse_leafs anyhow::Result<()>>( + modules: Vec<&FlowModule>, + cb: &mut C, + ) -> anyhow::Result<()> { + use FlowModuleValue::*; + for module in modules { + match serde_json::from_str::(module.value.get())? { + s @ (Script { .. } + | RawScript { .. } + | Flow { .. } + | FlowScript { .. } + | Identity) => cb(&s, &module.id)?, + ForloopFlow { modules, .. } | WhileloopFlow { modules, .. } => { + Self::traverse_leafs(modules.iter().collect(), cb)? + } + AIAgent { tools, .. } => { + for tool in tools { + match &tool.value { + ToolValue::FlowModule(module_value) => cb(module_value, &tool.id)?, + ToolValue::Mcp(_) => {} + ToolValue::Websearch(_) => {} + } + } + } + BranchOne { default, branches, .. } => { + Self::traverse_leafs(default.iter().collect(), cb)?; + for branch in branches { + Self::traverse_leafs(branch.modules.iter().collect(), cb)?; + } + } + BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_leafs(branch.modules.iter().collect(), cb)?; + } + } + } + } + Ok(()) + } +} + +#[derive(Debug, Copy, Clone)] +pub enum Step { + Step { idx: usize, len: usize }, + PreprocessorStep, + FailureStep, +} + +impl Step { + pub fn from_i32_and_len(step: i32, len: usize) -> Self { + if step < 0 { + Step::PreprocessorStep + } else if (step as usize) < len { + Step::Step { idx: step as usize, len } + } else { + Step::FailureStep + } + } + + pub fn get_step_index(&self) -> Option { + match self { + Step::Step { idx, .. } => Some(*idx), + _ => None, + } + } + + pub fn is_index_step(&self) -> bool { + matches!(self, Step::Step { .. }) + } + + pub fn is_preprocessor_step(&self) -> bool { + matches!(self, Step::PreprocessorStep) + } + + pub fn is_failure_step(&self) -> bool { + matches!(self, Step::FailureStep) + } + + pub fn is_last_step(&self) -> bool { + matches!(self, Step::Step { idx, len } if *idx == len - 1) + } +} + +#[derive(Default, Deserialize, Serialize, Debug, Clone)] +pub struct StopAfterIf { + pub expr: String, + pub skip_if_stopped: bool, + pub error_message: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +pub struct RetryIf { + pub expr: String, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct Retry { + pub constant: ConstantDelay, + pub exponential: ExponentialDelay, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry_if: Option, +} + +impl Retry { + pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option { + let Self { constant, exponential, .. } = self; + + if previous_attempts < constant.attempts { + Some(Duration::from_secs(constant.seconds as u64)) + } else if previous_attempts - constant.attempts < exponential.attempts { + let exp = previous_attempts.saturating_add(1) as u32; + let mut secs = exponential.multiplier * exponential.seconds.saturating_pow(exp); + if let Some(random_factor) = exponential.random_factor { + if random_factor > 0 { + let random_component = + rand::rng().random_range(0..(std::cmp::min(random_factor, 100) as u16)); + secs = match rand::rng().random_bool(1.0 / 2.0) { + true => secs.saturating_add(secs * random_component / 100), + false => secs.saturating_sub(secs * random_component / 100), + }; + } + } + if !silent { + tracing::warn!("Rescheduling job in {} seconds due to failure", secs); + } + Some(Duration::from_secs(secs as u64)) + } else { + None + } + } + + pub fn has_attempts(&self) -> bool { + self.constant.attempts != 0 || self.exponential.attempts != 0 + } + + pub fn max_attempts(&self) -> u32 { + self.constant + .attempts + .saturating_add(self.exponential.attempts) + } + + pub fn max_interval(&self) -> Option { + self.max_attempts() + .checked_sub(1) + .and_then(|p| self.interval(p, true)) + } +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] +#[serde(default)] +pub struct ConstantDelay { + pub attempts: u32, + pub seconds: u16, +} + +/// multiplier * seconds ^ failures (+/- jitter of the previous value, if any) +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(default)] +pub struct ExponentialDelay { + pub attempts: u32, + pub multiplier: u16, + pub seconds: u16, + pub random_factor: Option, +} + +impl Default for ExponentialDelay { + fn default() -> Self { + Self { attempts: 0, multiplier: 1, seconds: 0, random_factor: None } + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Suspend { + #[serde(skip_serializing_if = "Option::is_none")] + pub required_events: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub resume_form: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_auth_required: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_groups_required: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub self_approval_disabled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hide_cancel: Option, + #[serde(skip_serializing_if = "false_or_empty")] + pub continue_on_disapprove_timeout: Option, +} + +fn false_or_empty(v: &Option) -> bool { + v.is_none() || v.as_ref().is_some_and(|x| !x) +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Mock { + pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub return_value: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct FlowModule { + #[serde(default = "default_id")] + pub id: String, + pub value: Box, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_after_if: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_after_all_iters_if: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub suspend: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sleep: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, i32>", + skip_serializing_if = "Option::is_none" + )] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub continue_on_error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_if: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub apply_preprocessor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pass_flow_input_directly: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct SkipIf { + pub expr: String, +} + +#[derive(Deserialize)] +pub struct FlowModuleValueWithParallel { + #[serde(rename = "type")] + pub type_: String, + pub parallel: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, +} + +#[derive(Deserialize)] +pub struct FlowModuleValueWithSkipFailures { + pub skip_failures: Option, + pub parallel: Option, + #[serde( + default, + deserialize_with = "raw_value_to_input_transform::<_, u16>", + skip_serializing_if = "Option::is_none" + )] + pub parallelism: Option, +} + +#[derive(Deserialize)] +pub struct BranchWithSkipFailures { + pub skip_failure: Option, +} + +#[derive(Deserialize)] +pub struct FlowModuleWithBranches { + pub branches: Vec, +} + +impl FlowModule { + pub fn id_append(&mut self, s: &str) { + self.id = format!("{}-{}", self.id, s); + } + pub fn get_value(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn get_value_with_skip_failures(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn get_branches_skip_failures(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn is_flow(&self) -> bool { + self.get_type().is_ok_and(|x| x == "flow") + } + + pub fn get_value_with_parallel(&self) -> anyhow::Result { + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + } + + pub fn is_ai_agent(&self) -> bool { + self.get_type().is_ok_and(|x| x == "aiagent") + } + + pub fn is_simple(&self) -> bool { + self.get_type() + .is_ok_and(|x| x == "script" || x == "rawscript" || x == "flowscript") + } + + pub fn get_type(&self) -> anyhow::Result<&str> { + #[derive(Deserialize)] + pub struct FlowModuleValueType<'a> { + pub r#type: &'a str, + } + + serde_json::from_str::(self.value.get()) + .map_err(|e| anyhow::anyhow!("{}", e)) + .map(|x| x.r#type) + } + + pub fn traverse_modules anyhow::Result<()>>( + modules: &Vec, + cb: &mut C, + ) -> anyhow::Result<()> { + for module in modules { + cb(module)?; + match module + .get_value() + .map_err(|e| anyhow::anyhow!("Module '{}': {}", module.id, e))? + { + FlowModuleValue::ForloopFlow { modules, .. } + | FlowModuleValue::WhileloopFlow { modules, .. } => { + Self::traverse_modules(&modules, cb)?; + } + FlowModuleValue::BranchOne { branches, default, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + Self::traverse_modules(&default, cb)?; + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + } + FlowModuleValue::AIAgent { tools, .. } => { + for tool in tools { + match &tool.value { + ToolValue::FlowModule(module_value) => match module_value { + FlowModuleValue::ForloopFlow { modules, .. } + | FlowModuleValue::WhileloopFlow { modules, .. } => { + Self::traverse_modules(&modules, cb)?; + } + FlowModuleValue::BranchOne { branches, default, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + Self::traverse_modules(&default, cb)?; + } + FlowModuleValue::BranchAll { branches, .. } => { + for branch in branches { + Self::traverse_modules(&branch.modules, cb)?; + } + } + _ => {} + }, + ToolValue::Mcp(_) => {} + ToolValue::Websearch(_) => {} + } + } + } + _ => {} + } + } + Ok(()) + } +} + +#[derive(Deserialize)] +pub struct UntaggedInputTransform { + #[serde(rename = "type")] + pub type_: String, + pub value: Option>, + pub expr: Option, +} + +impl<'de> Deserialize<'de> for InputTransform { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let untagged: UntaggedInputTransform = UntaggedInputTransform::deserialize(deserializer)?; + + let input_transform = TryInto::::try_into(untagged) + .map_err(|e| serde::de::Error::custom(e))?; + + Ok(input_transform) + } +} + +#[derive(Serialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum InputTransform { + Static { + #[serde(default = "default_null")] + value: Box, + }, + Javascript { + #[serde(default = "default_empty_string")] + expr: String, + }, + Ai, +} + +impl InputTransform { + pub fn new_static_value(value: Box) -> InputTransform { + InputTransform::Static { value } + } + + pub fn new_javascript_expr(expr: &str) -> InputTransform { + InputTransform::Javascript { expr: expr.to_owned() } + } +} + +impl TryFrom for InputTransform { + type Error = anyhow::Error; + fn try_from(value: UntaggedInputTransform) -> Result { + let input_transform = match value.type_.as_str() { + "static" => InputTransform::new_static_value(value.value.unwrap_or_else(default_null)), + "javascript" => InputTransform::new_javascript_expr(&value.expr.unwrap_or_default()), + "ai" => InputTransform::Ai, + other => { + return Err(anyhow::anyhow!( + "got value: {other} for field `type`, expected value: `static` or `javascript`" + )) + } + }; + + Ok(input_transform) + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum RawValueOrFormatted { + RawValue(T), + Formatted { r#type: String, value: Option, expr: Option }, +} + +fn raw_value_to_input_transform<'de, D, T>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, + T: DeserializeOwned + Serialize, +{ + let val = Option::>::deserialize(deserializer)?; + let input_tranform = match val { + Some(RawValueOrFormatted::RawValue(v)) => { + Some(InputTransform::new_static_value(to_raw_value(&v))) + } + Some(RawValueOrFormatted::Formatted { r#type, expr, value }) => { + let untaged_input_transform = UntaggedInputTransform { + type_: r#type, + expr, + value: value.map(|val| to_raw_value(&val)), + }; + let input_transform = TryInto::::try_into(untaged_input_transform) + .map_err(|e| serde::de::Error::custom(e))?; + Some(input_transform) + } + _ => None, + }; + Ok(input_tranform) +} + +/// Id in the `flow_node` table. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] +#[serde(transparent)] +pub struct FlowNodeId(pub i64); + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Branch { + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default = "default_empty_string")] + pub expr: String, + pub modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub modules_node: Option, + #[serde(default = "default_true")] + pub skip_failure: bool, + #[serde(default = "default_true")] + pub parallel: bool, +} + +// Tool types for AI Agent +#[derive(Serialize, Debug, Clone, Deserialize)] +pub struct AgentTool { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + pub value: ToolValue, +} + +// Convert FlowModule -> AgentTool +impl From for AgentTool { + fn from(flow_module: FlowModule) -> Self { + let module_value = serde_json::from_str::(flow_module.value.get()) + .unwrap_or(FlowModuleValue::Identity); + + AgentTool { + id: flow_module.id, + summary: flow_module.summary, + value: ToolValue::FlowModule(module_value), + } + } +} + +// Convert AgentTool -> FlowModule (only for FlowModule type tools) +impl From<&AgentTool> for Option { + fn from(tool: &AgentTool) -> Self { + match &tool.value { + ToolValue::FlowModule(module_value) => Some(FlowModule { + id: tool.id.clone(), + value: to_raw_value(module_value), + summary: tool.summary.clone(), + ..Default::default() + }), + ToolValue::Mcp(_) => None, + ToolValue::Websearch(_) => None, + } + } +} + +#[derive(Serialize, Debug, Clone)] +#[serde(tag = "tool_type", rename_all = "lowercase")] +pub enum ToolValue { + FlowModule(FlowModuleValue), + Mcp(McpToolValue), + Websearch(WebsearchToolValue), +} + +// Custom deserializer for backward compatibility with old flows +impl<'de> Deserialize<'de> for ToolValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error; + + let content = serde_json::Value::deserialize(deserializer)?; + + #[derive(Deserialize)] + #[serde(tag = "tool_type", rename_all = "lowercase")] + enum TaggedToolValue { + FlowModule(FlowModuleValue), + Mcp(McpToolValue), + Websearch(WebsearchToolValue), + } + + if let Ok(tagged) = TaggedToolValue::deserialize(&content) { + return Ok(match tagged { + TaggedToolValue::FlowModule(v) => ToolValue::FlowModule(v), + TaggedToolValue::Mcp(v) => ToolValue::Mcp(v), + TaggedToolValue::Websearch(v) => ToolValue::Websearch(v), + }); + } + + FlowModuleValue::deserialize(&content) + .map(ToolValue::FlowModule) + .map_err(|_| { + D::Error::custom( + "expected ToolValue with tool_type field or legacy FlowModuleValue", + ) + }) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct McpToolValue { + pub resource_path: String, + #[serde(default)] + pub include_tools: Vec, + #[serde(default)] + pub exclude_tools: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub struct WebsearchToolValue {} + +fn is_none_or_empty_vec(expr: &Option>) -> bool { + expr.is_none() || expr.as_ref().unwrap().is_empty() +} + +#[derive(Serialize, Debug, Clone)] +#[serde( + tag = "type", + rename_all(serialize = "lowercase", deserialize = "lowercase") +)] +pub enum FlowModuleValue { + Script { + #[serde(default)] + #[serde(alias = "input_transform")] + input_transforms: HashMap, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tag_override: Option, + #[serde(skip_serializing_if = "Option::is_none")] + is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pass_flow_input_directly: Option, + }, + Flow { + #[serde(default)] + #[serde(alias = "input_transform")] + input_transforms: HashMap, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pass_flow_input_directly: Option, + }, + ForloopFlow { + iterator: InputTransform, + modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + modules_node: Option, + #[serde(default = "default_true")] + skip_failures: bool, + parallel: bool, + #[serde(skip_serializing_if = "Option::is_none")] + parallelism: Option, + #[serde(skip_serializing_if = "Option::is_none")] + squash: Option, + }, + WhileloopFlow { + modules: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + modules_node: Option, + #[serde(default = "default_false")] + skip_failures: bool, + #[serde(skip_serializing_if = "Option::is_none")] + squash: Option, + }, + BranchOne { + branches: Vec, + default: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + default_node: Option, + }, + BranchAll { + branches: Vec, + #[serde(default = "default_true")] + parallel: bool, + }, + RawScript { + #[serde(default)] + #[serde(alias = "input_transform", serialize_with = "ordered_map")] + input_transforms: HashMap, + content: String, + #[serde(skip_serializing_if = "Option::is_none")] + lock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "is_none_or_empty")] + tag: Option, + language: ScriptLang, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(skip_serializing_if = "Option::is_none")] + is_trigger: Option, + #[serde(skip_serializing_if = "is_none_or_empty_vec")] + assets: Option>, + }, + Identity, + FlowScript { + #[serde(default)] + #[serde(alias = "input_transform", serialize_with = "ordered_map")] + input_transforms: HashMap, + id: FlowNodeId, + #[serde(skip_serializing_if = "is_none_or_empty")] + tag: Option, + language: ScriptLang, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(skip_serializing_if = "Option::is_none")] + is_trigger: Option, + #[serde(skip_serializing_if = "is_none_or_empty_vec")] + assets: Option>, + }, + AIAgent { + input_transforms: HashMap, + tools: Vec, + }, +} + +fn is_none_or_empty(expr: &Option) -> bool { + expr.is_none() || expr.as_ref().unwrap().is_empty() +} + +#[derive(Deserialize)] +struct UntaggedFlowModuleValue { + #[serde(rename = "type")] + type_: String, + #[serde(alias = "input_transform")] + input_transforms: Option>, + path: Option, + hash: Option, + tag_override: Option, + iterator: Option, + modules: Option>, + skip_failures: Option, + parallel: Option, + #[serde(default, deserialize_with = "raw_value_to_input_transform::<_, u16>")] + parallelism: Option, + branches: Option>, + default: Option>, + content: Option, + lock: Option, + tag: Option, + language: Option, + is_trigger: Option, + id: Option, + default_node: Option, + modules_node: Option, + assets: Option>, + tools: Option>, + pass_flow_input_directly: Option, + squash: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, +} + +impl<'de> Deserialize<'de> for FlowModuleValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let untagged: UntaggedFlowModuleValue = UntaggedFlowModuleValue::deserialize(deserializer)?; + + match untagged.type_.as_str() { + "script" => Ok(FlowModuleValue::Script { + input_transforms: untagged.input_transforms.unwrap_or_default(), + path: untagged + .path + .ok_or_else(|| serde::de::Error::missing_field("path"))?, + hash: untagged.hash, + tag_override: untagged.tag_override, + is_trigger: untagged.is_trigger, + pass_flow_input_directly: untagged.pass_flow_input_directly, + }), + "flow" => Ok(FlowModuleValue::Flow { + input_transforms: untagged.input_transforms.unwrap_or_default(), + path: untagged + .path + .ok_or_else(|| serde::de::Error::missing_field("path"))?, + pass_flow_input_directly: untagged.pass_flow_input_directly, + }), + "forloopflow" => Ok(FlowModuleValue::ForloopFlow { + iterator: untagged + .iterator + .ok_or_else(|| serde::de::Error::missing_field("iterator"))?, + modules: untagged + .modules + .ok_or_else(|| serde::de::Error::missing_field("modules"))?, + modules_node: untagged.modules_node, + skip_failures: untagged.skip_failures.unwrap_or(true), + parallel: untagged.parallel.unwrap_or(false), + parallelism: untagged.parallelism, + squash: untagged.squash, + }), + "whileloopflow" => Ok(FlowModuleValue::WhileloopFlow { + modules: untagged + .modules + .ok_or_else(|| serde::de::Error::missing_field("modules"))?, + modules_node: untagged.modules_node, + skip_failures: untagged.skip_failures.unwrap_or(false), + squash: untagged.squash, + }), + "branchone" => Ok(FlowModuleValue::BranchOne { + branches: untagged + .branches + .ok_or_else(|| serde::de::Error::missing_field("branches"))?, + default: untagged + .default + .ok_or_else(|| serde::de::Error::missing_field("default"))?, + default_node: untagged.default_node, + }), + "branchall" => Ok(FlowModuleValue::BranchAll { + branches: untagged + .branches + .ok_or_else(|| serde::de::Error::missing_field("branches"))?, + parallel: untagged.parallel.unwrap_or(true), + }), + "rawscript" => Ok(FlowModuleValue::RawScript { + input_transforms: untagged.input_transforms.unwrap_or_default(), + content: untagged + .content + .ok_or_else(|| serde::de::Error::missing_field("content"))?, + lock: untagged.lock, + path: untagged.path, + tag: untagged.tag, + language: untagged + .language + .ok_or_else(|| serde::de::Error::missing_field("language"))?, + concurrency_settings: untagged.concurrency_settings, + is_trigger: untagged.is_trigger, + assets: untagged.assets, + }), + "flowscript" => Ok(FlowModuleValue::FlowScript { + input_transforms: untagged.input_transforms.unwrap_or_default(), + id: untagged + .id + .ok_or_else(|| serde::de::Error::missing_field("id"))?, + tag: untagged.tag, + language: untagged + .language + .ok_or_else(|| serde::de::Error::missing_field("language"))?, + concurrency_settings: untagged.concurrency_settings, + is_trigger: untagged.is_trigger, + assets: untagged.assets, + }), + "identity" => Ok(FlowModuleValue::Identity), + "aiagent" => Ok(FlowModuleValue::AIAgent { + input_transforms: untagged.input_transforms.unwrap_or_default(), + tools: untagged + .tools + .ok_or_else(|| serde::de::Error::missing_field("tools"))?, + }), + other => Err(serde::de::Error::unknown_variant( + other, + &[ + "script", + "flow", + "forloopflow", + "whileloopflow", + "branchone", + "branchall", + "rawscript", + "identity", + "aiagent", + ], + )), + } + } +} + +impl Into> for FlowModuleValue { + fn into(self) -> Box { + to_raw_value(&self) + } +} + +pub fn ordered_map(value: &HashMap, serializer: S) -> Result +where + S: Serializer, +{ + let ordered: BTreeMap<_, _> = value.iter().collect(); + ordered.serialize(serializer) +} + +#[derive(Deserialize)] +pub struct ListFlowQuery { + pub without_description: Option, + pub path_start: Option, + pub path_exact: Option, + pub edited_by: Option, + pub show_archived: Option, + pub order_by: Option, + pub order_desc: Option, + pub starred_only: Option, + pub include_draft_only: Option, + pub with_deployment_msg: Option, + pub dedicated_worker: Option, +} + +pub fn add_virtual_items_if_necessary(modules: &mut Vec) { + if modules.len() > 0 + && (modules[modules.len() - 1].sleep.is_some() + || modules[modules.len() - 1].suspend.is_some()) + { + modules.push(FlowModule { + id: format!("{}-v", modules[modules.len() - 1].id), + value: to_raw_value(&FlowModuleValue::Identity), + stop_after_if: None, + stop_after_all_iters_if: None, + summary: Some("Virtual module needed for suspend/sleep when last module".to_string()), + mock: None, + retry: None, + sleep: None, + suspend: None, + cache_ttl: None, + cache_ignore_s3_path: None, + timeout: None, + priority: None, + delete_after_use: None, + continue_on_error: None, + skip_if: None, + apply_preprocessor: None, + pass_flow_input_directly: None, + }); + } +} diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs new file mode 100644 index 0000000000..3c54b7956e --- /dev/null +++ b/backend/windmill-types/src/jobs.rs @@ -0,0 +1,510 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use uuid::Uuid; + +use crate::{ + apps::AppScriptId, + flow_status::{FlowStatus, RestartedFrom}, + flows::{FlowNodeId, FlowValue, Retry}, + runnable_settings::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, + scripts::{ScriptHash, ScriptLang}, +}; + +#[derive(Debug, Deserialize, Clone)] +pub struct DynamicInput { + #[serde(rename = "x-windmill-dyn-select-code")] + pub x_windmill_dyn_select_code: String, + #[serde(rename = "x-windmill-dyn-select-lang")] + pub x_windmill_dyn_select_lang: ScriptLang, +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] +#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum JobTriggerKind { + Webhook, + Http, + Websocket, + Kafka, + Email, + Nats, + Mqtt, + Sqs, + Postgres, + Schedule, + Gcp, + Nextcloud, +} + +impl std::fmt::Display for JobTriggerKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let kind = match self { + JobTriggerKind::Webhook => "webhook", + JobTriggerKind::Http => "http", + JobTriggerKind::Websocket => "websocket", + JobTriggerKind::Kafka => "kafka", + JobTriggerKind::Email => "email", + JobTriggerKind::Nats => "nats", + JobTriggerKind::Mqtt => "mqtt", + JobTriggerKind::Sqs => "sqs", + JobTriggerKind::Postgres => "postgres", + JobTriggerKind::Schedule => "schedule", + JobTriggerKind::Gcp => "gcp", + JobTriggerKind::Nextcloud => "nextcloud", + }; + write!(f, "{}", kind) + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] +#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum JobKind { + Script, + #[allow(non_camel_case_types)] + Script_Hub, + Preview, + Dependencies, + Flow, + FlowPreview, + SingleStepFlow, + Identity, + FlowDependencies, + AppDependencies, + #[default] + Noop, + DeploymentCallback, + FlowScript, + FlowNode, + AppScript, + AIAgent, + #[serde(rename = "unassigned_script")] + #[sqlx(rename = "unassigned_script")] + UnassignedScript, + #[serde(rename = "unassigned_flow")] + #[sqlx(rename = "unassigned_flow")] + UnassignedFlow, + #[serde(rename = "unassigned_singlestepflow")] + #[sqlx(rename = "unassigned_singlestepflow")] + UnassignedSinglestepFlow, +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] +#[sqlx(type_name = "JOB_STATUS", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum JobStatus { + Success, + Failure, + Canceled, + Skipped, +} + +impl JobKind { + pub fn is_flow(&self) -> bool { + matches!( + self, + JobKind::Flow | JobKind::FlowPreview | JobKind::SingleStepFlow | JobKind::FlowNode + ) + } + + pub fn is_dependency(&self) -> bool { + matches!( + self, + JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies + ) + } +} + +#[derive(sqlx::FromRow, Debug, Serialize, Clone)] +pub struct QueuedJob { + pub workspace_id: String, + pub id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub started_at: Option>, + pub scheduled_for: chrono::DateTime, + pub running: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_path: Option, + pub script_entrypoint_override: Option, + pub args: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub logs: Option, + pub canceled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_ping: Option>, + pub job_kind: JobKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + pub permissioned_as: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option>>, + pub is_flow_step: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + pub same_worker: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub pre_run_error: Option, + pub email: String, + pub visible_to_owner: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub suspend: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub leaf_jobs: Option, + pub tag: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_step_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preprocessed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub runnable_settings_handle: Option, +} + +impl QueuedJob { + pub fn script_path(&self) -> &str { + self.script_path + .as_ref() + .map(String::as_str) + .unwrap_or("tmp/main") + } + pub fn is_flow(&self) -> bool { + self.job_kind.is_flow() + } + + pub fn full_path_with_workspace(&self) -> String { + format!( + "{}/{}/{}", + self.workspace_id, + if self.is_flow() { "flow" } else { "script" }, + self.script_path() + ) + } + + pub fn parse_flow_status(&self) -> Option { + self.flow_status + .as_ref() + .and_then(|v| serde_json::from_str::((**v).get()).ok()) + } +} + +impl Default for QueuedJob { + fn default() -> Self { + Self { + workspace_id: "".to_string(), + id: Uuid::default(), + parent_job: None, + created_by: "".to_string(), + created_at: chrono::Utc::now(), + started_at: None, + scheduled_for: chrono::Utc::now(), + running: false, + script_hash: None, + script_path: None, + args: None, + logs: None, + canceled: false, + canceled_by: None, + canceled_reason: None, + last_ping: None, + job_kind: JobKind::Identity, + schedule_path: None, + permissioned_as: "".to_string(), + workflow_as_code_status: None, + flow_status: None, + is_flow_step: false, + language: None, + script_entrypoint_override: None, + same_worker: false, + pre_run_error: None, + email: "".to_string(), + visible_to_owner: false, + suspend: None, + mem_peak: None, + root_job: None, + leaf_jobs: None, + tag: "deno".to_string(), + concurrent_limit: None, + concurrency_time_window_s: None, + timeout: None, + flow_step_id: None, + cache_ttl: None, + cache_ignore_s3_path: None, + priority: None, + preprocessed: None, + runnable_settings_handle: None, + } + } +} + +#[derive(Debug, sqlx::FromRow, Serialize, Clone)] +pub struct CompletedJob { + pub workspace_id: String, + pub id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub completed_at: Option>, + pub duration_ms: i64, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_path: Option, + pub args: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option>>, + pub result_columns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub logs: Option, + pub deleted: bool, + pub canceled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_reason: Option, + pub job_kind: JobKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + pub permissioned_as: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option>>, + pub is_flow_step: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + pub is_skipped: bool, + pub email: String, + pub visible_to_owner: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + pub tag: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preprocessed: Option, +} + +impl CompletedJob { + pub fn json_result(&self) -> Option { + self.result + .as_ref() + .map(|r| serde_json::from_str(r.get()).ok()) + .flatten() + } + + pub fn parse_flow_status(&self) -> Option { + self.flow_status + .as_ref() + .and_then(|v| serde_json::from_str::((**v).get()).ok()) + } +} + +#[derive(Debug, Clone)] +pub enum JobPayload { + ScriptHub { + path: String, + apply_preprocessor: bool, + }, + ScriptHash { + hash: ScriptHash, + path: String, + cache_ttl: Option, + cache_ignore_s3_path: Option, + dedicated_worker: Option, + language: ScriptLang, + priority: Option, + apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, + }, + FlowNode { + id: FlowNodeId, + path: String, + }, + FlowScript { + id: FlowNodeId, + path: String, + language: ScriptLang, + cache_ttl: Option, + cache_ignore_s3_path: Option, + dedicated_worker: Option, + concurrency_settings: ConcurrencySettings, + }, + AppScript { + id: AppScriptId, + path: Option, + language: ScriptLang, + cache_ttl: Option, + }, + Code(RawCode), + Dependencies { + path: String, + hash: ScriptHash, + language: ScriptLang, + dedicated_worker: Option, + debouncing_settings: DebouncingSettings, + }, + FlowDependencies { + path: String, + dedicated_worker: Option, + version: i64, + debouncing_settings: DebouncingSettings, + }, + AppDependencies { + path: String, + version: i64, + debouncing_settings: DebouncingSettings, + }, + RawFlowDependencies { + path: String, + flow_value: FlowValue, + }, + RawScriptDependencies { + script_path: String, + content: String, + language: ScriptLang, + }, + Flow { + path: String, + dedicated_worker: Option, + apply_preprocessor: bool, + version: i64, + }, + RestartedFlow { + completed_job_id: Uuid, + step_id: String, + branch_or_iteration_n: Option, + flow_version: Option, + }, + RawFlow { + value: FlowValue, + path: Option, + restarted_from: Option, + }, + SingleStepFlow { + path: String, + hash: Option, + flow_version: Option, + args: HashMap>, + retry: Option, + error_handler_path: Option, + error_handler_args: Option>>, + skip_handler: Option, + cache_ttl: Option, + cache_ignore_s3_path: Option, + priority: Option, + tag_override: Option, + trigger_path: Option, + apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, + }, + DeploymentCallback { + path: String, + debouncing_settings: DebouncingSettings, + }, + Identity, + Noop, + AIAgent { + path: String, + }, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct SkipHandler { + pub path: String, + pub args: HashMap>, + pub stop_condition: String, + pub stop_message: String, +} + +#[derive(Clone, Deserialize, Debug, Default)] +pub struct RawCode { + pub content: String, + pub path: Option, + pub hash: Option, + pub language: ScriptLang, + pub lock: Option, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub dedicated_worker: Option, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, +} + +impl JobPayload { + pub fn job_kind(&self) -> JobKind { + match self { + JobPayload::Noop => JobKind::Noop, + JobPayload::Identity => JobKind::Identity, + JobPayload::Code { .. } => JobKind::Preview, + JobPayload::AIAgent { .. } => JobKind::AIAgent, + JobPayload::FlowNode { .. } => JobKind::FlowNode, + JobPayload::ScriptHash { .. } => JobKind::Script, + JobPayload::AppScript { .. } => JobKind::AppScript, + JobPayload::RawFlow { .. } => JobKind::FlowPreview, + JobPayload::ScriptHub { .. } => JobKind::Script_Hub, + JobPayload::FlowScript { .. } => JobKind::FlowScript, + JobPayload::Dependencies { .. } => JobKind::Dependencies, + JobPayload::SingleStepFlow { .. } => JobKind::SingleStepFlow, + JobPayload::AppDependencies { .. } => JobKind::AppDependencies, + JobPayload::FlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::RawScriptDependencies { .. } => JobKind::Dependencies, + JobPayload::RawFlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::DeploymentCallback { .. } => JobKind::DeploymentCallback, + JobPayload::Flow { .. } | JobPayload::RestartedFlow { .. } => JobKind::Flow, + } + } +} + +#[derive(Clone, Debug)] +pub struct OnBehalfOf { + pub email: String, + pub permissioned_as: String, +} + +pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; +pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000; +pub const EMAIL_ERROR_HANDLER_USER_EMAIL: &str = "email_error_handler@windmill.dev"; + +#[inline(always)] +pub fn generate_dynamic_input_key(workspace_id: &str, path: &str) -> String { + format!("{workspace_id}:{path}") +} diff --git a/backend/windmill-types/src/lib.rs b/backend/windmill-types/src/lib.rs new file mode 100644 index 0000000000..8b7e68d75c --- /dev/null +++ b/backend/windmill-types/src/lib.rs @@ -0,0 +1,17 @@ +pub mod apps; +pub mod assets; +pub mod flow_status; +pub mod flows; +pub mod jobs; +pub mod more_serde; +pub mod runnable_settings; +pub mod schedule; +pub mod scripts; +pub mod triggers; + +/// Duplicated from windmill-common::worker::to_raw_value. +/// windmill-types cannot depend on windmill-common (it would be circular). +pub fn to_raw_value(result: &T) -> Box { + serde_json::value::to_raw_value(result) + .unwrap_or_else(|_| serde_json::value::RawValue::from_string("{}".to_string()).unwrap()) +} diff --git a/backend/windmill-types/src/more_serde.rs b/backend/windmill-types/src/more_serde.rs new file mode 100644 index 0000000000..d4b648d2d5 --- /dev/null +++ b/backend/windmill-types/src/more_serde.rs @@ -0,0 +1,68 @@ +//! helpers for serde + serde derive attributes + +use rand::distr::Alphanumeric; +use rand::Rng; +use serde::{Deserialize, Deserializer}; +use serde_json::value::RawValue; +use std::{fmt::Display, str::FromStr}; + +pub fn default_true() -> bool { + true +} + +pub fn default_false() -> bool { + false +} + +pub fn default_null() -> Box { + RawValue::from_string("null".to_string()).unwrap() +} + +pub fn default_empty_string() -> String { + String::new() +} + +pub fn default_id() -> String { + rd_string(6) +} + +fn rd_string(len: usize) -> String { + rand::rng() + .sample_iter(&Alphanumeric) + .take(len) + .map(char::from) + .collect() +} + +pub fn is_default(t: &T) -> bool { + &T::default() == t +} + +pub fn maybe_number_opt<'de, T, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: FromStr + serde::Deserialize<'de>, + ::Err: Display, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum NumericOrNull<'a, T> { + String(String), + Str(&'a str), + RawT(T), + Null, + } + + match NumericOrNull::::deserialize(deserializer)? { + NumericOrNull::String(s) => match s.as_str() { + "" => Ok(None), + _ => T::from_str(&s).map(Some).map_err(serde::de::Error::custom), + }, + NumericOrNull::Str(s) => match s { + "" => Ok(None), + _ => T::from_str(s).map(Some).map_err(serde::de::Error::custom), + }, + NumericOrNull::RawT(i) => Ok(Some(i)), + NumericOrNull::Null => Ok(None), + } +} diff --git a/backend/windmill-types/src/runnable_settings.rs b/backend/windmill-types/src/runnable_settings.rs new file mode 100644 index 0000000000..dfff0df79e --- /dev/null +++ b/backend/windmill-types/src/runnable_settings.rs @@ -0,0 +1,113 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Clone, Copy, Serialize, Default, Hash)] +pub struct RunnableSettings { + pub debouncing_settings: Option, + pub concurrency_settings: Option, +} + +// TODO: Add validation logic. +#[derive( + Debug, Clone, Serialize, Deserialize, Default, Hash, PartialEq, sqlx::FromRow, sqlx::Type, +)] +pub struct DebouncingSettings { + #[serde(skip_serializing_if = "Option::is_none", alias = "custom_debounce_key")] + pub debounce_key: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_delay_s: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_total_debouncing_time: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_total_debounces_amount: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub debounce_args_to_accumulate: Option>, +} + +#[derive( + Debug, Default, Clone, Serialize, Deserialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode, +)] +pub struct ConcurrencySettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] +pub struct ConcurrencySettingsWithCustom { + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +impl DebouncingSettings { + pub fn maybe_fallback( + self, + debounce_key: Option, + debounce_delay_s: Option, + ) -> Self { + Self { + debounce_key: self.debounce_key.or(debounce_key), + debounce_delay_s: self.debounce_delay_s.or(debounce_delay_s), + ..self + } + } + + pub fn is_legacy_compatible(&self) -> bool { + self.max_total_debouncing_time.is_none() + && self.max_total_debounces_amount.is_none() + && self.debounce_args_to_accumulate.is_none() + } +} + +impl ConcurrencySettings { + pub fn maybe_fallback( + self, + concurrency_key: Option, + concurrent_limit: Option, + concurrency_time_window_s: Option, + ) -> Self { + Self { + concurrency_key: self.concurrency_key.or(concurrency_key), + concurrent_limit: self.concurrent_limit.or(concurrent_limit), + concurrency_time_window_s: self.concurrency_time_window_s.or(concurrency_time_window_s), + } + } +} + +impl From for ConcurrencySettingsWithCustom { + fn from( + ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }: ConcurrencySettings, + ) -> Self { + ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} + +impl From for ConcurrencySettings { + fn from( + ConcurrencySettingsWithCustom { + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }: ConcurrencySettingsWithCustom, + ) -> Self { + ConcurrencySettings { + concurrency_key: custom_concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} diff --git a/backend/windmill-types/src/schedule.rs b/backend/windmill-types/src/schedule.rs new file mode 100644 index 0000000000..a3a61b5765 --- /dev/null +++ b/backend/windmill-types/src/schedule.rs @@ -0,0 +1,67 @@ +use chrono::DateTime; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; + +use crate::flows::Retry; + +#[derive(FromRow, Serialize, Deserialize, Debug, Clone)] +pub struct Schedule { + pub workspace_id: String, + pub path: String, + pub edited_by: String, + pub edited_at: DateTime, + pub schedule: String, + pub timezone: String, + pub enabled: bool, + pub script_path: String, + pub is_flow: bool, + pub args: Option>>, + pub extra_perms: serde_json::Value, + pub email: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure_times: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure_exact: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_failure_extra_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_recovery: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_recovery_times: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_recovery_extra_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_success: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_success_extra_args: Option>>, + pub ws_error_handler_muted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option, + pub no_flow_overlap: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub paused_until: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub cron_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dynamic_skip: Option, +} + +impl Schedule { + pub fn parse_retry(self) -> Option { + self.retry.map(|r| serde_json::from_value(r).ok()).flatten() + } +} + +pub fn schedule_to_user(path: &str) -> String { + format!("schedule-{}", path.replace('/', "-")) +} diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs new file mode 100644 index 0000000000..a925bd2031 --- /dev/null +++ b/backend/windmill-types/src/scripts.rs @@ -0,0 +1,613 @@ +use std::{ + fmt::{self, Display}, + hash::{Hash, Hasher}, + ops::Deref, + str::FromStr, +}; + +use itertools::Itertools; +use serde::de::Error as _; +use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize}; + +use crate::{ + assets::AssetWithAltAccessType, + runnable_settings::{ConcurrencySettings, DebouncingSettings}, +}; + +#[derive( + Serialize, + Deserialize, + Debug, + PartialEq, + Copy, + Clone, + Hash, + Eq, + sqlx::Type, + Default, + Ord, + PartialOrd, +)] +#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum ScriptLang { + Nativets, + #[default] + Deno, + Python3, + Go, + Bash, + Powershell, + Postgresql, + Bun, + Bunnative, + Mysql, + Bigquery, + Snowflake, + Graphql, + Mssql, + OracleDB, + DuckDb, + Php, + Rust, + Ansible, + CSharp, + Nu, + Java, + Ruby, + // for related places search: ADD_NEW_LANG +} + +impl ScriptLang { + pub fn as_str(&self) -> &'static str { + match self { + ScriptLang::Bun => "bun", + ScriptLang::Bunnative => "bunnative", + ScriptLang::Nativets => "nativets", + ScriptLang::Deno => "deno", + ScriptLang::Python3 => "python3", + ScriptLang::Go => "go", + ScriptLang::Bash => "bash", + ScriptLang::Powershell => "powershell", + ScriptLang::Postgresql => "postgresql", + ScriptLang::Mysql => "mysql", + ScriptLang::Bigquery => "bigquery", + ScriptLang::Snowflake => "snowflake", + ScriptLang::Mssql => "mssql", + ScriptLang::Graphql => "graphql", + ScriptLang::OracleDB => "oracledb", + ScriptLang::DuckDb => "duckdb", + ScriptLang::Php => "php", + ScriptLang::Rust => "rust", + ScriptLang::Ansible => "ansible", + ScriptLang::CSharp => "csharp", + ScriptLang::Nu => "nu", + ScriptLang::Java => "java", + ScriptLang::Ruby => "ruby", + // for related places search: ADD_NEW_LANG + } + } + + pub fn as_dependencies_filename(&self) -> Option { + use ScriptLang::*; + Some( + match self { + Bun | Bunnative | Nativets => "package.json", + Python3 => "requirements.in", + // Go => "go.mod", + Php => "composer.json", + _ => return None, + } + .to_owned(), + ) + } + + pub fn as_comment_lit(&self) -> String { + use ScriptLang::*; + match self { + Nativets | Bun | Bunnative | Deno | Go | Php | CSharp | Java => "//", + Python3 | Bash | Powershell | Graphql | Ansible | Nu | Ruby => "#", + Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", + Rust => "//!", + // for related places search: ADD_NEW_LANG + } + .to_owned() + } +} + +impl FromStr for ScriptLang { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + let language = match s.to_lowercase().as_str() { + "bun" => ScriptLang::Bun, + "bunnative" => ScriptLang::Bunnative, + "nativets" => ScriptLang::Nativets, + "deno" => ScriptLang::Deno, + "python3" => ScriptLang::Python3, + "go" => ScriptLang::Go, + "bash" => ScriptLang::Bash, + "powershell" => ScriptLang::Powershell, + "postgresql" => ScriptLang::Postgresql, + "mysql" => ScriptLang::Mysql, + "bigquery" => ScriptLang::Bigquery, + "snowflake" => ScriptLang::Snowflake, + "mssql" => ScriptLang::Mssql, + "graphql" => ScriptLang::Graphql, + "oracledb" => ScriptLang::OracleDB, + "php" => ScriptLang::Php, + "rust" => ScriptLang::Rust, + "ansible" => ScriptLang::Ansible, + "csharp" => ScriptLang::CSharp, + "nu" => ScriptLang::Nu, + "java" => ScriptLang::Java, + "ruby" => ScriptLang::Ruby, + // for related places search: ADD_NEW_LANG + language => { + return Err(anyhow::anyhow!("{} is currently not supported", language)) + } + }; + + Ok(language) + } +} + +#[derive(Eq, PartialEq, Debug, Hash, Clone, Copy, sqlx::Type)] +#[sqlx(transparent)] +pub struct ScriptHash(pub i64); + +impl Deref for ScriptHash { + type Target = i64; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Into for ScriptHash { + fn into(self) -> u64 { + self.0 as u64 + } +} + +impl From for ScriptHash { + fn from(value: i64) -> Self { + Self(value) + } +} + +#[derive(PartialEq, sqlx::Type, Debug)] +#[sqlx(transparent, no_pg_array)] +pub struct ScriptHashes(pub Vec); + +impl Display for ScriptHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", to_hex_string(&self.0)) + } +} +impl Serialize for ScriptHash { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + serializer.serialize_str(to_hex_string(&self.0).as_str()) + } +} +impl<'de> Deserialize<'de> for ScriptHash { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + let i = to_i64(&s).map_err(|e| { + tracing::error!("Could not deserialize ScriptHash. Note, input should be in Hex and digit amount should be divisible by 16 (can be padded). err: {}", &e); + D::Error::custom(format!("{}", e)) + })?; + Ok(ScriptHash(i)) + } +} + +impl Serialize for ScriptHashes { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for element in &self.0 { + seq.serialize_element(&ScriptHash(*element))?; + } + seq.end() + } +} + +#[derive(Serialize, Deserialize, Debug, Hash, sqlx::Type)] +#[sqlx(type_name = "SCRIPT_KIND", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum ScriptKind { + Trigger, + Failure, + Script, + Approval, + Preprocessor, +} + +impl Display for ScriptKind { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.write_str(match self { + ScriptKind::Trigger => "trigger", + ScriptKind::Failure => "failure", + ScriptKind::Script => "script", + ScriptKind::Approval => "approval", + ScriptKind::Preprocessor => "preprocessor", + })?; + Ok(()) + } +} + +const PREVIEW_IS_CODEBASE_HASH: i64 = -42; +const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; +const PREVIEW_IS_ESM_CODEBASE_HASH: i64 = -44; +const PREVIEW_IS_TAR_ESM_CODEBASE_HASH: i64 = -45; + +pub fn is_special_codebase_hash(hash: i64) -> bool { + hash == PREVIEW_IS_CODEBASE_HASH + || hash == PREVIEW_IS_TAR_CODEBASE_HASH + || hash == PREVIEW_IS_ESM_CODEBASE_HASH + || hash == PREVIEW_IS_TAR_ESM_CODEBASE_HASH +} + +pub fn codebase_to_hash(is_tar: bool, is_esm: bool) -> i64 { + if is_tar { + if is_esm { + PREVIEW_IS_TAR_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_TAR_CODEBASE_HASH + } + } else { + if is_esm { + PREVIEW_IS_ESM_CODEBASE_HASH + } else { + PREVIEW_IS_CODEBASE_HASH + } + } +} + +pub fn hash_to_codebase_id(job_id: &str, hash: i64) -> Option { + match hash { + PREVIEW_IS_CODEBASE_HASH => Some(job_id.to_string()), + PREVIEW_IS_TAR_CODEBASE_HASH => Some(format!("{}.tar", job_id)), + PREVIEW_IS_ESM_CODEBASE_HASH => Some(format!("{}.esm", job_id)), + PREVIEW_IS_TAR_ESM_CODEBASE_HASH => Some(format!("{}.esm.tar", job_id)), + _ => None, + } +} + +pub struct CodebaseInfo { + pub is_tar: bool, + pub is_esm: bool, +} + +pub fn id_to_codebase_info(id: &str) -> CodebaseInfo { + let is_tar = id.ends_with(".tar"); + let is_esm = id.contains(".esm"); + CodebaseInfo { is_tar, is_esm } +} + +#[derive(Serialize, sqlx::FromRow, Debug)] +pub struct Script { + pub workspace_id: String, + pub hash: ScriptHash, + pub path: String, + pub parent_hashes: Option, + pub summary: String, + pub description: String, + pub content: String, + pub created_by: String, + pub created_at: chrono::DateTime, + pub archived: bool, + pub schema: Option, + pub deleted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_template: Option, + pub extra_perms: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub lock: Option, + pub lock_error_logs: Option, + pub language: ScriptLang, + pub kind: ScriptKind, + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub envs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub dedicated_worker: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub restart_unless_cancelled: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub visible_to_runner_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_main_func: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub codebase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, + #[serde(flatten)] + #[sqlx(flatten)] + pub runnable_settings: SR, +} + +// Not serializable +#[derive(sqlx::FromRow, Debug, Clone)] +pub struct ScriptRunnableSettingsHandle { + // legacy - for backwards compatibility + // don't add new values. + pub concurrency_key: Option, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub debounce_key: Option, + pub debounce_delay_s: Option, + + // add here as well. + pub runnable_settings_handle: Option, +} + +// Not sqlx queriable +#[derive(Serialize, Debug, Clone, Default)] +pub struct ScriptRunnableSettingsInline { + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct ScriptWithStarred { + #[sqlx(flatten)] + #[serde(flatten)] + pub script: Script, + #[serde(skip_serializing_if = "Option::is_none")] + pub starred: Option, +} + +#[derive(Serialize, sqlx::FromRow)] +pub struct ListableScript { + pub hash: ScriptHash, + pub path: String, + pub summary: String, + pub created_at: chrono::DateTime, + pub archived: bool, + pub extra_perms: serde_json::Value, + pub language: ScriptLang, + pub starred: bool, + pub tag: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_draft: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_only: Option, + pub has_deploy_errors: bool, + pub ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_main_func: Option, + #[serde(skip_serializing_if = "is_false")] + pub use_codebase: bool, + #[sqlx(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, + pub kind: ScriptKind, +} + +fn is_false(x: &bool) -> bool { + return !x; +} + +#[derive(Serialize)] +pub struct ScriptHistory { + pub script_hash: ScriptHash, + #[serde(skip_serializing_if = "Option::is_none")] + pub deployment_msg: Option, +} + +#[derive(Deserialize)] +pub struct ScriptHistoryUpdate { + pub deployment_msg: Option, +} + +#[derive(Serialize, Deserialize, Debug, sqlx::Type, Clone)] +#[sqlx(transparent)] +#[serde(transparent)] +pub struct Schema(pub sqlx::types::Json>); + +impl Hash for Schema { + fn hash(&self, state: &mut H) { + self.0.get().hash(state); + } +} + +#[derive(Serialize, Deserialize, Hash, Debug)] +pub struct NewScript { + pub path: String, + pub parent_hash: Option, + pub summary: String, + pub description: String, + pub content: String, + pub schema: Option, + pub is_template: Option, + #[serde(default = "Option::default")] + #[serde(deserialize_with = "lock_deserialize")] + pub lock: Option, + pub language: ScriptLang, + pub kind: Option, + pub tag: Option, + pub draft_only: Option, + pub envs: Option>, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub dedicated_worker: Option, + pub ws_error_handler_muted: Option, + pub priority: Option, + pub timeout: Option, + pub delete_after_use: Option, + pub restart_unless_cancelled: Option, + pub deployment_message: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub visible_to_runner_only: Option, + pub no_main_func: Option, + pub codebase: Option, + pub has_preprocessor: Option, + pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub assets: Option>, +} + +fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::de::Deserializer<'de>, +{ + struct StringOrArrayVisitor; + + impl<'de> serde::de::Visitor<'de> for StringOrArrayVisitor { + type Value = Option; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("either a string or an array of strings") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(Some(v.to_string())) + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_unit(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut split_lock: Vec = vec![]; + loop { + if let Ok(Some(elem)) = seq.next_element::() { + split_lock.push(elem); + } else { + break; + } + } + let lock = split_lock.join("\n"); + return Ok(Some(lock)); + } + } + deserializer.deserialize_any(StringOrArrayVisitor) +} + +#[derive(Debug, Deserialize)] +pub struct ListScriptQuery { + pub without_description: Option, + pub path_start: Option, + pub path_exact: Option, + pub created_by: Option, + pub first_parent_hash: Option, + pub last_parent_hash: Option, + pub parent_hash: Option, + pub show_archived: Option, + pub order_by: Option, + pub order_desc: Option, + pub is_template: Option, + pub kinds: Option, + pub starred_only: Option, + pub include_without_main: Option, + pub include_draft_only: Option, + pub with_deployment_msg: Option, + #[serde(default, deserialize_with = "from_seq")] + pub languages: Option>, + pub dedicated_worker: Option, +} + +fn from_seq<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let s = ::deserialize(deserializer)?; + + let languages: Vec = s + .split(",") + .map(ScriptLang::from_str) + .try_collect() + .map_err(|e: anyhow::Error| serde::de::Error::custom(e.to_string()))?; + + let languages = if languages.is_empty() { + None + } else { + Some(languages) + }; + + Ok(languages) +} + +pub fn to_i64(s: &str) -> anyhow::Result { + let v = hex::decode(s)?; + if v.len() < 8 { + return Err(anyhow::anyhow!( + "hex string did not decode to an u64: {s}", + )); + } + let nb: u64 = u64::from_be_bytes( + v[0..8] + .try_into() + .map_err(|_| hex::FromHexError::InvalidStringLength)?, + ); + Ok(nb as i64) +} + +pub fn to_hex_string(i: &i64) -> String { + hex::encode(i.to_be_bytes()) +} + +#[derive(Deserialize, Serialize)] +pub struct HubScript { + pub content: String, + pub lockfile: Option, + pub language: ScriptLang, + pub schema: Box, + pub summary: Option, +} + +pub fn hash_script(ns: &NewScript) -> i64 { + let mut dh = std::hash::DefaultHasher::new(); + ns.hash(&mut dh); + dh.finish() as i64 +} diff --git a/backend/windmill-types/src/triggers.rs b/backend/windmill-types/src/triggers.rs new file mode 100644 index 0000000000..64d9a2ce80 --- /dev/null +++ b/backend/windmill-types/src/triggers.rs @@ -0,0 +1,94 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; +use strum::EnumIter; + +use crate::jobs::JobTriggerKind; + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, EnumIter)] +#[sqlx(type_name = "TRIGGER_KIND", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum TriggerKind { + Webhook, + Http, + Websocket, + Kafka, + DefaultEmail, + Email, + Nats, + Mqtt, + Sqs, + Postgres, + Gcp, + Nextcloud, +} + +impl TriggerKind { + pub fn to_key(&self) -> String { + match self { + TriggerKind::Webhook => "webhook".to_string(), + TriggerKind::Http => "http".to_string(), + TriggerKind::Websocket => "websocket".to_string(), + TriggerKind::Kafka => "kafka".to_string(), + TriggerKind::Email => "email".to_string(), + TriggerKind::DefaultEmail => "email".to_string(), + TriggerKind::Nats => "nats".to_string(), + TriggerKind::Mqtt => "mqtt".to_string(), + TriggerKind::Sqs => "sqs".to_string(), + TriggerKind::Postgres => "postgres".to_string(), + TriggerKind::Gcp => "gcp".to_string(), + TriggerKind::Nextcloud => "nextcloud".to_string(), + } + } +} + +impl fmt::Display for TriggerKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + TriggerKind::Webhook => "webhook", + TriggerKind::Http => "http", + TriggerKind::Websocket => "websocket", + TriggerKind::Kafka => "kafka", + TriggerKind::Email => "email", + TriggerKind::DefaultEmail => "default_email", + TriggerKind::Nats => "nats", + TriggerKind::Mqtt => "mqtt", + TriggerKind::Sqs => "sqs", + TriggerKind::Postgres => "postgres", + TriggerKind::Gcp => "gcp", + TriggerKind::Nextcloud => "nextcloud", + }; + write!(f, "{}", s) + } +} + +#[derive(Eq, PartialEq, Hash)] +pub enum HubOrWorkspaceId { + Hub, + WorkspaceId(String), +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub struct RunnableFormat { + pub version: RunnableFormatVersion, + pub has_preprocessor: bool, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub enum RunnableFormatVersion { + V1, + V2, +} + +pub type RunnableFormatCacheKey = (HubOrWorkspaceId, i64, TriggerKind); + +#[derive(Debug, Clone)] +pub struct TriggerMetadata { + pub trigger_path: Option, + pub trigger_kind: JobTriggerKind, +} + +impl TriggerMetadata { + pub fn new(trigger_path: Option, trigger_kind: JobTriggerKind) -> TriggerMetadata { + TriggerMetadata { trigger_path, trigger_kind } + } +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 8ba01c71ab..d95321dd77 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -13,37 +13,40 @@ default = [] private = [] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] +mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth +mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth bigquery = ["dep:gcp_auth"] benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"] parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] cloud = [] sqlx = [] -deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", - "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile", - "quickjs"] +deno_core = ["dep:windmill-runtime-nativets"] libffi_mac = ["dep:libffi-sys"] otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"] dind = ["dep:bollard"] php = ["dep:windmill-parser-php"] mysql = ["dep:mysql_async"] oracledb = ["dep:oracle"] -python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports"] +python = ["dep:windmill-parser-py", "dep:windmill-parser-py-imports", "windmill-dep-map/python"] csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] ruby = ["dep:windmill-parser-ruby"] duckdb = ["dep:libloading"] -quickjs = ["dep:rquickjs"] -bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock"] +quickjs = ["windmill-jseval/quickjs"] +bedrock = ["dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"] [dependencies] windmill-queue.workspace = true +windmill-dep-map.workspace = true windmill-audit.workspace = true # there isn't really a reason for audit-worth actions to happen in the worker. windmill-common = { workspace = true, default-features = false } +windmill-jseval.workspace = true +windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } windmill-macros.workspace = true windmill-parser.workspace = true @@ -63,9 +66,9 @@ windmill-parser-graphql.workspace = true windmill-parser-php = { workspace = true, optional = true } windmill-git-sync.workspace = true aws-sdk-bedrockruntime = { workspace = true, optional = true } -aws-config.workspace = true -aws-credential-types.workspace = true -aws-smithy-types.workspace = true +aws-config = { workspace = true, optional = true } +aws-credential-types = { workspace = true, optional = true } +aws-smithy-types = { workspace = true, optional = true } flume.workspace = true sqlx.workspace = true uuid.workspace = true @@ -94,20 +97,6 @@ once_cell.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true url.workspace = true -deno_telemetry = { workspace = true, optional = true } -deno_fetch = { workspace = true, optional = true } -deno_webidl = { workspace = true, optional = true } -deno_web = { workspace = true, optional = true } -deno_net = { workspace = true, optional = true } -deno_console = { workspace = true, optional = true } -deno_url = { workspace = true, optional = true } -deno_core = { workspace = true, optional = true } -deno_ast = { workspace = true, optional = true } -deno_tls = { workspace = true, optional = true } -deno_permissions = { workspace = true, optional = true } -deno_io = { workspace = true, optional = true } -deno_error = { workspace = true, optional = true } -rustls-pemfile = { workspace = true, optional = true } async-stream.workspace = true postgres-native-tls.workspace = true @@ -134,39 +123,23 @@ object_store = { workspace = true, optional = true} convert_case.workspace = true yaml-rust.workspace = true backon.workspace = true -winapi = { workspace = true, optional = true } pep440_rs.workspace = true process-wrap.workspace = true async-once-cell.workspace = true libloading = { workspace = true, optional = true } -opentelemetry-proto.workspace = true +opentelemetry-proto = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } tracing-opentelemetry = { workspace = true, optional = true } -prost.workspace = true +prost = { workspace = true, optional = true } axum.workspace = true bollard = { workspace = true, optional = true } oracle = { workspace = true, optional = true } -rquickjs = { workspace = true, optional = true } -hudsucker.workspace = true -hyper-http-proxy.workspace = true -hyper-tls.workspace = true -hyper-util.workspace = true -rcgen.workspace = true +hudsucker = { workspace = true, optional = true } +hyper-http-proxy = { workspace = true, optional = true } +hyper-tls = { workspace = true, optional = true } +hyper-util = { workspace = true, optional = true } +rcgen = { workspace = true, optional = true } [build-dependencies] -deno_fetch = { workspace = true, optional = true } -deno_webidl = { workspace = true, optional = true } -deno_web = { workspace = true, optional = true } -deno_net = { workspace = true, optional = true } -deno_console = { workspace = true, optional = true } -deno_url = { workspace = true, optional = true } -deno_core = { workspace = true, optional = true } -deno_ast = { workspace = true, optional = true } -deno_tls = { workspace = true, optional = true } -deno_permissions = { workspace = true, optional = true } -deno_io = { workspace = true, optional = true } -deno_runtime = { workspace = true, optional = true } -deno_telemetry = { workspace = true, optional = true } -winapi = { workspace = true, optional = true } libffi-sys = { workspace = true, optional = true } diff --git a/backend/windmill-worker/build.rs b/backend/windmill-worker/build.rs index 8534e10edf..f328e4d9d0 100644 --- a/backend/windmill-worker/build.rs +++ b/backend/windmill-worker/build.rs @@ -1,154 +1 @@ -#[cfg(feature = "deno_core")] -use deno_fetch::FetchPermissions; -#[cfg(feature = "deno_core")] -use deno_net::NetPermissions; -#[cfg(feature = "deno_core")] -use deno_web::{BlobStore, TimersPermission}; -#[cfg(feature = "deno_core")] -use std::borrow::Cow; -#[cfg(feature = "deno_core")] -use std::env; -#[cfg(feature = "deno_core")] -use std::io::Write; -#[cfg(feature = "deno_core")] -use std::path::{Path, PathBuf}; -#[cfg(feature = "deno_core")] -use std::sync::Arc; - -// #[cfg(feature = "deno_core")] -pub struct PermissionsContainer; - -#[cfg(feature = "deno_core")] -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - _resolved: bool, - _p: &'a std::path::Path, - _api_name: &str, - ) -> Result, deno_io::fs::FsError> { - unreachable!("snapshotting") - } -} - -#[cfg(feature = "deno_core")] -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -#[cfg(feature = "deno_core")] -impl NetPermissions for PermissionsContainer { - fn check_read<'a>( - &mut self, - _p: &'a str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_write<'a>( - &mut self, - _p: &'a str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_net>( - &mut self, - _host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - fn check_write_path<'a>( - &mut self, - _: &'a Path, - _: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - todo!() - } -} - -#[cfg(feature = "deno_core")] -deno_core::extension!( - fetch, - esm_entry_point = "ext:fetch/src/runtime.js", - esm = ["src/runtime.js"], -); - -#[cfg(feature = "deno_core")] -fn main() { - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); - println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - - let exts = vec![ - deno_telemetry::deno_telemetry::init_ops_and_esm(), - deno_webidl::deno_webidl::init_ops_and_esm(), - deno_url::deno_url::init_ops_and_esm(), - deno_console::deno_console::init_ops_and_esm(), - deno_web::deno_web::init_ops_and_esm::( - Arc::new(BlobStore::default()), - None, - ), - deno_fetch::deno_fetch::init_ops_and_esm::(Default::default()), - deno_net::deno_net::init_ops_and_esm::(None, None), - fetch::init_ops_and_esm(), - ]; - - // Build the file path to the snapshot. - let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let snapshot_path = o.join("FETCH_SNAPSHOT.bin"); - - // Create the snapshot. - let output = deno_core::snapshot::create_snapshot( - deno_core::snapshot::CreateSnapshotOptions { - cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), - startup_snapshot: None, - extension_transpiler: Some(std::rc::Rc::new(|specifier, source| { - deno_runtime::transpile::maybe_transpile_source(specifier, source) - })), - extensions: exts, - with_runtime_cb: None, - skip_op_registration: false, - }, - None, - ) - .unwrap(); - - // NOTE(bartlomieju): Compressing the TSC snapshot in debug build took - // ~45s on M1 MacBook Pro; without compression it took ~1s. - // Thus we're not using compressed snapshot, trading off - // a lot of build time for some startup time in debug build. - let mut file = std::fs::File::create(snapshot_path).unwrap(); - // if cfg!(debug_assertions) { - file.write_all(&output.output).unwrap(); - // } else { - // let mut vec = Vec::with_capacity(output.output.len()); - // vec.extend((output.output.len() as u32).to_le_bytes()); - // vec.extend_from_slice( - // &zstd::bulk::compress(&output.output, 22).expect("snapshot compression failed"), - // ); - // file.write_all(&vec).unwrap(); - // } - - for path in output.files_loaded_during_snapshot { - println!("cargo:rerun-if-changed={}", path.display()); - } -} - -#[cfg(not(feature = "deno_core"))] fn main() {} diff --git a/backend/windmill-worker/src/ai/providers/anthropic.rs b/backend/windmill-worker/src/ai/providers/anthropic.rs index 726a8e27e2..8eb3609816 100644 --- a/backend/windmill-worker/src/ai/providers/anthropic.rs +++ b/backend/windmill-worker/src/ai/providers/anthropic.rs @@ -500,7 +500,11 @@ impl QueryBuilder for AnthropicQueryBuilder { // For Vertex AI, the model is specified in the URL path // Expected base_url format: https://{region}-aiplatform.googleapis.com/v1/projects/{project}/locations/{location}/publishers/anthropic/models // We append the model and :streamRawPredict - format!("{}/{}:streamRawPredict", base_url.trim_end_matches('/'), model) + format!( + "{}/{}:streamRawPredict", + base_url.trim_end_matches('/'), + model + ) } else { format!("{}/messages", base_url) } diff --git a/backend/windmill-worker/src/ai/providers/bedrock.rs b/backend/windmill-worker/src/ai/providers/bedrock.rs index a98fed44af..a86bf3250c 100644 --- a/backend/windmill-worker/src/ai/providers/bedrock.rs +++ b/backend/windmill-worker/src/ai/providers/bedrock.rs @@ -17,13 +17,13 @@ use std::collections::HashMap; use windmill_common::{client::AuthedClient, error::Error}; // Re-export from shared module for use by other parts of the worker -pub use windmill_common::ai_bedrock::{check_env_credentials, BedrockClient}; use windmill_common::ai_bedrock::{ bedrock_stream_event_is_block_stop, bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_start, build_tool_config, create_inference_config, format_bedrock_error, openai_messages_to_bedrock, streaming_tool_calls_to_openai, StreamingToolCall, }; +pub use windmill_common::ai_bedrock::{check_env_credentials, BedrockClient}; // ============================================================================ // Query Builder (Worker-specific orchestration) @@ -161,9 +161,7 @@ impl BedrockQueryBuilder { if let Some(processor) = stream_event_processor.as_ref() { processor .send( - StreamingEvent::TokenDelta { - content: text_delta, - }, + StreamingEvent::TokenDelta { content: text_delta }, &mut events_str, ) .await?; @@ -185,9 +183,8 @@ impl BedrockQueryBuilder { } // Extract usage from Metadata event - if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata( - metadata, - ) = &event + if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = + &event { if let Some(token_usage) = metadata.usage() { usage = Some( diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index 7f841ea07f..51bb6e3ec3 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -473,8 +473,9 @@ impl QueryBuilder for OpenAIQueryBuilder { parser.parse_events(response).await?; // Convert OpenAI Responses usage to TokenUsage - let usage = - parser.usage.map(|u| TokenUsage::new(u.input_tokens, u.output_tokens, u.total_tokens)); + let usage = parser + .usage + .map(|u| TokenUsage::new(u.input_tokens, u.output_tokens, u.total_tokens)); Ok(ParsedResponse::Text { content: if parser.accumulated_content.is_empty() { @@ -518,9 +519,7 @@ impl QueryBuilder for OpenAIQueryBuilder { "image_generation_call" => { if output.status.as_deref() == Some("completed") { if let Some(ref base64_image) = output.result { - return Ok(ParsedResponse::Image { - base64_data: base64_image.clone(), - }); + return Ok(ParsedResponse::Image { base64_data: base64_image.clone() }); } } } diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs index e15ac94ec7..62cee60420 100644 --- a/backend/windmill-worker/src/ai/providers/openrouter.rs +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -117,9 +117,7 @@ impl QueryBuilder for OpenRouterQueryBuilder { .and_then(|images| images.first()) { if let Some(base64_data) = image.image_url.url.strip_prefix("data:image/png;base64,") { - return Ok(ParsedResponse::Image { - base64_data: base64_data.to_string(), - }); + return Ok(ParsedResponse::Image { base64_data: base64_data.to_string() }); } } diff --git a/backend/windmill-worker/src/ai/providers/other.rs b/backend/windmill-worker/src/ai/providers/other.rs index e000cd11b7..a92caca923 100644 --- a/backend/windmill-worker/src/ai/providers/other.rs +++ b/backend/windmill-worker/src/ai/providers/other.rs @@ -225,8 +225,8 @@ impl QueryBuilder for OtherQueryBuilder { } // Convert OpenAI Chat Completions usage to TokenUsage - let usage = - openai_usage.map(|u| TokenUsage::new(u.prompt_tokens, u.completion_tokens, u.total_tokens)); + let usage = openai_usage + .map(|u| TokenUsage::new(u.prompt_tokens, u.completion_tokens, u.total_tokens)); Ok(ParsedResponse::Text { content: if accumulated_content.is_empty() { diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 2bbf93c69a..10e510a492 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -7,10 +7,8 @@ use windmill_queue::MiniPulledJob; use crate::{ ai::{ providers::{ - anthropic::AnthropicQueryBuilder, - google_ai::GoogleAIQueryBuilder, - openai::{OpenAIQueryBuilder}, - openrouter::OpenRouterQueryBuilder, + anthropic::AnthropicQueryBuilder, google_ai::GoogleAIQueryBuilder, + openai::OpenAIQueryBuilder, openrouter::OpenRouterQueryBuilder, other::OtherQueryBuilder, }, types::*, diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 34a2d789ec..5db641b80d 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -1,4 +1,3 @@ -use windmill_common::ai_types::OpenAIToolCall; use crate::ai::query_builder::StreamEventProcessor; use crate::ai::types::McpToolSource; use crate::ai::types::*; @@ -21,6 +20,7 @@ use mappable_rc::Marc; use serde_json::value::RawValue; use std::{collections::HashMap, sync::Arc}; use uuid::Uuid; +use windmill_common::ai_types::OpenAIToolCall; use windmill_common::flows::InputTransform; use windmill_common::jobs::JobPayload; diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index 0955e7c7a1..4177680e43 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -174,10 +174,18 @@ pub struct ProviderResource { #[serde(default, deserialize_with = "empty_string_as_none")] pub region: Option, #[allow(dead_code)] - #[serde(alias = "awsAccessKeyId", default, deserialize_with = "empty_string_as_none")] + #[serde( + alias = "awsAccessKeyId", + default, + deserialize_with = "empty_string_as_none" + )] pub aws_access_key_id: Option, #[allow(dead_code)] - #[serde(alias = "awsSecretAccessKey", default, deserialize_with = "empty_string_as_none")] + #[serde( + alias = "awsSecretAccessKey", + default, + deserialize_with = "empty_string_as_none" + )] pub aws_secret_access_key: Option, /// Platform for Anthropic API (standard or google_vertex_ai) #[serde(default)] @@ -202,10 +210,7 @@ impl ProviderWithResource { pub async fn get_base_url(&self, db: &DB) -> Result { self.kind - .get_base_url( - self.resource.base_url.clone(), - db, - ) + .get_base_url(self.resource.base_url.clone(), db) .await } @@ -295,8 +300,10 @@ impl TokenUsage { self.total_tokens = add_option(self.total_tokens, other.total_tokens); self.cache_read_input_tokens = add_option(self.cache_read_input_tokens, other.cache_read_input_tokens); - self.cache_write_input_tokens = - add_option(self.cache_write_input_tokens, other.cache_write_input_tokens); + self.cache_write_input_tokens = add_option( + self.cache_write_input_tokens, + other.cache_write_input_tokens, + ); } } @@ -453,6 +460,11 @@ impl OpenAPISchema { Typ::Bool => Self::from_str("boolean"), Typ::Bytes => Self::from_str("string"), Typ::Datetime => Self::datetime(), + Typ::Date => Self { + r#type: Some(SchemaType::Single("string".to_string())), + format: Some("date".to_string()), + ..Default::default() + }, Typ::Resource(_) => Self::from_str("string"), Typ::Email => Self::from_str("string"), Typ::Sql => Self::from_str("string"), @@ -704,7 +716,9 @@ impl OpenAPISchema { if let Some(ref other_definitions) = other.definitions { let definitions = self.definitions.get_or_insert_with(HashMap::new); for (key, value) in other_definitions { - definitions.entry(key.clone()).or_insert_with(|| value.clone()); + definitions + .entry(key.clone()) + .or_insert_with(|| value.clone()); } } @@ -861,7 +875,10 @@ mod tests { schema.make_strict(); assert!( - matches!(schema.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + schema.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Expected additionalProperties to be false" ); } @@ -874,33 +891,36 @@ mod tests { schema.make_strict(); assert!( - matches!(schema.additional_properties, Some(AdditionalProperties::Bool(true))), + matches!( + schema.additional_properties, + Some(AdditionalProperties::Bool(true)) + ), "Expected additionalProperties to remain true (user-specified)" ); } #[test] fn test_make_strict_all_properties_required() { - let mut schema = object_schema(vec![ - ("name", string_schema()), - ("age", integer_schema()), - ]); + let mut schema = object_schema(vec![("name", string_schema()), ("age", integer_schema())]); schema.required = Some(vec!["name".to_string()]); // Only name is required initially schema.make_strict(); let required = schema.required.as_ref().expect("required should be set"); - assert!(required.contains(&"name".to_string()), "name should be required"); - assert!(required.contains(&"age".to_string()), "age should be required"); + assert!( + required.contains(&"name".to_string()), + "name should be required" + ); + assert!( + required.contains(&"age".to_string()), + "age should be required" + ); assert_eq!(required.len(), 2, "Should have exactly 2 required fields"); } #[test] fn test_make_strict_non_required_becomes_nullable() { - let mut schema = object_schema(vec![ - ("name", string_schema()), - ("age", integer_schema()), - ]); + let mut schema = object_schema(vec![("name", string_schema()), ("age", integer_schema())]); schema.required = Some(vec!["name".to_string()]); // Only name is required schema.make_strict(); @@ -915,7 +935,10 @@ mod tests { match &age_prop.r#type { Some(SchemaType::Multiple(types)) => { - assert!(types.contains(&"integer".to_string()), "Should contain integer"); + assert!( + types.contains(&"integer".to_string()), + "Should contain integer" + ); assert!(types.contains(&"null".to_string()), "Should contain null"); } _ => panic!("Expected age to have multiple types including null"), @@ -953,12 +976,18 @@ mod tests { .expect("nested property should exist"); assert!( - matches!(nested_prop.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + nested_prop.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Nested object should have additionalProperties: false" ); // Nested object should have all properties required - let nested_required = nested_prop.required.as_ref().expect("nested required should be set"); + let nested_required = nested_prop + .required + .as_ref() + .expect("nested required should be set"); assert!(nested_required.contains(&"field".to_string())); } @@ -975,7 +1004,10 @@ mod tests { let items = schema.items.as_ref().expect("items should exist"); assert!( - matches!(items.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + items.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Array items should have additionalProperties: false" ); } @@ -994,7 +1026,10 @@ mod tests { for (i, variant) in schema.one_of.as_ref().unwrap().iter().enumerate() { assert!( - matches!(variant.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + variant.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "oneOf variant {} should have additionalProperties: false", i ); @@ -1015,7 +1050,10 @@ mod tests { for (i, variant) in schema.any_of.as_ref().unwrap().iter().enumerate() { assert!( - matches!(variant.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + variant.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "anyOf variant {} should have additionalProperties: false", i ); @@ -1028,10 +1066,7 @@ mod tests { let mut defs = HashMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); - let mut schema = OpenAPISchema { - defs: Some(defs), - ..Default::default() - }; + let mut schema = OpenAPISchema { defs: Some(defs), ..Default::default() }; schema.make_strict(); @@ -1043,7 +1078,10 @@ mod tests { .expect("MyType def should exist"); assert!( - matches!(my_type.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + my_type.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "$defs schema should have additionalProperties: false" ); } @@ -1054,10 +1092,7 @@ mod tests { let mut definitions = HashMap::new(); definitions.insert("MyType".to_string(), Box::new(def_schema)); - let mut schema = OpenAPISchema { - definitions: Some(definitions), - ..Default::default() - }; + let mut schema = OpenAPISchema { definitions: Some(definitions), ..Default::default() }; schema.make_strict(); @@ -1069,7 +1104,10 @@ mod tests { .expect("MyType definition should exist"); assert!( - matches!(my_type.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + my_type.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "definitions schema should have additionalProperties: false" ); } @@ -1168,7 +1206,10 @@ mod tests { schema.make_strict(); // allOf should be removed - assert!(schema.all_of.is_none(), "allOf should be removed after flattening"); + assert!( + schema.all_of.is_none(), + "allOf should be removed after flattening" + ); // Properties should be merged let props = schema.properties.as_ref().expect("properties should exist"); @@ -1183,7 +1224,10 @@ mod tests { // Should have additionalProperties: false (from make_strict) assert!( - matches!(schema.additional_properties, Some(AdditionalProperties::Bool(false))), + matches!( + schema.additional_properties, + Some(AdditionalProperties::Bool(false)) + ), "Should have additionalProperties: false" ); } @@ -1205,8 +1249,14 @@ mod tests { // Both name and age should be in required (from merge + make_strict makes all required) let required = schema.required.as_ref().expect("required should be set"); - assert!(required.contains(&"name".to_string()), "name should be required"); - assert!(required.contains(&"age".to_string()), "age should be required"); + assert!( + required.contains(&"name".to_string()), + "name should be required" + ); + assert!( + required.contains(&"age".to_string()), + "age should be required" + ); } #[test] @@ -1265,7 +1315,7 @@ mod tests { let schema2 = OpenAPISchema { r#type: Some(SchemaType::Single("integer".to_string())), - minimum: Some(5.0), // More restrictive + minimum: Some(5.0), // More restrictive maximum: Some(100.0), ..Default::default() }; @@ -1278,8 +1328,16 @@ mod tests { schema.flatten_all_of(); // Should take the more restrictive minimum (5.0) - assert_eq!(schema.minimum, Some(5.0), "Should have more restrictive minimum"); - assert_eq!(schema.maximum, Some(100.0), "Should have maximum from schema2"); + assert_eq!( + schema.minimum, + Some(5.0), + "Should have more restrictive minimum" + ); + assert_eq!( + schema.maximum, + Some(100.0), + "Should have maximum from schema2" + ); } #[test] @@ -1288,15 +1346,10 @@ mod tests { let mut defs = HashMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); - let schema_with_defs = OpenAPISchema { - defs: Some(defs), - ..Default::default() - }; + let schema_with_defs = OpenAPISchema { defs: Some(defs), ..Default::default() }; - let mut schema = OpenAPISchema { - all_of: Some(vec![Box::new(schema_with_defs)]), - ..Default::default() - }; + let mut schema = + OpenAPISchema { all_of: Some(vec![Box::new(schema_with_defs)]), ..Default::default() }; schema.flatten_all_of(); @@ -1343,9 +1396,15 @@ mod tests { schema.sanitize_for_google(); - assert!(schema.schema_url.is_none(), "Root $schema should be removed"); + assert!( + schema.schema_url.is_none(), + "Root $schema should be removed" + ); let field = schema.properties.as_ref().unwrap().get("field").unwrap(); - assert!(field.schema_url.is_none(), "Nested $schema should be removed"); + assert!( + field.schema_url.is_none(), + "Nested $schema should be removed" + ); } #[test] @@ -1365,7 +1424,10 @@ mod tests { schema.sanitize_for_google(); let items = schema.items.as_ref().unwrap(); - assert!(items.schema_url.is_none(), "Array items $schema should be removed"); + assert!( + items.schema_url.is_none(), + "Array items $schema should be removed" + ); } #[test] @@ -1376,15 +1438,16 @@ mod tests { ..Default::default() }; - let mut schema = OpenAPISchema { - one_of: Some(vec![Box::new(variant)]), - ..Default::default() - }; + let mut schema = + OpenAPISchema { one_of: Some(vec![Box::new(variant)]), ..Default::default() }; schema.sanitize_for_google(); let variant = &schema.one_of.as_ref().unwrap()[0]; - assert!(variant.schema_url.is_none(), "oneOf variant $schema should be removed"); + assert!( + variant.schema_url.is_none(), + "oneOf variant $schema should be removed" + ); } #[test] @@ -1405,9 +1468,15 @@ mod tests { schema.sanitize_for_google(); - assert!(schema.schema_url.is_none(), "Root $schema should be removed"); + assert!( + schema.schema_url.is_none(), + "Root $schema should be removed" + ); let my_type = schema.defs.as_ref().unwrap().get("MyType").unwrap(); - assert!(my_type.schema_url.is_none(), "$defs schema $schema should be removed"); + assert!( + my_type.schema_url.is_none(), + "$defs schema $schema should be removed" + ); } #[test] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index c576ad9854..0b820e3816 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -21,7 +21,7 @@ use windmill_mcp::McpClient; #[cfg(not(feature = "mcp"))] use crate::ai::tools::McpClientStub as McpClient; use windmill_common::{ - ai_providers::{AIProvider}, + ai_providers::AIProvider, cache, client::AuthedClient, db::DB, @@ -417,7 +417,7 @@ pub async fn run_agent( args.provider.get_base_url(db).await? }; let api_key = args.provider.get_api_key().unwrap_or(""); - + // Create the query builder for the provider let query_builder = create_query_builder(&args.provider); @@ -666,7 +666,10 @@ pub async fn run_agent( let parsed = if args.provider.kind == AIProvider::AWSBedrock { #[cfg(feature = "bedrock")] { - let region = args.provider.get_region().unwrap_or(windmill_common::ai_providers::USE_ENV_REGION); + let region = args + .provider + .get_region() + .unwrap_or(windmill_common::ai_providers::USE_ENV_REGION); // Use Bedrock SDK via dedicated query builder crate::ai::providers::bedrock::BedrockQueryBuilder::default() .execute_request( @@ -770,10 +773,8 @@ pub async fn run_agent( .build_request_without_usage(&build_args, client, &job.workspace_id) .await?; - let retry_resp = build_http_request(retry_body) - .send() - .await - .map_err(|e| { + let retry_resp = + build_http_request(retry_body).send().await.map_err(|e| { Error::internal_err(format!("Failed to call API on retry: {}", e)) })?; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 5795c1ccd7..d1e54f97ad 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -752,10 +752,12 @@ pub async fn get_git_ssh_cmd( })?; content.push_str("\n"); - let file = write_file(job_dir, &id_file_name, &content)?; + #[cfg(not(unix))] + let _ = write_file(job_dir, &id_file_name, &content)?; #[cfg(unix)] { + let file = write_file(job_dir, &id_file_name, &content)?; let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); file.set_permissions(perm)?; } @@ -1218,10 +1220,10 @@ fi ANSIBLE_PLAYBOOK_PATH.as_str() ); - let file = write_file(job_dir, "wrapper.sh", &wrapper)?; + let _file = write_file(job_dir, "wrapper.sh", &wrapper)?; #[cfg(unix)] - file.metadata()?.permissions().set_mode(0o777); + _file.metadata()?.permissions().set_mode(0o777); // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 2d2d143eba..38183ed774 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -333,10 +333,7 @@ pub async fn read_file_bytes(path: &str) -> error::Result> { Ok(content) } -//this skips more steps than from_str at the cost of being unsafe. The source must ALWAUS gemerate valid json or this can cause UB in the worst case -pub fn unsafe_raw(json: String) -> Box { - unsafe { std::mem::transmute::, Box>(json.into()) } -} +pub use windmill_common::utils::unsafe_raw; fn check_result_too_big(size: usize) -> error::Result<()> { if *CLOUD_HOSTED && size > MAX_RESULT_SIZE { @@ -617,10 +614,25 @@ impl OccupancyMetrics { //drop all elements before the oldest one in 30m windows metrics.worker_occupancy_rate_history.drain(..index30m); + // Only report occupancy for a window if the worker has been running + // long enough to have meaningful data for that window. Otherwise, + // short-lived workers would report misleadingly high occupancy rates. ( - Some(total_occupation_15s), - Some(total_occupation_5m), - Some(total_occupation_30m), + if elapsed >= 15.0 { + Some(total_occupation_15s) + } else { + None + }, + if elapsed >= 300.0 { + Some(total_occupation_5m) + } else { + None + }, + if elapsed >= 1800.0 { + Some(total_occupation_30m) + } else { + None + }, ) } else { (None, None, None) diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 314e1760d3..35f7d10938 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -30,7 +30,8 @@ use crate::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, DEV_CONF_NSJAIL, }, - handle_child::handle_child, get_proxy_envs_for_lang, + get_proxy_envs_for_lang, + handle_child::handle_child, CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 23977e47a2..606ea4ec11 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,11 +7,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, - start_child_process, OccupancyMetrics, StreamNotifier, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics, + StreamNotifier, }, + get_proxy_envs_for_lang, handle_child::handle_child, - get_proxy_envs_for_lang, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, + DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -94,7 +96,10 @@ async fn get_common_deno_proc_envs( } // Add proxy envs (including OTEL tracing proxy if enabled for deno) - for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno).await.unwrap_or_default() { + for (k, v) in get_proxy_envs_for_lang(&ScriptLang::Deno) + .await + .unwrap_or_default() + { deno_envs.insert(k.to_string(), v); } diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index b0475ee14a..d1b2bdff35 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,6 +1,6 @@ use crate::{common::MaybeLock, get_proxy_envs_for_lang}; -use windmill_common::scripts::ScriptLang; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; +use windmill_common::scripts::ScriptLang; use itertools::Itertools; use serde_json::value::RawValue; diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 6232f99be2..b3c8bfec28 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -26,8 +26,8 @@ use crate::{ }, handle_child, universal_pkg_installer::{par_install_language_dependencies_all_at_once, RequiredDependency}, - COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, - JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_REPOSITORY_DIR, + MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; use windmill_common::client::AuthedClient; diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 51b1007964..5ee3b34cf9 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -6,351 +6,23 @@ * LICENSE-AGPL for a copy of the license. */ -#[cfg(feature = "deno_core")] -use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc}; - use std::{collections::HashMap, sync::Arc}; -#[cfg(feature = "deno_core")] -use deno_ast::ParseParams; -#[cfg(feature = "deno_core")] -use deno_core::{ - op2, serde_v8, url, - v8::{self, IsolateHandle}, - Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, -}; -#[cfg(feature = "deno_core")] -use deno_fetch::FetchPermissions; -#[cfg(feature = "deno_core")] -use deno_net::NetPermissions; - -#[cfg(feature = "deno_core")] -use deno_web::{BlobStore, TimersPermission}; -#[cfg(feature = "deno_core")] -use itertools::Itertools; -use lazy_static::lazy_static; -#[cfg(feature = "quickjs")] -use once_cell::sync::Lazy; -use regex::Regex; use serde_json::value::RawValue; use sqlx::types::Json; - -#[cfg(feature = "deno_core")] -use tokio::{ - sync::{mpsc, oneshot}, - time::timeout, -}; use uuid::Uuid; -#[cfg(feature = "deno_core")] -use windmill_common::error::Error; -#[cfg(feature = "deno_core")] -use windmill_common::utils::configure_client; -#[cfg(feature = "deno_core")] -use windmill_common::worker::{write_file, TMP_DIR}; - -use windmill_common::flow_status::JobResult; +use windmill_common::client::AuthedClient; +use windmill_common::worker::Connection; use windmill_queue::CanceledBy; use crate::common::{OccupancyMetrics, StreamNotifier}; -use windmill_common::client::AuthedClient; #[cfg(feature = "deno_core")] -use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller}; +use crate::handle_child::run_future_with_polling_update_job_poller; -#[derive(Debug, Clone)] -pub struct IdContext { - pub flow_job: Uuid, - #[allow(dead_code)] - pub steps_results: HashMap, - pub previous_id: String, -} - -// #[cfg(feature = "deno_core")] -// pub struct ContainerRootCertStoreProvider { -// root_cert_store: RootCertStore, -// } - -// #[cfg(feature = "deno_core")] -// impl ContainerRootCertStoreProvider { -// fn new() -> ContainerRootCertStoreProvider { -// return ContainerRootCertStoreProvider { -// root_cert_store: deno_tls::create_default_root_cert_store(), -// }; -// } - -// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> { -// let cert_file = std::fs::File::open(cert_path)?; -// let mut reader = BufReader::new(cert_file); -// let pem_file = rustls_pemfile::certs(&mut reader).collect::, _>>()?; - -// self.root_cert_store.add_parsable_certificates(pem_file); -// Ok(()) -// } -// } - -// #[cfg(feature = "deno_core")] -// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { -// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { -// Ok(&self.root_cert_store) -// } -// } - -#[cfg(feature = "deno_core")] -pub struct PermissionsContainer; - -#[cfg(feature = "deno_core")] -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - Ok(()) - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - _resolved: bool, - p: &'a std::path::Path, - _api_name: &str, - ) -> Result, deno_io::fs::FsError> { - Ok(Cow::Borrowed(p)) - } -} - -#[cfg(feature = "deno_core")] -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -#[cfg(feature = "deno_core")] -impl NetPermissions for PermissionsContainer { - fn check_read<'a>( - &mut self, - p: &'a str, - _api_name: &str, - ) -> Result { - Ok(PathBuf::from(p)) - } - - fn check_write<'a>( - &mut self, - p: &'a str, - _api_name: &str, - ) -> Result { - Ok(PathBuf::from(p)) - } - - fn check_net>( - &mut self, - _host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - Ok(()) - } - - fn check_write_path<'a>( - &mut self, - p: &'a std::path::Path, - _api_name: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - Ok(Cow::Borrowed(p)) - } -} - -#[cfg(feature = "deno_core")] -pub struct OptAuthedClient(Option); - -const FLOW_INPUT_PREFIX: &'static str = "flow_input"; -const ENV_KEY_PREFIX: &'static str = "flow_env"; -const DOT_PATTERN: &'static str = "."; -const START_BRACKET_PATTERN: &'static str = "[\""; -const END_BRACKET_PATTERN: &'static str = "\"]"; - -/// Determines if we should prepend "return" to the expression -#[cfg(feature = "deno_core")] -fn should_add_return(expr: &str) -> bool { - // Trim whitespace - let trimmed = expr.trim(); - - // If it's empty, add return - if trimmed.is_empty() { - return true; - } - - // Check if it already starts with 'return' keyword (as a statement) - // Use word boundary to avoid matching "return" in variable names - if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" { - return false; - } - - // Check for common statement patterns that shouldn't have return prepended - let statement_prefixes = [ - "const ", - "let ", - "var ", - "if ", - "if(", - "for ", - "for(", - "while ", - "while(", - "switch ", - "switch(", - "try ", - "try{", - "throw ", - "function ", - "class ", - "async ", - "await ", - ]; - - for prefix in &statement_prefixes { - if trimmed.starts_with(prefix) { - return false; - } - } - - // Check for multiple statements (contains semicolon not in a string) - // This is still not perfect but better than current logic - if contains_semicolon_outside_strings(trimmed) { - return false; - } - - // Default: assume it's an expression that needs return - true -} - -/// Checks if the expression contains a semicolon outside of strings -#[cfg(feature = "deno_core")] -fn contains_semicolon_outside_strings(expr: &str) -> bool { - let mut in_single_quote = false; - let mut in_double_quote = false; - let mut in_template = false; - let mut prev_char = '\0'; - - for ch in expr.chars() { - match ch { - '\'' if prev_char != '\\' && !in_double_quote && !in_template => { - in_single_quote = !in_single_quote; - } - '"' if prev_char != '\\' && !in_single_quote && !in_template => { - in_double_quote = !in_double_quote; - } - '`' if prev_char != '\\' && !in_single_quote && !in_double_quote => { - in_template = !in_template; - } - ';' if !in_single_quote && !in_double_quote && !in_template => { - return true; - } - _ => {} - } - prev_char = ch; - } - - false -} - -fn try_exact_property_access( - expr: &str, - flow_input: Option<&mappable_rc::Marc>>>, - flow_env: Option<&HashMap>>, -) -> Option> { - let obj = if expr.starts_with(FLOW_INPUT_PREFIX) { - Some(( - FLOW_INPUT_PREFIX, - flow_input.as_ref().map(|obj| obj.as_ref()), - )) - } else if expr.starts_with(ENV_KEY_PREFIX) { - Some((ENV_KEY_PREFIX, flow_env)) - } else { - None - }; - - if let Some((prefix, obj)) = obj { - let access_pattern_pos = prefix.len(); - let suffix = &expr[access_pattern_pos..]; - let maybe_key_name = if suffix.starts_with(DOT_PATTERN) { - let key_name_pos = DOT_PATTERN.len(); - Some(&suffix[key_name_pos..]) - } else if suffix.starts_with(START_BRACKET_PATTERN) { - let key_name_pos = START_BRACKET_PATTERN.len(); - let suffix = &suffix[key_name_pos..]; - - let flow_arg_name = suffix - .ends_with(END_BRACKET_PATTERN) - .then(|| { - let start_key_name_pos = access_pattern_pos + key_name_pos; - let end_key_name_pos = expr.len() - END_BRACKET_PATTERN.len(); - &expr[start_key_name_pos..end_key_name_pos] - }) - .filter(|s| s.len() > 0); - flow_arg_name - } else { - None - }; - - if let Some(key_name) = maybe_key_name { - if let Some(key_value) = obj.and_then(|obj| obj.get(key_name)) { - return Some(key_value.clone()); - } - } - } - None -} - -async fn handle_full_regex( - expr: &str, - authed_client: &AuthedClient, - by_id: &IdContext, -) -> Option>> { - if let Some(captures) = RE_FULL.captures(&expr) { - let obj_name = captures.get(1).unwrap().as_str(); - let obj_key = captures.get(2).unwrap().as_str(); - let idx_o = captures.get(3).map(|y| y.as_str()); - let rest = captures.get(4).map(|y| y.as_str()); - let query = if let Some(idx) = idx_o { - match rest { - Some(rest) => Some(format!("{}{}", idx, rest)), - None => Some(idx.to_string()), - } - } else { - rest.map(|x| x.trim_start_matches('.').to_string()) - }; - - let result = if obj_name == "results" { - // Use .ok() to match deno_core op_get_id behavior: return null for non-existent steps - // instead of throwing an error - let res = authed_client - .get_result_by_id::>>(&by_id.flow_job.to_string(), obj_key, query) - .await - .ok() - .flatten(); - match res { - Some(v) => Ok(v), - None => serde_json::value::to_raw_value(&serde_json::Value::Null) - .map_err(|e| anyhow::anyhow!("Failed to serialize null: {}", e)), - } - } else if obj_name == "flow_env" { - authed_client - .get_flow_env_by_flow_job_id(&by_id.flow_job.to_string(), obj_key, query) - .await - } else { - unreachable!(); - }; - - return Some(result); - } - - return None; -} +// Re-export IdContext from windmill-jseval for backward compatibility +pub use windmill_jseval::IdContext; pub async fn eval_timeout( expr: String, @@ -359,7 +31,7 @@ pub async fn eval_timeout( flow_env: Option<&HashMap>>, authed_client: Option<&AuthedClient>, by_id: Option<&IdContext>, - #[allow(unused_variables)] ctx: Option>, + ctx: Option>, ) -> anyhow::Result> { let expr = expr.trim().to_string(); @@ -373,7 +45,9 @@ pub async fn eval_timeout( return Ok(value.as_ref().to_owned()); } - if let Some(value) = try_exact_property_access(&expr, flow_input.as_ref(), flow_env) { + if let Some(value) = + windmill_jseval::try_exact_property_access(&expr, flow_input.as_ref(), flow_env) + { return Ok(value); } @@ -390,7 +64,6 @@ pub async fn eval_timeout( && transform_context.contains_key("previous_result") && p_ids.as_ref().unwrap().iter().any(|x| x == &expr) { - // tracing::error!("PREVIOUS_RESULT"); return Ok(transform_context .get("previous_result") .unwrap() @@ -399,571 +72,27 @@ pub async fn eval_timeout( } if let (Some(by_id), Some(authed_client)) = (by_id, authed_client) { - if let Some(result) = handle_full_regex(&expr, authed_client, by_id).await { + if let Some(result) = windmill_jseval::handle_full_regex(&expr, authed_client, by_id).await + { return result; } } - // Use QuickJS if enabled and either deno_core is not available or USE_QUICKJS env var is set - #[cfg(all(feature = "quickjs", not(feature = "deno_core")))] - { - return crate::js_eval_quickjs::eval_timeout_quickjs( - expr, - transform_context, - flow_input, - flow_env, - authed_client, - by_id, - ctx, - ) - .await; - } - - #[cfg(all(feature = "quickjs", feature = "deno_core"))] - if *USE_QUICKJS { - return crate::js_eval_quickjs::eval_timeout_quickjs( - expr, - transform_context, - flow_input, - flow_env, - authed_client, - by_id, - ctx, - ) - .await; - } - - #[cfg(not(feature = "deno_core"))] - { - #[allow(unreachable_code)] - return Err(anyhow::anyhow!("Deno core is not enabled".to_string()).into()); - } - - #[cfg(feature = "deno_core")] - { - let expr2 = expr.clone(); - let by_id = by_id.cloned(); - let (sender, mut receiver) = oneshot::channel::(); - let has_client = authed_client.is_some(); - let authed_client = authed_client.cloned(); - return timeout( - std::time::Duration::from_millis(10000), - tokio::task::spawn_blocking(move || { - let mut ops = vec![op_get_context()]; - - if authed_client.is_some() { - ops.extend([ - // An op for summing an array of numbers - // The op-layer automatically deserializes inputs - // and serializes the returned Result & value - op_variable(), - op_resource(), - ]) - } - - if by_id.is_some() && authed_client.is_some() { - ops.push(op_get_result()); - ops.push(op_get_id()); - ops.push(op_get_flow_env()); - } - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; - let exts = vec![ext]; - // Use our snapshot to provision our new runtime - let options = RuntimeOptions { - extensions: exts, - // startup_snapshot: Some(Snapshot::Static(buffer)), - ..Default::default() - }; - - let mut context_keys = transform_context - .keys() - .filter(|x| expr.contains(&x.to_string())) - .map(|x| x.clone()) - .collect_vec(); - - if !context_keys.contains(&"previous_result".to_string()) - && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) - || expr.contains("error") - { - // tracing::error!("PREVIOUS_RESULT"); - context_keys.push("previous_result".to_string()); - } - let has_flow_input = expr.contains("flow_input"); - if has_flow_input { - context_keys.push("flow_input".to_string()) - } - - let mut js_runtime = JsRuntime::new(options); - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - let mut client = authed_client.clone(); - if let Some(client) = client.as_mut() { - client.force_client = Some( - configure_client( - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .danger_accept_invalid_certs( - std::env::var("ACCEPT_INVALID_CERTS").is_ok(), - ), - ) - .build() - .unwrap(), - ); - } - op_state.put(OptAuthedClient(client)); - op_state.put(TransformContext { - flow_input: if has_flow_input { flow_input } else { None }, - envs: transform_context - .into_iter() - .filter(|(a, _)| context_keys.contains(a)) - .collect(), - }); - } - - sender - .send(js_runtime.v8_isolate().thread_safe_handle()) - .map_err(|_| { - Error::ExecutionErr("impossible to send v8 isolate".to_string()) - })?; - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - // pretty frail but this it to make the expr more user friendly and not require the user to write await - let expr = ["variable", "resource"] - .into_iter() - .fold(expr, replace_with_await); - - let expr = replace_with_await_result(expr); - - let r = runtime.block_on(eval( - &mut js_runtime, - &expr, - context_keys, - by_id, - has_client, - ctx, - ))?; - - Ok(r) as anyhow::Result> - }), - ) - .await - .map_err(|_| { - if let Ok(isolate) = receiver.try_recv() { - isolate.terminate_execution(); - }; - Error::ExecutionErr(format!( - "The expression of evaluation `{expr2}` took too long to execute (>10000ms)" - )) - })??; - } -} - -#[cfg(any(feature = "deno_core", feature = "quickjs"))] -pub fn replace_with_await(expr: String, fn_name: &str) -> String { - let sep = format!("{}(", fn_name); - let mut split = expr.split(&sep); - let mut s = split.next().unwrap_or_else(|| "").to_string(); - for x in split { - s.push_str(&format!("(await {}({}", fn_name, add_closing_bracket(x))) - } - s -} -lazy_static! { - static ref RE: Regex = Regex::new( - r#"(?m)(?P(?:results|flow_env)(?:\?)?(?:(?:\.[a-zA-Z_0-9]+)|(?:\[\".*?\"\])))"# + windmill_jseval::eval_timeout_quickjs( + expr, + transform_context, + flow_input, + flow_env, + authed_client, + by_id, + ctx, ) - .unwrap(); - static ref RE_FULL: Regex = Regex::new( - r"(?m)^(results|flow_env)(?:\?)?\.([a-zA-Z_0-9]+)(?:\[(\d+)\])?((?:\.[a-zA-Z_0-9]+)+)?$" - ) - .unwrap(); - static ref RE_PROXY: Regex = - Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); -} - -#[cfg(feature = "quickjs")] -#[allow(dead_code)] // Only used when both quickjs and deno_core features are enabled -static USE_QUICKJS: Lazy = Lazy::new(|| { - // When enterprise is not enabled, default to QuickJS unless USE_DENO_FOR_FLOW_EVAL is set - // When enterprise is enabled, default to Deno unless USE_QUICKJS_FOR_FLOW_EVAL is set - #[cfg(not(feature = "enterprise"))] - { - std::env::var("USE_DENO_FOR_FLOW_EVAL").is_err() - } - #[cfg(feature = "enterprise")] - { - std::env::var("USE_QUICKJS_FOR_FLOW_EVAL").is_ok() - } -}); - -#[cfg(any(feature = "deno_core", feature = "quickjs"))] -pub fn replace_with_await_result(expr: String) -> String { - RE.replace_all(&expr, "(await $r)").to_string() -} - -#[cfg(any(feature = "deno_core", feature = "quickjs"))] -fn add_closing_bracket(s: &str) -> String { - let mut s = s.to_string(); - let mut level = 1; - let mut idx = 0; - for c in s.chars() { - match c { - '(' => level += 1, - ')' => level -= 1, - _ => (), - }; - if level == 0 { - break; - } - idx += 1; - } - s.insert_str(idx, ")"); - s -} - -#[cfg(feature = "deno_core")] -async fn eval( - context: &mut JsRuntime, - expr: &str, - transform_context: Vec, - by_id: Option, - has_client: bool, - ctx: Option>, -) -> anyhow::Result> { - tracing::debug!("evaluating: {} {:#?}", expr, by_id); - - let (api_code, by_id_code) = if has_client { - let by_id_code = if let Some(by_id) = by_id { - format!( - r#" -async function result_by_id(node_id) {{ - let id_map = {{ {} }}; - let id = id_map[node_id]; - if (node_id == "{}") {{ - return previous_result; - }} else if (id) {{ - if (Array.isArray(id)) {{ - return await Promise.all(id.map(async (id) => await get_result(id))); - }} else {{ - return await get_result(id); - }} - }} else {{ - let flow_job_id = "{}"; - return JSON.parse(await Deno.core.ops.op_get_id(flow_job_id, node_id)); - }} -}} - -async function get_result(id) {{ - return JSON.parse(await Deno.core.ops.op_get_result(id)); -}} -const results = new Proxy({{}}, {{ - get: function(target, name, receiver) {{ - return result_by_id(name); - }} -}}); - -async function flow_env_by_var_name(var_name) {{ - let root_job_id = "{}"; - return JSON.parse(await Deno.core.ops.op_get_flow_env(root_job_id, var_name, null)); -}} - -const flow_env = new Proxy({{}}, {{ - get: function(target, name, receiver) {{ - return flow_env_by_var_name(name); - }} -}}); - -"#, - by_id - .steps_results - .into_iter() - .map(|(k, v)| { - let v_str = match v { - JobResult::SingleJob(x) => format!("\"{x}\""), - JobResult::ListJob(x) => { - format!("[{}]", x.iter().map(|x| format!("\"{x}\"")).join(",")) - } - }; - format!("\"{k}\": {v_str}") - }) - .join(","), - by_id.previous_id, - by_id.flow_job, - by_id.flow_job - ) - } else { - String::new() - }; - - let api_code = format!( - r#" -async function variable(path) {{ - return await Deno.core.ops.op_variable(path); -}} -async function resource(path) {{ - return JSON.parse(await Deno.core.ops.op_resource(path)); -}} - "#, - ); - (api_code, by_id_code) - } else { - (String::new(), String::new()) - }; - - let f = if should_add_return(expr) { - format!("return {expr}") - } else { - expr.to_string() - }; - - let ctx_str = ctx - .map(|x| { - x.into_iter() - .map(|(k, v)| format!("let {} = \"{}\";", k, v)) - .join("\n") - }) - .unwrap_or_default(); - let code = format!( - r#" -function get_from_env(name) {{ - return JSON.parse(Deno.core.ops.op_get_context(name)); -}} -{ctx_str} - -{api_code} -{} -{} -{by_id_code} -((async () => {{ - {f}; -}})()).then((x) => JSON.stringify(x ?? null)) - "#, - transform_context - .iter() - .map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) }) - .join(""), - if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) { - r#"let error = previous_result?.error; -if (!error) { - if (Array.isArray(previous_result)) { - const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item); - if (errors.length === 1) { - error = errors[0].error; - } else if (errors.length > 1) { - error = { - name: 'MultipleErrors', - message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '), - errors: previous_result - }; - } else { - error = { - name: 'MultipleErrors', - message: "Could not parse errors", - errors: previous_result - }; - } - } else { - if (previous_result) { - error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result }; - } else { - error = { name: 'UnknownError', message: 'No error found' }; - } - } -}"# - } else { - "" - }, - ); - - let script = context.execute_script("", code)?; - let fut = context.resolve(script); - let global = context - .with_event_loop_promise(fut, PollEventLoopOptions::default()) - .await?; - - let scope = &mut context.handle_scope(); - let local = v8::Local::new(scope, global); - // Deserialize a `v8` object into a Rust type using `serde_v8`, - // in this case deserialize to a JSON `Value`. - let r = serde_v8::from_v8::(scope, local)?; - Ok(unsafe_raw(r)) -} - -// #[warn(dead_code)] -// async fn op_test( -// _state: Rc>, -// path: String, -// _buf: Option, -// ) -> Result { -// tokio::time::sleep(std::time::Duration::from_secs(1)).await; -// Ok(path) -// } - -// TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client? -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_variable( - op_state: Rc>, - #[string] path: String, -) -> Result { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - Ok(client - .get_variable_value(&path) - .await - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string()))?) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_get_result( - op_state: Rc>, - #[string] id: String, -) -> Result { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - client - .get_completed_job_result::>(&id, None) - .await - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) - .map(|x| x.get().to_string()) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_get_id( - op_state: Rc>, - #[string] flow_job_id: String, - #[string] node_id: String, -) -> Result, deno_error::JsErrorBox> { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - let result = client - .get_result_by_id::>>(&flow_job_id, &node_id, None) - .await - .ok(); - if let Some(result) = result { - Ok(result.map(|x| x.get().to_string())) - } else { - Ok(None) - } - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_resource( - op_state: Rc>, - #[string] path: String, -) -> Result, deno_error::JsErrorBox> { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - client - .get_resource_value_interpolated::>>(&path, None) - .await - .map(|x| x.map(|x| x.get().to_string())) - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -#[op2(async)] -#[string] -async fn op_get_flow_env( - op_state: Rc>, - #[string] root_job_id: String, - #[string] var_name: String, - #[string] json_path: Option, -) -> Result, deno_error::JsErrorBox> { - let client = op_state.borrow().borrow::().0.clone(); - if let Some(client) = client { - client - .get_flow_env_by_flow_job_id::>>( - &root_job_id, - &var_name, - json_path, - ) - .await - .map(|value| value.map(|val| val.get().to_string())) - .map_err(|e| deno_error::JsErrorBox::generic(e.to_string())) - } else { - Err(deno_error::JsErrorBox::generic( - "No client found in op state", - )) - } -} - -#[cfg(feature = "deno_core")] -pub struct TransformContext { - pub envs: HashMap>>, - pub flow_input: Option>>>, -} - -#[cfg(feature = "deno_core")] -#[op2] -#[string] -fn op_get_context(op_state: Rc>, #[string] id: &str) -> String { - let ops = op_state.borrow(); - let client = ops.borrow::(); - if id == "flow_input" { - client - .flow_input - .as_ref() - .and_then(|x| serde_json::to_string(x.as_ref()).ok()) - .unwrap_or_else(|| "null".to_string()) - } else { - client - .envs - .get(id) - .and_then(|x| serde_json::to_string(x).ok()) - .unwrap_or_else(String::new) - } + .await } #[cfg(feature = "deno_core")] pub fn transpile_ts(expr: String) -> anyhow::Result { - let parsed = deno_ast::parse_module(ParseParams { - specifier: url::Url::parse("file:///eval.ts")?, - capture_tokens: false, - scope_analysis: false, - media_type: deno_ast::MediaType::TypeScript, - maybe_syntax: None, - text: deno_core::ModuleCodeString::from(expr).into(), - })?; - Ok(parsed - .transpile( - &Default::default(), - &Default::default(), - &Default::default(), - )? - .into_source() - .text) + windmill_runtime_nativets::transpile_ts(expr) } #[cfg(not(feature = "deno_core"))] @@ -971,68 +100,6 @@ pub fn transpile_ts(_expr: String) -> anyhow::Result { Ok("require deno".to_string()) } -#[cfg(feature = "deno_core")] -static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); - -#[cfg(feature = "deno_core")] -pub struct MainArgs { - args: Vec>>, -} - -#[cfg(feature = "deno_core")] -pub struct LogString { - pub s: mpsc::UnboundedSender, -} - -#[cfg(feature = "deno_core")] -pub struct NativeAnnotation { - pub useragent: Option, - pub proxy: Option<(String, Option<(String, String)>)>, -} -#[cfg(feature = "deno_core")] -pub fn get_annotation(inner_content: &str) -> NativeAnnotation { - let mut res = NativeAnnotation { useragent: None, proxy: None }; - - let anns = inner_content - .lines() - .take_while(|x| x.starts_with("//")) - .map(|x| x.to_string().trim_start_matches("//").trim().to_string()) - .collect_vec(); - - for ann in anns.iter() { - if ann.starts_with("useragent") { - res.useragent = Some(ann.trim_start_matches("useragent").trim().to_string()); - } else if ann.starts_with("proxy") { - res.proxy = capture_proxy(ann.trim_start_matches("proxy").trim()); - } - } - res -} - -#[cfg(feature = "deno_core")] -fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { - RE_PROXY.captures(s).map(|x| { - ( - format!( - "{}://{}{}", - x.get(1).map(|x| x.as_str()).unwrap_or_default(), - x.get(5).map(|x| x.as_str()).unwrap_or_default(), - x.get(7) - .map(|x| format!(":{}", x.as_str())) - .unwrap_or_default(), - ), - x.get(3).map(|y| { - ( - y.as_str().to_string(), - x.get(4).map(|x| x.as_str().to_string()).unwrap_or_default(), - ) - }), - ) - }) -} - -use windmill_common::worker::Connection; - #[cfg(not(feature = "deno_core"))] pub async fn eval_fetch_timeout( _env_code: String, @@ -1075,220 +142,37 @@ pub async fn eval_fetch_timeout( stream_notifier: Option, has_stream: &mut bool, ) -> windmill_common::error::Result> { - let (sender, mut receiver) = oneshot::channel::(); - let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); - let (result_stream_sender, mut result_stream_receiver) = mpsc::unbounded_channel::(); - - let conn_ = conn.clone(); - let w_id_ = w_id.to_string(); - tokio::spawn(async move { - while let Some(log) = append_logs_receiver.recv().await { - windmill_queue::append_logs(&job_id, &w_id_, log, &conn_).await - } - }); - - let conn_ = conn.clone(); - let w_id_ = w_id.to_string(); - tokio::spawn(async move { - let mut offset = -1; - while let Some(stream) = result_stream_receiver.recv().await { - use crate::job_logger::append_result_stream; - offset += 1; - if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream, offset).await { - tracing::error!("failed to append result stream: {e}"); - } - } - }); - let parsed_args = windmill_parser_ts::parse_deno_signature( - &ts_expr, - true, - false, - script_entrypoint_override.clone(), - )? - .args; - let spread = parsed_args - .into_iter() - .map(|x| { - args.as_ref() - .and_then(|args| args.0.get(&x.name).map(|x| x.clone())) - }) - .collect::>(); - - let ann = get_annotation(&ts_expr); - - #[cfg(not(feature = "enterprise"))] - if ann.proxy.is_some() { - return Err(Error::ExecutionErr("Proxy is an EE feature".to_string()).into()); - } - - let mut extra_logs = String::new(); - if ann.useragent.is_some() { - extra_logs.push_str(&format!("useragent: {}\n", ann.useragent.as_ref().unwrap())); - } - if ann.proxy.is_some() { - let (proxy, auth) = ann.proxy.as_ref().unwrap(); - extra_logs.push_str(&format!( - "proxy: {proxy} (basic auth: {})\n", - auth.is_some() - )); - } - - let result_f = tokio::task::spawn_blocking(move || { - let ops = vec![op_get_static_args(), op_log()]; - let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; - - let fetch_options = deno_fetch::Options { - root_cert_store_provider: None, - user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), - proxy: ann.proxy.map(|x| deno_tls::Proxy { - url: x.0, - basic_auth: x - .1 - .map(|(username, password)| deno_tls::BasicAuth { username, password }), - }), - ..Default::default() - }; - - let exts: Vec = vec![ - deno_telemetry::deno_telemetry::init_ops(), - deno_webidl::deno_webidl::init_ops(), - deno_url::deno_url::init_ops(), - deno_console::deno_console::init_ops(), - deno_web::deno_web::init_ops::( - Arc::new(BlobStore::default()), - None, - ), - deno_fetch::deno_fetch::init_ops::(fetch_options), - deno_net::deno_net::init_ops::(None, None), - ext, - ]; - - // Use our snapshot to provision our new runtime - let options = RuntimeOptions { - is_main: true, - extensions: exts, - create_params: Some( - deno_core::v8::CreateParams::default() - .heap_limits(0 as usize, 1024 * 1024 * 128 as usize), - ), - // startup_snapshot: None, - startup_snapshot: Some(RUNTIME_SNAPSHOT), - module_loader: Some(Rc::new(deno_core::FsModuleLoader)), - extension_transpiler: None, - ..Default::default() - }; - - let (memory_limit_tx, mut memory_limit_rx) = mpsc::unbounded_channel::<()>(); - - // tracing::info!("starting isolate"); - // let instant = Instant::now(); - - let mut js_runtime: JsRuntime = JsRuntime::new(options); - // tracing::info!("ttc: {:?}", instant.elapsed()); - - // Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized. - // We call the function exposed by runtime.js since we can't dynamically import ext: modules. + let otel_initialized = { #[cfg(all(feature = "private", feature = "enterprise"))] - if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { - if let Err(e) = - js_runtime.execute_script("", "globalThis.__bootstrapOtel()") - { - tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e); - } + { + crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + { + false + } + }; - js_runtime.add_near_heap_limit_callback(move |x,y| { - tracing::error!("heap limit reached: {x} {y}"); - - if memory_limit_tx.send(()).is_err() { - tracing::error!("failed to send memory limit reached notification - isolate may already be terminating"); - }; - //to give a bit of time to kill the worker without v8 crashing - return y*2; + let stream_notifier_update: Option> = stream_notifier + .map(|sn| { + Arc::new(move || { + sn.update_flow_status_with_stream_job(); + }) as Arc }); - let (log_sender, mut log_receiver) = mpsc::unbounded_channel::(); - - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(PermissionsContainer {}); - //reqwest client seems to not be sharable between runtimes unfortunately - // op_state.put(HTTP_CLIENT.clone()); - op_state.put(MainArgs { args: spread }); - op_state.put(LogString { s: log_sender }); - } - - sender - .send(js_runtime.v8_isolate().thread_safe_handle()) - .map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?; - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - let future = async { - use crate::common::merge_result_stream; - - if !extra_logs.is_empty() { - if let Err(e) = append_logs_sender.send(extra_logs) { - tracing::error!("failed to send extra logs: {e}"); - } - } - let handle = tokio::spawn(async move { - let mut result_stream = String::new(); - let mut is_stream = false; - while let Some(log) = log_receiver.recv().await { - use windmill_common::result_stream::extract_stream_from_logs; - - if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { - if let Some(sn) = stream_notifier.as_ref() { - if !is_stream { - is_stream = true; - sn.update_flow_status_with_stream_job(); - } - } - - result_stream.push_str(&stream); - if let Err(e) = result_stream_sender.send(stream) { - tracing::error!("failed to send result stream: {e}"); - } - } else { - if let Err(e) = append_logs_sender.send(log) { - tracing::error!("failed to send log: {e}"); - } - } - } - if !result_stream.is_empty() { - Some(result_stream) - } else { - None - } - }); - - let r = tokio::select! { - r = eval_fetch(&mut js_runtime, &js_expr, Some(env_code), script_entrypoint_override, load_client, &job_id) => Ok(r), - _ = memory_limit_rx.recv() => Err(Error::ExecutionErr("Memory limit reached, killing isolate".to_string())) - }; - drop(js_runtime); - if let Ok(r) = r { - match handle.await { - Ok(Some(logs)) => { - Ok(merge_result_stream(r, Some(logs)).await.map(|r| (r, true))) - } - Ok(None) => Ok(r.map(|r| (r, false))), - Err(e) => Err(Error::ExecutionErr(e.to_string())), - } - } else { - r.map(|r| r.map(|r| (r, false))) - } - // r - }; - let r = runtime.block_on(future)?; - // tracing::info!("total: {:?}", instant.elapsed()); - - r as windmill_common::error::Result<(Box, bool)> - }); + let result_f = windmill_runtime_nativets::eval_fetch_timeout( + env_code, + ts_expr, + js_expr, + args, + script_entrypoint_override, + job_id, + conn, + w_id, + load_client, + otel_initialized, + stream_notifier_update, + ); let (res, new_has_stream) = run_future_with_polling_update_job_poller( job_id, @@ -1296,475 +180,14 @@ pub async fn eval_fetch_timeout( conn, mem_peak, canceled_by, - async { result_f.await.map_err(windmill_common::error::to_anyhow)? }, + result_f, worker_name, w_id, &mut Some(occupation_metrics), Box::pin(futures::stream::once(async { 0 })), ) - .await - .map_err(|e| { - if let Ok(isolate) = receiver.try_recv() { - isolate.terminate_execution(); - } - e - })?; + .await?; *has_stream = new_has_stream; *mem_peak = (res.get().len() / 1000) as i32; Ok(res) } - -#[cfg(feature = "deno_core")] -const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); - -#[cfg(feature = "deno_core")] -const ERROR_DIR: &str = const_format::concatcp!(TMP_DIR, "/native_errors"); - -#[cfg(feature = "deno_core")] -fn write_error_expr(expr: &str, uuid: &Uuid) { - if let Err(e) = std::fs::create_dir_all(ERROR_DIR) { - tracing::error!("failed to create error dir {ERROR_DIR}: {e}"); - return; - } - let dir_entries = match std::fs::read_dir(ERROR_DIR) { - Ok(entries) => entries.count(), - Err(_) => { - tracing::error!("failed to read error dir {ERROR_DIR}"); - return; - } - }; - - if std::env::var("PRINT_NATIVE_ERRORS").is_ok() { - tracing::info!("native error for job {uuid}: {expr}"); - } - if dir_entries >= 100 { - tracing::info!("Too many error files in {ERROR_DIR}, skipping write"); - return; - } - - let path = format!("/{uuid}.js"); - tracing::info!( - "nativets job {uuid} failed, writing error expr to {ERROR_DIR}/{path} for debugging: {path}" - ); - if let Err(e) = write_file(ERROR_DIR, &path, expr) { - tracing::error!("failed to write error expr to file {path}: {e}"); - } -} - -#[cfg(feature = "deno_core")] -async fn eval_fetch( - js_runtime: &mut JsRuntime, - expr: &str, - env_code: Option, - script_entrypoint_override: Option, - load_client: bool, - job_id: &Uuid, -) -> windmill_common::error::Result> { - if load_client { - if let Some(env_code) = env_code.as_ref() { - let _ = js_runtime - .load_side_es_module_from_code( - &deno_core::resolve_url("file:///windmill.ts").map_err(error::to_anyhow)?, - format!("{env_code}\n{}", WINDMILL_CLIENT.to_string()), - ) - .await - .map_err(error::to_anyhow)?; - } - } - use anyhow::Context; - use deno_core::error::CoreError; - use windmill_common::{error, worker::to_raw_value}; - let source = format!("{}\n{expr}", env_code.unwrap_or_default()); - let _ = js_runtime - .load_side_es_module_from_code( - &deno_core::resolve_url("file:///eval.ts").map_err(error::to_anyhow)?, - source.to_string(), - ) - .await - .map_err(|e| { - write_error_expr(expr, &job_id); - e - }) - .context("failed to load module")?; - - let main_override = script_entrypoint_override.unwrap_or("main".to_string()); - - // Inject parent trace context using enterSpan with a duck-typed span object. - // Uses job_id as trace_id so all spans are linked to the job. - // span_id is a placeholder - it gets overwritten by the OTLP handler with the real parent span_id. - #[cfg(all(feature = "private", feature = "enterprise"))] - let otel_context_inject = if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { - let trace_id = job_id.as_simple().to_string(); - format!( -r#"globalThis.__enterSpan?.({{ - isRecording: () => true, - spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }}) -}});"# - ) - } else { - String::new() - }; - - #[cfg(not(all(feature = "private", feature = "enterprise")))] - let otel_context_inject = ""; - - let script = js_runtime - .execute_script( - "", - format!( - r#" -function isAsyncIterable(obj) {{ - // return true; // TODO: remove this - return obj != null && typeof obj[Symbol.asyncIterator] === 'function'; -}} - -function processStreamIterative(res) {{ - const iterator = res[Symbol.asyncIterator](); - - function processLoop() {{ - return new Promise(function(resolve) {{ - function step() {{ - iterator.next().then(function(result) {{ - if (!result.done) {{ - const chunk = result.value; - console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); - // Continue the loop - step(); - }} else {{ - resolve("null"); - }} - }}).catch(function(error) {{ - resolve("null"); - }}); - }} - step(); - }}); - }} - - return processLoop(); -}} - -{otel_context_inject} - -let args = Deno.core.ops.op_get_static_args().map(JSON.parse) -import("file:///eval.ts").then((module) => module.{main_override}(...args)) - .then(res => {{ - if (isAsyncIterable(res)) {{ - return processStreamIterative(res) - }} else {{ - return JSON.stringify(res ?? null); - }} - }}) -"# - ), - ) - .map_err(|e| { - write_error_expr(expr, &job_id); - e - }) - .context("native script initialization")?; - - let fut = js_runtime.resolve(script); - let global = js_runtime - .with_event_loop_promise(fut, PollEventLoopOptions::default()) - .await - .map_err(|e| { - write_error_expr(expr, &job_id); - e - }); - - match global { - Ok(global) => { - let scope = &mut js_runtime.handle_scope(); - let local = v8::Local::new(scope, global); - // Deserialize a `v8` object into a Rust type using `serde_v8`, - // in this case deserialize to a JSON `Value`. - let r = serde_v8::from_v8::>(scope, local).map_err(error::to_anyhow)?; - Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) - } - Err(CoreError::Js(e)) => { - let stack_head = e.frames.first().and_then(|f| { - if f.file_name.as_ref().is_some_and(|x| x == "file:///eval.ts") { - Some(format!( - "{}\n", - source - .lines() - .nth((f.line_number.unwrap_or(1)) as usize - 1) - .unwrap_or("") - .to_string() - )) - } else { - None - } - }); - let stack_s = format!( - "{}{}", - stack_head.unwrap_or("".to_string()), - e.stack.unwrap_or("".to_string()) - ); - let stack = if stack_s.is_empty() { - None - } else { - Some(stack_s) - }; - Err(Error::ExecutionRawError(to_raw_value(&serde_json::json!({ - "message": e.message, - "stack": stack, - "name": e.name, - })))) - } - Err(e) => Err(Error::ExecutionErr(e.print_with_cause())), - } -} - -#[cfg(feature = "deno_core")] -#[op2] -#[serde] -fn op_get_static_args(op_state: Rc>) -> Vec> { - op_state - .borrow() - .borrow::() - .args - .iter() - .map(|x| x.as_ref().map(|y| y.get().to_string())) - .collect_vec() -} - -#[cfg(feature = "deno_core")] -#[op2(fast)] -fn op_log(op_state: Rc>, #[string] log: &str) { - // tracing::error!("log: |{}|", log); - if let Err(e) = op_state - .borrow_mut() - .borrow_mut::() - .s - .send(log.to_string()) - { - tracing::error!("failed to send log: {e}"); - } -} - -#[cfg(feature = "deno_core")] -#[cfg(test)] -mod tests { - - use serde_json::json; - use windmill_common::worker::to_raw_value; - - // Note this useful idiom: importing names from outer (for mod tests) scope. - use super::*; - - #[tokio::test] - async fn test_eval() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "params".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - env.insert( - "value".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - - let code = "value.test + params.test"; - - let ops = vec![op_get_context()]; - - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; - let exts = vec![ext]; - - let options = RuntimeOptions { extensions: exts, ..Default::default() }; - - let mut runtime = JsRuntime::new(options); - { - let op_state = runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(TransformContext { flow_input: None, envs: env.clone() }) - } - - let res = eval( - &mut runtime, - code, - vec!["params".to_string(), "value".to_string()], - None, - false, - None, - ) - .await?; - assert_eq!(res.get(), "4"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_multiline() -> anyhow::Result<()> { - let env = vec![]; - let code = "let x = 5; -return `my ${x} -multiline template`"; - - let mut runtime = JsRuntime::new(RuntimeOptions::default()); - let res = eval(&mut runtime, code, env, None, false, None).await?; - assert_eq!(res.get(), "\"my 5\\nmultiline template\""); - Ok(()) - } - - #[tokio::test] - async fn test_eval_timeout() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "params".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - env.insert( - "value".to_string(), - Arc::new(to_raw_value(&json!({"test": 2}))), - ); - - let code = r#"params.test"#; - - let mut js_runtime = JsRuntime::new(RuntimeOptions::default()); - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(TransformContext { flow_input: None, envs: env.clone() }) - } - - let res = eval_timeout(code.to_string(), env, None, None, None, None, None).await?; - assert_eq!(res.get(), "2"); - Ok(()) - } - - // #[tokio::test] - // async fn test_eval_timeout_bug() -> anyhow::Result<()> { - // let ops = vec![op_get_static_args(), op_log()]; - // let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; - - // let deno_fetch_options = if let Some(cert_path) = env::var("DENO_CERT").ok() { - // let mut cert_store_provider = ContainerRootCertStoreProvider::new(); - // cert_store_provider.add_certificate(cert_path)?; - - // deno_fetch::Options { - // root_cert_store_provider: Some(Arc::new(cert_store_provider)), - // ..Default::default() - // } - // } else { - // Default::default() - // }; - - // let exts: Vec = vec![ - // deno_webidl::deno_webidl::init_ops(), - // deno_url::deno_url::init_ops(), - // deno_console::deno_console::init_ops(), - // deno_web::deno_web::init_ops::( - // Arc::new(BlobStore::default()), - // None, - // ), - // deno_fetch::deno_fetch::init_ops::(deno_fetch_options), - // deno_net::deno_net::init_ops::(None, None), - // ext, - // ]; - - // // Use our snapshot to provision our new runtime - // let options = RuntimeOptions { - // is_main: true, - // extensions: exts, - // create_params: Some( - // deno_core::v8::CreateParams::default() - // .heap_limits(0 as usize, 1024 * 1024 * 128 as usize), - // ), - // // startup_snapshot: None, - // startup_snapshot: Some(RUNTIME_SNAPSHOT), - // module_loader: Some(Rc::new(deno_core::FsModuleLoader)), - // extension_transpiler: None, - // ..Default::default() - // }; - - // let mut js_runtime: JsRuntime = JsRuntime::new(options); - // Ok(()) - // } - - // #[tokio::test] - // async fn test_eval_fetch_timeout() -> anyhow::Result<()> { - // let code = r#"export async function main() { return "" }"#; - - // let res = eval_fetch_timeout(code.to_string(), code.to_string(), None, Uuid::new_v4(), None, ).await?; - // assert_eq!(res.0.get(), "\"\""); - // Ok(()) - // } - - #[test] - fn test_should_add_return() { - // Simple expressions should get return added - assert_eq!(should_add_return("5"), true); - assert_eq!(should_add_return("x + y"), true); - assert_eq!(should_add_return("foo()"), true); - assert_eq!(should_add_return("obj.property"), true); - - // Object literals should get return added - assert_eq!(should_add_return("{ foo: 'bar' }"), true); - assert_eq!(should_add_return("{ a: 1, b: 2 }"), true); - assert_eq!(should_add_return("{}"), true); - - // Already has return - assert_eq!(should_add_return("return 5"), false); - assert_eq!(should_add_return("return x + y"), false); - assert_eq!(should_add_return("return;"), false); - assert_eq!(should_add_return("return"), false); - - // Should NOT add return for statements - assert_eq!(should_add_return("const x = 5"), false); - assert_eq!(should_add_return("let y = 10"), false); - assert_eq!(should_add_return("var z = 15"), false); - assert_eq!(should_add_return("if (x > 5) { return x; }"), false); - assert_eq!(should_add_return("for (let i = 0; i < 10; i++) {}"), false); - assert_eq!(should_add_return("while (true) {}"), false); - assert_eq!(should_add_return("function foo() {}"), false); - assert_eq!(should_add_return("throw new Error('test')"), false); - - // Multiple statements with semicolons (including block statements) - assert_eq!(should_add_return("let x = 5; x + 1"), false); - assert_eq!(should_add_return("{ const x = 5; return x; }"), false); - - // Edge case: "return" in a string should still get return prepended - assert_eq!(should_add_return("\"return this string\""), true); - assert_eq!(should_add_return("'return in single quotes'"), true); - assert_eq!(should_add_return("`return in template literal`"), true); - - // Semicolons in strings should not trigger multi-statement detection - assert_eq!(should_add_return("\"hello; world\""), true); - assert_eq!(should_add_return("'test; string'"), true); - assert_eq!(should_add_return("`template; literal`"), true); - } - - #[test] - fn test_contains_semicolon_outside_strings() { - // Semicolons outside strings - assert_eq!(contains_semicolon_outside_strings("let x = 5; x + 1"), true); - assert_eq!(contains_semicolon_outside_strings("x; y"), true); - - // Semicolons inside strings (should NOT be detected) - assert_eq!( - contains_semicolon_outside_strings("\"hello; world\""), - false - ); - assert_eq!(contains_semicolon_outside_strings("'test; string'"), false); - assert_eq!( - contains_semicolon_outside_strings("`template; literal`"), - false - ); - - // Mixed cases - assert_eq!( - contains_semicolon_outside_strings("let x = 'hello; world'; x"), - true - ); - assert_eq!( - contains_semicolon_outside_strings("console.log(\"test; string\")"), - false - ); - - // No semicolons - assert_eq!(contains_semicolon_outside_strings("x + y"), false); - assert_eq!(contains_semicolon_outside_strings("foo()"), false); - } -} diff --git a/backend/windmill-worker/src/js_eval_parity_tests.rs b/backend/windmill-worker/src/js_eval_parity_tests.rs deleted file mode 100644 index 7b403e8ac9..0000000000 --- a/backend/windmill-worker/src/js_eval_parity_tests.rs +++ /dev/null @@ -1,4124 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -//! Feature parity tests for deno_core vs rquickjs expression evaluation. -//! -//! This module ensures both JavaScript engines produce identical results for -//! the same expressions, validating that QuickJS can be used as a drop-in -//! replacement for deno_core in flow expression evaluation. - -#[cfg(all(test, feature = "deno_core", feature = "quickjs"))] -mod parity_tests { - use std::collections::HashMap; - use std::sync::Arc; - - use serde_json::json; - use serde_json::value::RawValue; - use windmill_common::worker::to_raw_value; - - use crate::js_eval::eval_timeout; - use crate::js_eval_quickjs::eval_timeout_quickjs; - - /// Helper to run the same test on both engines and compare results - async fn test_parity( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - ) -> anyhow::Result<()> { - test_parity_with_flow_env(expr, transform_context, flow_input, None).await - } - - /// Helper to run the same test on both engines with flow_env support - async fn test_parity_with_flow_env( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option>>, - ) -> anyhow::Result<()> { - let deno_result = eval_timeout( - expr.to_string(), - transform_context.clone(), - flow_input.clone(), - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - let quickjs_result = eval_timeout_quickjs( - expr.to_string(), - transform_context, - flow_input, - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - // Parse both results to compare as JSON values (handles formatting differences) - let deno_value: serde_json::Value = serde_json::from_str(deno_result.get())?; - let quickjs_value: serde_json::Value = serde_json::from_str(quickjs_result.get())?; - - assert_eq!( - deno_value, quickjs_value, - "Results differ for expression '{}'\ndeno_core: {}\nquickjs: {}", - expr, deno_result.get(), quickjs_result.get() - ); - - Ok(()) - } - - #[tokio::test] - async fn parity_simple_arithmetic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3)))); - - test_parity("x + y", env.clone(), None).await?; - test_parity("x - y", env.clone(), None).await?; - test_parity("x * y", env.clone(), None).await?; - test_parity("x / y", env.clone(), None).await?; - test_parity("x % y", env.clone(), None).await?; - test_parity("x ** 2", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_object_property_access() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "test", - "value": 42, - "nested": { - "deep": { - "property": "found" - } - } - }))), - ); - - test_parity("obj.name", env.clone(), None).await?; - test_parity("obj.value", env.clone(), None).await?; - test_parity("obj.nested.deep.property", env.clone(), None).await?; - test_parity("obj['name']", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - test_parity("arr.length", env.clone(), None).await?; - test_parity("arr[0]", env.clone(), None).await?; - test_parity("arr.map(x => x * 2)", env.clone(), None).await?; - test_parity("arr.filter(x => x > 2)", env.clone(), None).await?; - test_parity("arr.reduce((a, b) => a + b, 0)", env.clone(), None).await?; - test_parity("arr.find(x => x > 3)", env.clone(), None).await?; - test_parity("arr.some(x => x > 4)", env.clone(), None).await?; - test_parity("arr.every(x => x > 0)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_string_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "s".to_string(), - Arc::new(to_raw_value(&json!("Hello World"))), - ); - - test_parity("s.toLowerCase()", env.clone(), None).await?; - test_parity("s.toUpperCase()", env.clone(), None).await?; - test_parity("s.length", env.clone(), None).await?; - test_parity("s.split(' ')", env.clone(), None).await?; - test_parity("s.replace('World', 'QuickJS')", env.clone(), None).await?; - test_parity("s.includes('World')", env.clone(), None).await?; - test_parity("s.startsWith('Hello')", env.clone(), None).await?; - test_parity("s.trim()", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_ternary_and_conditionals() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(10)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(5)))); - - test_parity("x > y ? 'bigger' : 'smaller'", env.clone(), None).await?; - test_parity("x === 10 ? true : false", env.clone(), None).await?; - test_parity("x > 5 && y < 10", env.clone(), None).await?; - test_parity("x > 20 || y < 10", env.clone(), None).await?; - test_parity("!false", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_object_creation() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("name".to_string(), Arc::new(to_raw_value(&json!("test")))); - env.insert("value".to_string(), Arc::new(to_raw_value(&json!(42)))); - - test_parity("({ foo: 'bar' })", env.clone(), None).await?; - test_parity("({ name, value })", env.clone(), None).await?; - test_parity("({ ...{ a: 1 }, b: 2 })", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_null_undefined() -> anyhow::Result<()> { - let env = HashMap::new(); - - test_parity("null", env.clone(), None).await?; - test_parity("undefined", env.clone(), None).await?; - - let mut env_with_null = HashMap::new(); - env_with_null.insert("x".to_string(), Arc::new(to_raw_value(&json!(null)))); - test_parity("x", env_with_null.clone(), None).await?; - test_parity("x ?? 'default'", env_with_null.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input() -> anyhow::Result<()> { - let mut flow_input = HashMap::new(); - flow_input.insert("name".to_string(), to_raw_value(&json!("test_flow"))); - flow_input.insert("count".to_string(), to_raw_value(&json!(100))); - flow_input.insert( - "config".to_string(), - to_raw_value(&json!({"enabled": true})), - ); - - let fi = Some(mappable_rc::Marc::new(flow_input)); - - test_parity("flow_input.name", HashMap::new(), fi.clone()).await?; - test_parity("flow_input.count", HashMap::new(), fi.clone()).await?; - test_parity("flow_input.config.enabled", HashMap::new(), fi.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_template_literals() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("name".to_string(), Arc::new(to_raw_value(&json!("World")))); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - test_parity("`Hello ${name}!`", env.clone(), None).await?; - test_parity("`The answer is ${x * 2}`", env.clone(), None).await?; - test_parity("`Multi\nline`", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_json_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), - ); - - test_parity("JSON.stringify(obj)", env.clone(), None).await?; - test_parity("Object.keys(obj)", env.clone(), None).await?; - test_parity("Object.values(obj)", env.clone(), None).await?; - // Note: Object.entries order might differ, so we skip that - - Ok(()) - } - - #[tokio::test] - async fn parity_math_operations() -> anyhow::Result<()> { - let env = HashMap::new(); - - test_parity("Math.max(1, 5, 3)", env.clone(), None).await?; - test_parity("Math.min(1, 5, 3)", env.clone(), None).await?; - test_parity("Math.abs(-5)", env.clone(), None).await?; - test_parity("Math.floor(3.7)", env.clone(), None).await?; - test_parity("Math.ceil(3.2)", env.clone(), None).await?; - test_parity("Math.round(3.5)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_type_coercion() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert("str".to_string(), Arc::new(to_raw_value(&json!("123")))); - - test_parity("String(num)", env.clone(), None).await?; - test_parity("Number(str)", env.clone(), None).await?; - test_parity("Boolean(num)", env.clone(), None).await?; - test_parity("parseInt('42px')", env.clone(), None).await?; - test_parity("parseFloat('3.14')", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_spread() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr1".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3]))), - ); - env.insert( - "arr2".to_string(), - Arc::new(to_raw_value(&json!([4, 5, 6]))), - ); - - test_parity("[...arr1, ...arr2]", env.clone(), None).await?; - test_parity("[0, ...arr1, 99]", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_statements() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - test_parity( - r#"let y = x * 2; - return y + 1"#, - env.clone(), - None, - ) - .await?; - - test_parity( - r#"const result = x > 3 ? 'big' : 'small'; - return result"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_nullish() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": {"b": 1}}))), - ); - env.insert("empty".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining - test_parity("obj?.a?.b", env.clone(), None).await?; - test_parity("obj?.a?.c", env.clone(), None).await?; - test_parity("obj?.x?.y", env.clone(), None).await?; - test_parity("empty?.foo", env.clone(), None).await?; - - // Nullish coalescing - test_parity("null ?? 'default'", env.clone(), None).await?; - test_parity("undefined ?? 'default'", env.clone(), None).await?; - test_parity("0 ?? 'default'", env.clone(), None).await?; - test_parity("'' ?? 'default'", env.clone(), None).await?; - test_parity("false ?? 'default'", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_destructuring() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"name": "test", "value": 42}))), - ); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Object destructuring - test_parity( - "const { name, value } = obj; return { name, value }", - env.clone(), - None, - ) - .await?; - - // Array destructuring - test_parity( - "const [first, second, ...rest] = arr; return { first, second, rest }", - env.clone(), - None, - ) - .await?; - - // Default values - test_parity( - "const { missing = 'default' } = obj; return missing", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_number_edge_cases() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Basic number operations - test_parity("Number.MAX_SAFE_INTEGER", env.clone(), None).await?; - test_parity("Number.MIN_SAFE_INTEGER", env.clone(), None).await?; - test_parity("Number.isInteger(5)", env.clone(), None).await?; - test_parity("Number.isInteger(5.5)", env.clone(), None).await?; - test_parity("Number.isFinite(Infinity)", env.clone(), None).await?; - test_parity("Number.isNaN(NaN)", env.clone(), None).await?; - - // Floating point - test_parity("0.1 + 0.2", env.clone(), None).await?; - test_parity("Math.round((0.1 + 0.2) * 10) / 10", env.clone(), None).await?; - - // Special values (these serialize to null in JSON) - test_parity("isNaN(NaN)", env.clone(), None).await?; - test_parity("isFinite(100)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_basic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("hello world 123"))), - ); - - // Basic regex operations - test_parity("/hello/.test(str)", env.clone(), None).await?; - test_parity("str.match(/\\d+/)?.[0]", env.clone(), None).await?; - test_parity("str.replace(/world/, 'universe')", env.clone(), None).await?; - test_parity("str.split(/\\s+/)", env.clone(), None).await?; - - // Global flag - test_parity("'aaa'.replace(/a/g, 'b')", env.clone(), None).await?; - - // Case insensitive - test_parity("/HELLO/i.test(str)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_unicode_strings() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "emoji".to_string(), - Arc::new(to_raw_value(&json!("Hello 👋 World 🌍"))), - ); - env.insert( - "chinese".to_string(), - Arc::new(to_raw_value(&json!("你好世界"))), - ); - env.insert( - "mixed".to_string(), - Arc::new(to_raw_value(&json!("Héllo Wörld"))), - ); - - // Basic operations on unicode strings - test_parity("emoji.includes('👋')", env.clone(), None).await?; - test_parity("chinese.length", env.clone(), None).await?; - test_parity("mixed.toUpperCase()", env.clone(), None).await?; - test_parity("mixed.toLowerCase()", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_date_basic() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Static Date methods (deterministic) - test_parity("Date.parse('2024-01-15T00:00:00.000Z')", env.clone(), None).await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').getUTCFullYear()", - env.clone(), - None, - ) - .await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').getUTCMonth()", - env.clone(), - None, - ) - .await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').getUTCDate()", - env.clone(), - None, - ) - .await?; - test_parity( - "new Date('2024-01-15T00:00:00.000Z').toISOString()", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_date_serialization() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Test direct Date object serialization (the key issue that was fixed) - // Both engines should serialize Date to ISO string via toJSON - test_parity("new Date('2024-01-15T12:30:00.000Z')", env.clone(), None).await?; - - // Date within an object - test_parity( - "({ date: new Date('2024-01-15T00:00:00.000Z'), name: 'test' })", - env.clone(), - None, - ) - .await?; - - // Date within an array - test_parity( - "[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]", - env.clone(), - None, - ) - .await?; - - // Deeply nested Date - test_parity( - "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })", - env.clone(), - None, - ) - .await?; - - // Custom object with toJSON (arrow function style) - test_parity( - "({ value: 42, toJSON: () => ({ converted: 84 }) })", - env.clone(), - None, - ) - .await?; - - // toJSON that returns a Date (should be further serialized) - test_parity( - "({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_special_object_serialization() -> anyhow::Result<()> { - let env = HashMap::new(); - - // RegExp serialization (both should return {}) - test_parity("/test/gi", env.clone(), None).await?; - - // Map serialization (both should return {}) - test_parity("new Map([['key', 'value']])", env.clone(), None).await?; - - // Set serialization (both should return {}) - test_parity("new Set([1, 2, 3])", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9, 2, 6]))), - ); - env.insert( - "nested".to_string(), - Arc::new(to_raw_value(&json!([[1, 2], [3, 4], [5, 6]]))), - ); - - // Sorting (note: sort mutates, so we slice first) - test_parity("[...arr].sort((a, b) => a - b)", env.clone(), None).await?; - test_parity("[...arr].sort((a, b) => b - a)", env.clone(), None).await?; - - // Flat operations - test_parity("nested.flat()", env.clone(), None).await?; - test_parity("nested.flatMap(x => x)", env.clone(), None).await?; - - // indexOf, includes - test_parity("arr.indexOf(5)", env.clone(), None).await?; - test_parity("arr.indexOf(99)", env.clone(), None).await?; - test_parity("arr.includes(9)", env.clone(), None).await?; - - // slice, splice behavior - test_parity("arr.slice(2, 5)", env.clone(), None).await?; - test_parity("arr.slice(-3)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_logical_operators() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("a".to_string(), Arc::new(to_raw_value(&json!(true)))); - env.insert("b".to_string(), Arc::new(to_raw_value(&json!(false)))); - env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - // Short-circuit evaluation - test_parity("a && 'yes'", env.clone(), None).await?; - test_parity("b && 'yes'", env.clone(), None).await?; - test_parity("b || 'no'", env.clone(), None).await?; - test_parity("a || 'no'", env.clone(), None).await?; - - // Logical assignment (ES2021) - test_parity("let y = null; y ??= 10; return y", env.clone(), None).await?; - test_parity("let y = 5; y ??= 10; return y", env.clone(), None).await?; - - // Complex conditions - test_parity("(a && x > 3) || (b && x < 3)", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_typeof_instanceof() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("str".to_string(), Arc::new(to_raw_value(&json!("hello")))); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([1, 2, 3])))); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1}))), - ); - env.insert("n".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // typeof - test_parity("typeof str", env.clone(), None).await?; - test_parity("typeof num", env.clone(), None).await?; - test_parity("typeof arr", env.clone(), None).await?; - test_parity("typeof obj", env.clone(), None).await?; - test_parity("typeof n", env.clone(), None).await?; - test_parity("typeof undefined", env.clone(), None).await?; - - // Array.isArray - test_parity("Array.isArray(arr)", env.clone(), None).await?; - test_parity("Array.isArray(obj)", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // COMPLEX MULTILINE EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_multiline_complex_logic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "users".to_string(), - Arc::new(to_raw_value(&json!([ - {"name": "Alice", "age": 30, "role": "admin"}, - {"name": "Bob", "age": 25, "role": "user"}, - {"name": "Charlie", "age": 35, "role": "admin"}, - {"name": "Diana", "age": 28, "role": "user"} - ]))), - ); - - // Complex filtering and mapping - test_parity( - r#" - const admins = users.filter(u => u.role === 'admin'); - const names = admins.map(u => u.name); - return names.join(', ') - "#, - env.clone(), - None, - ) - .await?; - - // Aggregation with reduce - test_parity( - r#" - const totalAge = users.reduce((sum, u) => sum + u.age, 0); - const avgAge = totalAge / users.length; - return Math.round(avgAge) - "#, - env.clone(), - None, - ) - .await?; - - // Group by operation - test_parity( - r#" - const grouped = users.reduce((acc, u) => { - if (!acc[u.role]) acc[u.role] = []; - acc[u.role].push(u.name); - return acc; - }, {}); - return grouped - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_data_transformation() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [ - {"id": 1, "price": 100, "quantity": 2}, - {"id": 2, "price": 50, "quantity": 5}, - {"id": 3, "price": 75, "quantity": 3} - ], - "discount": 0.1 - }))), - ); - - // Calculate total with discount - test_parity( - r#" - const subtotals = data.items.map(item => item.price * item.quantity); - const total = subtotals.reduce((a, b) => a + b, 0); - const discounted = total * (1 - data.discount); - return { subtotals, total, discounted } - "#, - env.clone(), - None, - ) - .await?; - - // Transform data structure - test_parity( - r#" - const result = data.items.map(item => ({ - ...item, - subtotal: item.price * item.quantity, - discountedSubtotal: item.price * item.quantity * (1 - data.discount) - })); - return result - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_conditional_logic() -> anyhow::Result<()> { - // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout - - let mut env = HashMap::new(); - env.insert("status".to_string(), Arc::new(to_raw_value(&json!("pending")))); - env.insert("retries".to_string(), Arc::new(to_raw_value(&json!(3)))); - env.insert("maxRetries".to_string(), Arc::new(to_raw_value(&json!(5)))); - - // Complex conditional with multiple branches - test_parity( - r#" - let action; - if (status === 'success') { - action = 'complete'; - } else if (status === 'pending' && retries < maxRetries) { - action = 'retry'; - } else if (status === 'pending') { - action = 'fail'; - } else { - action = 'unknown'; - } - return { action, retriesLeft: maxRetries - retries } - "#, - env.clone(), - None, - ) - .await?; - - // Switch-like using object lookup - test_parity( - r#" - const actions = { - 'success': () => ({ next: 'complete', message: 'Done!' }), - 'pending': () => ({ next: 'retry', message: `Retry ${retries + 1}/${maxRetries}` }), - 'failed': () => ({ next: 'stop', message: 'Giving up' }) - }; - const handler = actions[status] || (() => ({ next: 'fallback', message: 'Unknown status' })); - return handler() - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // ARROW FUNCTION VARIATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_arrow_functions() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "numbers".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Concise body (implicit return) - test_parity("numbers.map(n => n * 2)", env.clone(), None).await?; - - // Block body (explicit return) - test_parity( - "numbers.map(n => { return n * 2; })", - env.clone(), - None, - ) - .await?; - - // Multiple parameters - test_parity( - "numbers.reduce((acc, n) => acc + n, 0)", - env.clone(), - None, - ) - .await?; - - // Destructuring in parameters - test_parity( - r#" - const pairs = [[1, 2], [3, 4], [5, 6]]; - return pairs.map(([a, b]) => a + b) - "#, - HashMap::new(), - None, - ) - .await?; - - // Object destructuring in parameters - test_parity( - r#" - const items = [{x: 1, y: 2}, {x: 3, y: 4}]; - return items.map(({x, y}) => x * y) - "#, - HashMap::new(), - None, - ) - .await?; - - // Nested arrow functions - test_parity( - "numbers.map(n => numbers.filter(m => m !== n))", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // TRY-CATCH EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_try_catch() -> anyhow::Result<()> { - // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout - - let env = HashMap::new(); - - // Basic try-catch - test_parity( - r#" - try { - return JSON.parse('{"valid": true}'); - } catch (e) { - return { problem: e.message }; - } - "#, - env.clone(), - None, - ) - .await?; - - // Try-catch with invalid JSON - test_parity( - r#" - try { - return JSON.parse('invalid json'); - } catch (e) { - return { problem: 'parse_failed' }; - } - "#, - env.clone(), - None, - ) - .await?; - - // Try-catch-finally - test_parity( - r#" - let result = 'initial'; - try { - result = 'try'; - } catch (e) { - result = 'catch'; - } finally { - result = result + '_finally'; - } - return result - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // COMPLEX OBJECT OPERATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_object_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "config".to_string(), - Arc::new(to_raw_value(&json!({ - "server": {"host": "localhost", "port": 8080}, - "database": {"host": "db.local", "port": 5432}, - "features": ["auth", "logging", "cache"] - }))), - ); - - // Object.assign - test_parity( - "Object.assign({}, config.server, { secure: true })", - env.clone(), - None, - ) - .await?; - - // Object spread with override - test_parity( - "({ ...config.server, port: 443, secure: true })", - env.clone(), - None, - ) - .await?; - - // Object.entries and Object.fromEntries - test_parity( - r#" - const entries = Object.entries(config.server); - const reversed = entries.map(([k, v]) => [k.toUpperCase(), v]); - return Object.fromEntries(reversed) - "#, - env.clone(), - None, - ) - .await?; - - // Deep clone pattern - test_parity( - "JSON.parse(JSON.stringify(config))", - env.clone(), - None, - ) - .await?; - - // Computed property names - test_parity( - r#" - const key = 'dynamic'; - return { [key]: 'value', [`${key}_2`]: 'value2' } - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // STRING MANIPULATION ADVANCED - // ========================================================================= - - #[tokio::test] - async fn parity_string_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!(" Hello, World! "))), - ); - env.insert( - "path".to_string(), - Arc::new(to_raw_value(&json!("/api/v1/users/123/profile"))), - ); - - // Trim variants - test_parity("text.trim()", env.clone(), None).await?; - test_parity("text.trimStart()", env.clone(), None).await?; - test_parity("text.trimEnd()", env.clone(), None).await?; - - // Padding - test_parity("'42'.padStart(5, '0')", env.clone(), None).await?; - test_parity("'42'.padEnd(5, '-')", env.clone(), None).await?; - - // Repeat - test_parity("'ab'.repeat(3)", env.clone(), None).await?; - - // Path manipulation - test_parity( - "path.split('/').filter(p => p.length > 0)", - env.clone(), - None, - ) - .await?; - - // Template literal with expressions - test_parity( - r#"`Path parts: ${path.split('/').filter(p => p).length}`"#, - env.clone(), - None, - ) - .await?; - - // String search methods - test_parity("path.indexOf('/users/')", env.clone(), None).await?; - test_parity("path.lastIndexOf('/')", env.clone(), None).await?; - test_parity("path.substring(0, 7)", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // ARRAY MANIPULATION ADVANCED - // ========================================================================= - - #[tokio::test] - async fn parity_array_manipulation() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "items".to_string(), - Arc::new(to_raw_value(&json!([ - {"id": 1, "name": "Apple", "category": "fruit"}, - {"id": 2, "name": "Carrot", "category": "vegetable"}, - {"id": 3, "name": "Banana", "category": "fruit"}, - {"id": 4, "name": "Broccoli", "category": "vegetable"} - ]))), - ); - - // find and findIndex - test_parity( - "items.find(i => i.name === 'Banana')", - env.clone(), - None, - ) - .await?; - - test_parity( - "items.findIndex(i => i.name === 'Banana')", - env.clone(), - None, - ) - .await?; - - // Filter and sort chain - test_parity( - "items.filter(i => i.category === 'fruit').map(i => i.name).sort()", - env.clone(), - None, - ) - .await?; - - // Array.from with map function - test_parity( - "Array.from({length: 5}, (_, i) => i * 2)", - env.clone(), - None, - ) - .await?; - - // Array fill - test_parity("Array(3).fill(0)", env.clone(), None).await?; - - // Reverse (on copy to avoid mutation) - test_parity( - "[...items].reverse().map(i => i.name)", - env.clone(), - None, - ) - .await?; - - // concat - test_parity( - "[1, 2].concat([3, 4], [5, 6])", - env.clone(), - None, - ) - .await?; - - // join variations - test_parity( - "items.map(i => i.name).join(' | ')", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // REAL-WORLD FLOW EXPRESSION PATTERNS - // ========================================================================= - - #[tokio::test] - async fn parity_flow_patterns_api_response() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "status": 200, - "data": { - "users": [ - {"id": 1, "email": "alice@example.com", "active": true}, - {"id": 2, "email": "bob@example.com", "active": false}, - {"id": 3, "email": "charlie@example.com", "active": true} - ], - "pagination": {"page": 1, "total": 50, "per_page": 10} - } - }))), - ); - - // Extract active users' emails - test_parity( - "previous_result.data.users.filter(u => u.active).map(u => u.email)", - env.clone(), - None, - ) - .await?; - - // Check if more pages exist - test_parity( - r#" - const { page, total, per_page } = previous_result.data.pagination; - return page * per_page < total - "#, - env.clone(), - None, - ) - .await?; - - // Transform to different structure - test_parity( - r#"({ - emails: previous_result.data.users.map(u => u.email), - activeCount: previous_result.data.users.filter(u => u.active).length, - hasMore: previous_result.data.pagination.page * previous_result.data.pagination.per_page < previous_result.data.pagination.total - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_patterns_failure_handling() -> anyhow::Result<()> { - // NOTE: We avoid using the literal word "error" in expressions because - // it triggers special error-handling code that has a bug with duplicate declarations. - - // Test with failure info in previous_result - let mut env_failure = HashMap::new(); - env_failure.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "failure": { - "name": "APIFailure", - "message": "Rate limit exceeded", - "code": 429 - } - }))), - ); - - // Check for failure presence - test_parity( - "previous_result?.failure ? true : false", - env_failure.clone(), - None, - ) - .await?; - - // Extract failure details - test_parity( - "previous_result.failure?.code ?? 500", - env_failure.clone(), - None, - ) - .await?; - - // Test with successful result (no failure) - let mut env_success = HashMap::new(); - env_success.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "data": "success" - }))), - ); - - test_parity( - "previous_result?.failure ? 'failed' : 'ok'", - env_success.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_patterns_conditional_branching() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "step_a".to_string(), - Arc::new(to_raw_value(&json!({"count": 5}))), - ); - env.insert( - "step_b".to_string(), - Arc::new(to_raw_value(&json!({"count": 10}))), - ); - env.insert("threshold".to_string(), Arc::new(to_raw_value(&json!(7)))); - - // Branch selection based on condition - test_parity( - "step_a.count > threshold ? 'high' : step_b.count > threshold ? 'medium' : 'low'", - env.clone(), - None, - ) - .await?; - - // Aggregate from multiple steps - test_parity( - "({ total: step_a.count + step_b.count, average: (step_a.count + step_b.count) / 2 })", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_patterns_data_mapping() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "source".to_string(), - Arc::new(to_raw_value(&json!({ - "firstName": "John", - "lastName": "Doe", - "birthDate": "1990-05-15", - "addresses": [ - {"type": "home", "city": "New York"}, - {"type": "work", "city": "Boston"} - ] - }))), - ); - - // Map to different schema - test_parity( - r#"({ - fullName: `${source.firstName} ${source.lastName}`, - birth_date: source.birthDate, - primary_city: source.addresses.find(a => a.type === 'home')?.city ?? source.addresses[0]?.city ?? 'Unknown', - all_cities: source.addresses.map(a => a.city) - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // EDGE CASES AND SPECIAL VALUES - // ========================================================================= - - #[tokio::test] - async fn parity_edge_cases_empty_values() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("emptyArray".to_string(), Arc::new(to_raw_value(&json!([])))); - env.insert("emptyObject".to_string(), Arc::new(to_raw_value(&json!({})))); - env.insert("emptyString".to_string(), Arc::new(to_raw_value(&json!("")))); - env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - - // Operations on empty values - test_parity("emptyArray.length", env.clone(), None).await?; - test_parity("emptyArray.map(x => x * 2)", env.clone(), None).await?; - test_parity("emptyArray.filter(x => x > 0)", env.clone(), None).await?; - test_parity("emptyArray.reduce((a, b) => a + b, 100)", env.clone(), None).await?; - - test_parity("Object.keys(emptyObject)", env.clone(), None).await?; - test_parity("Object.values(emptyObject)", env.clone(), None).await?; - - test_parity("emptyString.length", env.clone(), None).await?; - test_parity("emptyString || 'default'", env.clone(), None).await?; - test_parity("emptyString ?? 'default'", env.clone(), None).await?; - - test_parity("zero || 'default'", env.clone(), None).await?; - test_parity("zero ?? 'default'", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_nested_access() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "deep".to_string(), - Arc::new(to_raw_value(&json!({ - "a": {"b": {"c": {"d": {"e": "found!"}}}} - }))), - ); - - // Deep property access - test_parity("deep.a.b.c.d.e", env.clone(), None).await?; - test_parity("deep?.a?.b?.c?.d?.e", env.clone(), None).await?; - test_parity("deep?.a?.b?.x?.y?.z", env.clone(), None).await?; - test_parity("deep?.a?.b?.x?.y?.z ?? 'not found'", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_special_characters() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "key-with-dash": "value1", - "key.with.dots": "value2", - "key with spaces": "value3" - }))), - ); - - // Bracket notation for special keys - test_parity("data['key-with-dash']", env.clone(), None).await?; - test_parity("data['key.with.dots']", env.clone(), None).await?; - test_parity("data['key with spaces']", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_large_numbers() -> anyhow::Result<()> { - let mut env = HashMap::new(); - // Large but safe integers - env.insert( - "bigNum".to_string(), - Arc::new(to_raw_value(&json!(9007199254740991_i64))), // MAX_SAFE_INTEGER - ); - env.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1704067200000_i64))), // 2024-01-01 UTC - ); - - test_parity("bigNum", env.clone(), None).await?; - test_parity("timestamp", env.clone(), None).await?; - test_parity("new Date(timestamp).toISOString()", env.clone(), None).await?; - - // Arithmetic on large numbers - test_parity("bigNum - 1", env.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_edge_cases_boolean_coercion() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Falsy values - test_parity("Boolean(0)", env.clone(), None).await?; - test_parity("Boolean('')", env.clone(), None).await?; - test_parity("Boolean(null)", env.clone(), None).await?; - test_parity("Boolean(undefined)", env.clone(), None).await?; - test_parity("Boolean(NaN)", env.clone(), None).await?; - - // Truthy values - test_parity("Boolean(1)", env.clone(), None).await?; - test_parity("Boolean('hello')", env.clone(), None).await?; - test_parity("Boolean([])", env.clone(), None).await?; - test_parity("Boolean({})", env.clone(), None).await?; - - // Double negation coercion - test_parity("!!0", env.clone(), None).await?; - test_parity("!!1", env.clone(), None).await?; - test_parity("!!''", env.clone(), None).await?; - test_parity("!!'hello'", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // PROMISES AND ASYNC PATTERNS (without client) - // ========================================================================= - - #[tokio::test] - async fn parity_promise_resolve() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Basic Promise.resolve - test_parity( - "Promise.resolve(42)", - env.clone(), - None, - ) - .await?; - - test_parity( - "Promise.resolve({ key: 'value' })", - env.clone(), - None, - ) - .await?; - - // Promise.all with resolved values - test_parity( - "Promise.all([Promise.resolve(1), Promise.resolve(2), Promise.resolve(3)])", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // SET AND MAP OPERATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_set_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 2, 3, 3, 3, 4]))), - ); - - // Deduplicate using Set - test_parity( - "[...new Set(arr)]", - env.clone(), - None, - ) - .await?; - - // Set size - test_parity( - "new Set(arr).size", - env.clone(), - None, - ) - .await?; - - // Set.has - test_parity( - "new Set(arr).has(3)", - env.clone(), - None, - ) - .await?; - - test_parity( - "new Set(arr).has(99)", - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_map_operations() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Create Map and convert to object - test_parity( - r#" - const map = new Map([['a', 1], ['b', 2], ['c', 3]]); - return Object.fromEntries(map) - "#, - env.clone(), - None, - ) - .await?; - - // Map operations - test_parity( - r#" - const map = new Map(); - map.set('key1', 'value1'); - map.set('key2', 'value2'); - return map.get('key1') - "#, - env.clone(), - None, - ) - .await?; - - test_parity( - r#" - const map = new Map([['a', 1], ['b', 2]]); - return map.size - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - // ========================================================================= - // COMPARISON OPERATORS - // ========================================================================= - - #[tokio::test] - async fn parity_comparisons() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Strict equality - test_parity("1 === 1", env.clone(), None).await?; - test_parity("1 === '1'", env.clone(), None).await?; - test_parity("null === undefined", env.clone(), None).await?; - test_parity("null === null", env.clone(), None).await?; - - // Loose equality - test_parity("1 == '1'", env.clone(), None).await?; - test_parity("null == undefined", env.clone(), None).await?; - test_parity("0 == false", env.clone(), None).await?; - test_parity("'' == false", env.clone(), None).await?; - - // Inequality - test_parity("5 !== '5'", env.clone(), None).await?; - test_parity("5 != '5'", env.clone(), None).await?; - - // Comparison operators - test_parity("5 > 3", env.clone(), None).await?; - test_parity("5 >= 5", env.clone(), None).await?; - test_parity("3 < 5", env.clone(), None).await?; - test_parity("5 <= 5", env.clone(), None).await?; - - // String comparison - test_parity("'apple' < 'banana'", env.clone(), None).await?; - test_parity("'10' < '9'", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // BITWISE OPERATIONS - // ========================================================================= - - #[tokio::test] - async fn parity_bitwise() -> anyhow::Result<()> { - let env = HashMap::new(); - - test_parity("5 & 3", env.clone(), None).await?; - test_parity("5 | 3", env.clone(), None).await?; - test_parity("5 ^ 3", env.clone(), None).await?; - test_parity("~5", env.clone(), None).await?; - test_parity("5 << 2", env.clone(), None).await?; - test_parity("20 >> 2", env.clone(), None).await?; - test_parity("-5 >>> 0", env.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // COMPLEX REAL-WORLD SCENARIOS - // ========================================================================= - - #[tokio::test] - async fn parity_scenario_batch_processing() -> anyhow::Result<()> { - // NOTE: We avoid the word "error" in expressions due to special handling in eval_timeout - - let mut env = HashMap::new(); - env.insert( - "jobs".to_string(), - Arc::new(to_raw_value(&json!([ - {"id": 1, "status": "completed", "result": 100}, - {"id": 2, "status": "failed", "reason": "timeout"}, - {"id": 3, "status": "completed", "result": 200}, - {"id": 4, "status": "failed", "reason": "connection"}, - {"id": 5, "status": "completed", "result": 150} - ]))), - ); - - // Aggregate batch results - test_parity( - r#" - const completed = jobs.filter(j => j.status === 'completed'); - const failed = jobs.filter(j => j.status === 'failed'); - const totalResult = completed.reduce((sum, j) => sum + j.result, 0); - return { - totalJobs: jobs.length, - completedCount: completed.length, - failedCount: failed.length, - successRate: completed.length / jobs.length, - totalResult, - failureReasons: failed.map(j => j.reason) - } - "#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_webhook_payload() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "webhook".to_string(), - Arc::new(to_raw_value(&json!({ - "event": "user.created", - "timestamp": "2024-01-15T10:30:00Z", - "data": { - "user": { - "id": "usr_123", - "email": "newuser@example.com", - "metadata": { - "source": "signup", - "campaign": "winter_2024" - } - } - } - }))), - ); - - // Extract and transform webhook data - test_parity( - r#"({ - eventType: webhook.event.split('.')[1], - userId: webhook.data.user.id, - userEmail: webhook.data.user.email, - source: webhook.data.user.metadata?.source ?? 'unknown', - campaign: webhook.data.user.metadata?.campaign, - processedAt: new Date().toISOString().split('T')[0] - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_config_merge() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "defaults".to_string(), - Arc::new(to_raw_value(&json!({ - "timeout": 5000, - "retries": 3, - "headers": {"Content-Type": "application/json"}, - "features": {"logging": true, "caching": false} - }))), - ); - env.insert( - "overrides".to_string(), - Arc::new(to_raw_value(&json!({ - "timeout": 10000, - "headers": {"Authorization": "Bearer token"}, - "features": {"caching": true} - }))), - ); - - // Deep merge configuration - test_parity( - r#"({ - ...defaults, - ...overrides, - headers: { ...defaults.headers, ...overrides.headers }, - features: { ...defaults.features, ...overrides.features } - })"#, - env.clone(), - None, - ) - .await?; - - Ok(()) - } -} - -#[cfg(test)] -mod benchmark_tests { - use std::collections::HashMap; - use std::sync::Arc; - use std::time::Instant; - - use serde_json::json; - use windmill_common::worker::to_raw_value; - - /// Benchmark QuickJS expression evaluation startup time - #[cfg(feature = "quickjs")] - #[tokio::test] - async fn benchmark_quickjs_startup() -> anyhow::Result<()> { - use crate::js_eval_quickjs::eval_timeout_quickjs; - - let iterations = 100; - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - let start = Instant::now(); - for _ in 0..iterations { - let _ = eval_timeout_quickjs( - "x + 1".to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - } - let duration = start.elapsed(); - - println!( - "QuickJS: {} iterations in {:?} ({:?} per iteration)", - iterations, - duration, - duration / iterations - ); - - Ok(()) - } - - /// Benchmark deno_core expression evaluation startup time - #[cfg(feature = "deno_core")] - #[tokio::test] - async fn benchmark_deno_startup() -> anyhow::Result<()> { - use crate::js_eval::eval_timeout; - - let iterations = 100; - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - - let start = Instant::now(); - for _ in 0..iterations { - let _ = eval_timeout( - "x + 1".to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - } - let duration = start.elapsed(); - - println!( - "deno_core: {} iterations in {:?} ({:?} per iteration)", - iterations, - duration, - duration / iterations - ); - - Ok(()) - } - - /// Benchmark both engines with a complex expression - #[cfg(all(feature = "deno_core", feature = "quickjs"))] - #[tokio::test] - async fn benchmark_complex_expression() -> anyhow::Result<()> { - use crate::js_eval::eval_timeout; - use crate::js_eval_quickjs::eval_timeout_quickjs; - - let iterations = 50; - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "multiplier": 2 - }))), - ); - - let expr = "data.items.filter(x => x > 3).map(x => x * data.multiplier).reduce((a, b) => a + b, 0)"; - - // QuickJS - let start = Instant::now(); - for _ in 0..iterations { - let _ = eval_timeout_quickjs( - expr.to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - } - let quickjs_duration = start.elapsed(); - - // deno_core - let start = Instant::now(); - for _ in 0..iterations { - let _ = eval_timeout( - expr.to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - } - let deno_duration = start.elapsed(); - - println!( - "Complex expression benchmark ({} iterations):\n QuickJS: {:?} ({:?}/iter)\n deno_core: {:?} ({:?}/iter)\n Speedup: {:.2}x", - iterations, - quickjs_duration, quickjs_duration / iterations, - deno_duration, deno_duration / iterations, - deno_duration.as_secs_f64() / quickjs_duration.as_secs_f64() - ); - - Ok(()) - } -} - -/// Comprehensive flow simulation parity tests -/// Tests expression evaluation in contexts that simulate real flow execution -#[cfg(all(test, feature = "deno_core", feature = "quickjs"))] -mod flow_simulation_parity_tests { - use std::collections::HashMap; - use std::sync::Arc; - - use serde_json::json; - use serde_json::value::RawValue; - use windmill_common::worker::to_raw_value; - - use crate::js_eval::eval_timeout; - use crate::js_eval_quickjs::eval_timeout_quickjs; - - /// Helper to run the same test on both engines and compare results - async fn test_parity( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option>>, - ) -> anyhow::Result<()> { - let deno_result = eval_timeout( - expr.to_string(), - transform_context.clone(), - flow_input.clone(), - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - let quickjs_result = eval_timeout_quickjs( - expr.to_string(), - transform_context, - flow_input, - flow_env.as_ref(), - None, - None, - None, - ) - .await?; - - let deno_value: serde_json::Value = serde_json::from_str(deno_result.get())?; - let quickjs_value: serde_json::Value = serde_json::from_str(quickjs_result.get())?; - - assert_eq!( - deno_value, quickjs_value, - "Results differ for expression '{}'\ndeno_core: {}\nquickjs: {}", - expr, deno_result.get(), quickjs_result.get() - ); - - Ok(()) - } - - // ========================================================================= - // SIMULATED FLOW CONTEXT: Multi-step flow with various step results - // ========================================================================= - - fn create_multi_step_flow_context() -> ( - HashMap>>, - Option>>>, - Option>>, - ) { - let mut transform_context = HashMap::new(); - - // Step 'a' result: simple number - transform_context.insert( - "a".to_string(), - Arc::new(to_raw_value(&json!(42))), - ); - - // Step 'b' result: object with nested data - transform_context.insert( - "b".to_string(), - Arc::new(to_raw_value(&json!({ - "status": "success", - "data": { - "users": [ - {"id": 1, "name": "Alice", "active": true, "roles": ["admin", "user"]}, - {"id": 2, "name": "Bob", "active": false, "roles": ["user"]}, - {"id": 3, "name": "Charlie", "active": true, "roles": ["moderator", "user"]} - ], - "total": 3, - "metadata": { - "page": 1, - "hasMore": true - } - } - }))), - ); - - // Step 'c' result: array of numbers (from a for-loop) - transform_context.insert( - "c".to_string(), - Arc::new(to_raw_value(&json!([10, 20, 30, 40, 50]))), - ); - - // Step 'd' result: null (simulating a step that returned null) - transform_context.insert( - "d".to_string(), - Arc::new(to_raw_value(&json!(null))), - ); - - // Step 'e' result: error object (simulating a failed step with continue_on_error) - transform_context.insert( - "e".to_string(), - Arc::new(to_raw_value(&json!({ - "error": { - "name": "ValidationError", - "message": "Invalid input provided", - "step_id": "e" - } - }))), - ); - - // Step 'f' result: deeply nested object - transform_context.insert( - "f".to_string(), - Arc::new(to_raw_value(&json!({ - "level1": { - "level2": { - "level3": { - "level4": { - "value": "deeply_nested" - } - } - } - } - }))), - ); - - // Step 'g' result: array of mixed types - transform_context.insert( - "g".to_string(), - Arc::new(to_raw_value(&json!([ - "string", - 123, - true, - null, - {"key": "value"}, - [1, 2, 3] - ]))), - ); - - // previous_result (the last executed step, 'g') - transform_context.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!([ - "string", - 123, - true, - null, - {"key": "value"}, - [1, 2, 3] - ]))), - ); - - // flow_input - let mut flow_input = HashMap::new(); - flow_input.insert("name".to_string(), to_raw_value(&json!("test_flow"))); - flow_input.insert("count".to_string(), to_raw_value(&json!(100))); - flow_input.insert("enabled".to_string(), to_raw_value(&json!(true))); - flow_input.insert( - "config".to_string(), - to_raw_value(&json!({ - "timeout": 30, - "retries": 3, - "options": ["fast", "secure"] - })), - ); - flow_input.insert( - "items".to_string(), - to_raw_value(&json!([ - {"id": 1, "value": "first"}, - {"id": 2, "value": "second"}, - {"id": 3, "value": "third"} - ])), - ); - - // flow_env - let mut flow_env = HashMap::new(); - flow_env.insert("ENV".to_string(), to_raw_value(&json!("production"))); - flow_env.insert("DEBUG".to_string(), to_raw_value(&json!(false))); - flow_env.insert("VERSION".to_string(), to_raw_value(&json!("1.2.3"))); - - ( - transform_context, - Some(mappable_rc::Marc::new(flow_input)), - Some(flow_env), - ) - } - - // ========================================================================= - // INPUT TRANSFORM EXPRESSIONS (step inputs) - // ========================================================================= - - #[tokio::test] - async fn parity_input_transform_direct_reference() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Direct step result reference - test_parity("a", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_input_transform_property_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Nested property access - test_parity("b.status", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.users[0].name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.users[1].roles", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.metadata.hasMore", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Deeply nested - test_parity("f.level1.level2.level3.level4.value", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_input_transform_array_operations() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Array indexing - test_parity("c[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c[c.length - 1]", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Array methods - test_parity("c.map(x => x * 2)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.filter(x => x > 25)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.reduce((acc, x) => acc + x, 0)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.find(x => x === 30)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.some(x => x > 40)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("c.every(x => x > 0)", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Chained operations - test_parity("c.filter(x => x > 20).map(x => x / 10)", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_input_transform_complex_expressions() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Complex data extraction from step 'b' - test_parity( - "b.data.users.filter(u => u.active).map(u => u.name)", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - test_parity( - "b.data.users.filter(u => u.roles.includes('admin'))[0]?.name", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - test_parity( - "b.data.users.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - // Combining multiple step results - test_parity("a + c[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("a * b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - // ========================================================================= - // FLOW_INPUT EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_flow_input_simple() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("flow_input.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.count", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.enabled", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input_nested() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("flow_input.config.timeout", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.retries", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.options", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.options[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input_array_operations() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("flow_input.items.length", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.items[0].id", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.items.map(i => i.value)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.items.find(i => i.id === 2)", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_input_combined_with_steps() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Combining flow_input with step results - test_parity("flow_input.count + a", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.config.timeout * b.data.total", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Conditional based on flow_input - test_parity( - "flow_input.enabled ? b.data.users : []", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - Ok(()) - } - - // ========================================================================= - // FLOW_ENV EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_flow_env_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("flow_env.ENV", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_env.DEBUG", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_env.VERSION", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_env_conditionals() -> anyhow::Result<()> { - // Test flow_env conditionals with explicit flow_env reference in context - let mut ctx = HashMap::new(); - ctx.insert("env_val".to_string(), Arc::new(to_raw_value(&json!("production")))); - ctx.insert("debug_val".to_string(), Arc::new(to_raw_value(&json!(false)))); - - test_parity( - "env_val === 'production' ? 'prod' : 'dev'", - ctx.clone(), None, None - ).await?; - - test_parity( - "debug_val ? 'debug mode' : 'normal'", - ctx.clone(), None, None - ).await?; - - Ok(()) - } - - // ========================================================================= - // ITERATOR EXPRESSIONS (for forloopflow) - // ========================================================================= - - #[tokio::test] - async fn parity_iterator_expressions() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Typical iterator expressions - test_parity("c", ctx.clone(), fi.clone(), fe.clone()).await?; // Direct array - test_parity("b.data.users", ctx.clone(), fi.clone(), fe.clone()).await?; // Nested array - test_parity("flow_input.items", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Transformed iterators - test_parity("c.map(x => ({value: x, doubled: x * 2}))", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b.data.users.filter(u => u.active)", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Range-like iteration - test_parity("Array.from({length: 5}, (_, i) => i)", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_forloop_inner_expressions() -> anyhow::Result<()> { - // Simulate expressions inside a for-loop where flow_input.iter exists - let mut ctx = HashMap::new(); - ctx.insert("previous_result".to_string(), Arc::new(to_raw_value(&json!({"value": 42, "index": 2})))); - - let mut flow_input = HashMap::new(); - flow_input.insert("iter".to_string(), to_raw_value(&json!({ - "index": 2, - "value": {"id": 3, "name": "test_item"} - }))); - flow_input.insert("name".to_string(), to_raw_value(&json!("parent_flow"))); - - let fi = Some(mappable_rc::Marc::new(flow_input)); - - test_parity("flow_input.iter.index", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.iter.value", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.iter.value.id", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.iter.value.name", ctx.clone(), fi.clone(), None).await?; - - // Combining iter with other flow_input - test_parity( - "`Item ${flow_input.iter.index} of ${flow_input.name}`", - ctx.clone(), fi.clone(), None - ).await?; - - Ok(()) - } - - // ========================================================================= - // BRANCH CONDITION EXPRESSIONS (for branchone) - // ========================================================================= - - #[tokio::test] - async fn parity_branch_conditions() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - // Simple boolean conditions - test_parity("a > 40", ctx.clone(), fi.clone(), None).await?; - test_parity("b.status === 'success'", ctx.clone(), fi.clone(), None).await?; - test_parity("flow_input.enabled", ctx.clone(), fi.clone(), None).await?; - - // Complex boolean conditions - test_parity("a > 40 && b.status === 'success'", ctx.clone(), fi.clone(), None).await?; - test_parity("a < 50 || b.data.total > 5", ctx.clone(), fi.clone(), None).await?; - - // Conditions with array checks - test_parity("b.data.users.length > 0", ctx.clone(), fi.clone(), None).await?; - test_parity("b.data.users.some(u => u.active)", ctx.clone(), fi.clone(), None).await?; - test_parity("c.includes(30)", ctx.clone(), fi.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // SKIP_IF / STOP_AFTER_IF EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_skip_if_expressions() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Skip based on previous result - test_parity("previous_result === null", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result.length === 0", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Skip based on flow_input - test_parity("!flow_input.enabled", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input.count === 0", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Skip based on step result - test_parity("d === null", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("e?.error !== undefined", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_stop_after_if_expressions() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - // Stop conditions (avoid previous_result?.error pattern which has issues with error extraction) - test_parity("a >= 42", ctx.clone(), fi.clone(), None).await?; - test_parity("b.data.metadata.hasMore === false", ctx.clone(), fi.clone(), None).await?; - test_parity("b.status !== 'success'", ctx.clone(), fi.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // UNDEFINED/MISSING STEP RESULTS (simulating non-executed branches) - // ========================================================================= - - #[tokio::test] - async fn parity_missing_step_with_optional_chaining() -> anyhow::Result<()> { - // Context where some steps weren't executed (e.g., branch not taken) - let mut ctx = HashMap::new(); - ctx.insert("a".to_string(), Arc::new(to_raw_value(&json!(42)))); - // 'b' was never executed (branch not taken) - ctx.insert("c".to_string(), Arc::new(to_raw_value(&json!(null)))); // Step returned null - - // Safe access to potentially missing step - test_parity("a", ctx.clone(), None, None).await?; - test_parity("c", ctx.clone(), None, None).await?; - - // Optional chaining on null - test_parity("c?.value", ctx.clone(), None, None).await?; - test_parity("c?.nested?.deep", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_null_coalescing_for_missing_data() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Nullish coalescing - test_parity("d ?? 'default'", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("d?.value ?? 'not found'", ctx.clone(), fi.clone(), fe.clone()).await?; - - // With nested access - test_parity("b.data.missing?.value ?? 'fallback'", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - // ========================================================================= - // ERROR HANDLING EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_error_object_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Accessing error from step 'e' - test_parity("e.error.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("e.error.message", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("e.error.step_id", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Conditional based on error - test_parity( - "e.error ? `Error: ${e.error.message}` : 'OK'", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_error_variable_extraction() -> anyhow::Result<()> { - // Simulate previous_result being an error - let mut ctx = HashMap::new(); - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!({ - "error": { - "name": "RuntimeError", - "message": "Something went wrong" - } - }))), - ); - - // The 'error' variable is extracted from previous_result - test_parity("error.name", ctx.clone(), None, None).await?; - test_parity("error.message", ctx.clone(), None, None).await?; - test_parity("`${error.name}: ${error.message}`", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_error_from_parallel_results() -> anyhow::Result<()> { - // Simulate previous_result being an array with errors (from parallel branches) - let mut ctx = HashMap::new(); - ctx.insert( - "previous_result".to_string(), - Arc::new(to_raw_value(&json!([ - {"result": "success"}, - {"error": {"name": "Error1", "message": "First error", "step_id": "branch_1"}}, - {"result": "also success"}, - {"error": {"name": "Error2", "message": "Second error", "step_id": "branch_2"}} - ]))), - ); - - // Access the aggregated error - test_parity("error.name", ctx.clone(), None, None).await?; - test_parity("error.message", ctx.clone(), None, None).await?; - test_parity("error.errors", ctx.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // OBJECT CONSTRUCTION EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_object_construction() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Building objects from step results - test_parity( - "({ count: a, users: b.data.users })", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - test_parity( - "({ ...flow_input.config, extra: 'value' })", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - // Computed properties - test_parity( - "({ [`step_${a}`]: b.status })", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_array_construction() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Spread operator - test_parity("[...c, 60, 70]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("[a, ...c.slice(0, 2)]", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Array from step results - test_parity( - "[b.data.users[0], b.data.users[2]]", - ctx.clone(), fi.clone(), fe.clone() - ).await?; - - Ok(()) - } - - // ========================================================================= - // MULTILINE / COMPLEX EXPRESSIONS - // ========================================================================= - - #[tokio::test] - async fn parity_multiline_data_processing() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity(r#" - let users = b.data.users; - let activeUsers = users.filter(u => u.active); - let adminUsers = activeUsers.filter(u => u.roles.includes('admin')); - return adminUsers.map(u => u.name); - "#, ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_conditional_logic() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - test_parity(r#" - if (flow_input.enabled) { - return { mode: 'enabled', data: b.data.users.filter(u => u.active) }; - } else { - return { mode: 'disabled', data: b.data.users }; - } - "#, ctx.clone(), fi.clone(), None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_multiline_aggregation() -> anyhow::Result<()> { - let (ctx, fi, _fe) = create_multi_step_flow_context(); - - test_parity(r#" - const summary = { - stepA: a, - stepB_status: b.status, - stepC_sum: c.reduce((acc, x) => acc + x, 0), - stepC_count: c.length, - activeUserCount: b.data.users.filter(u => u.active).length, - flowName: flow_input.name, - enabled: flow_input.enabled - }; - return summary; - "#, ctx.clone(), fi.clone(), None).await?; - - Ok(()) - } - - // ========================================================================= - // EDGE CASES - // ========================================================================= - - #[tokio::test] - async fn parity_empty_arrays_and_objects() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert("emptyArr".to_string(), Arc::new(to_raw_value(&json!([])))); - ctx.insert("emptyObj".to_string(), Arc::new(to_raw_value(&json!({})))); - - test_parity("emptyArr.length", ctx.clone(), None, None).await?; - test_parity("emptyArr.map(x => x)", ctx.clone(), None, None).await?; - test_parity("emptyArr.filter(x => true)", ctx.clone(), None, None).await?; - test_parity("Object.keys(emptyObj)", ctx.clone(), None, None).await?; - test_parity("Object.values(emptyObj)", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_large_numbers() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert("bigInt".to_string(), Arc::new(to_raw_value(&json!(9007199254740991_i64)))); // MAX_SAFE_INTEGER - ctx.insert("timestamp".to_string(), Arc::new(to_raw_value(&json!(1703980800000_i64)))); // Typical timestamp - - test_parity("bigInt", ctx.clone(), None, None).await?; - test_parity("timestamp", ctx.clone(), None, None).await?; - test_parity("bigInt + 1", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_special_characters_in_strings() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "message": "Hello, \"World\"!", - "path": "C:\\Users\\test", - "newlines": "line1\nline2\nline3", - "unicode": "こんにちは 🌍", - "empty": "" - }))), - ); - - test_parity("data.message", ctx.clone(), None, None).await?; - test_parity("data.path", ctx.clone(), None, None).await?; - test_parity("data.newlines.split('\\n').length", ctx.clone(), None, None).await?; - test_parity("data.unicode", ctx.clone(), None, None).await?; - test_parity("data.empty.length", ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_boolean_coercion_edge_cases() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - ctx.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - ctx.insert("emptyString".to_string(), Arc::new(to_raw_value(&json!("")))); - ctx.insert("nullVal".to_string(), Arc::new(to_raw_value(&json!(null)))); - ctx.insert("falseVal".to_string(), Arc::new(to_raw_value(&json!(false)))); - ctx.insert("emptyArr".to_string(), Arc::new(to_raw_value(&json!([])))); - ctx.insert("emptyObj".to_string(), Arc::new(to_raw_value(&json!({})))); - - // Truthy/falsy checks - test_parity("!!zero", ctx.clone(), None, None).await?; - test_parity("!!emptyString", ctx.clone(), None, None).await?; - test_parity("!!nullVal", ctx.clone(), None, None).await?; - test_parity("!!falseVal", ctx.clone(), None, None).await?; - test_parity("!!emptyArr", ctx.clone(), None, None).await?; // [] is truthy! - test_parity("!!emptyObj", ctx.clone(), None, None).await?; // {} is truthy! - - // Logical operators with falsy values - test_parity("zero || 'default'", ctx.clone(), None, None).await?; - test_parity("zero ?? 'default'", ctx.clone(), None, None).await?; // 0 is not nullish - test_parity("nullVal ?? 'default'", ctx.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // PREVIOUS_RESULT SPECIAL HANDLING - // ========================================================================= - - #[tokio::test] - async fn parity_previous_result_access() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - test_parity("previous_result", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result.length", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result[4].key", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - // ========================================================================= - // REAL-WORLD FLOW SCENARIOS - // ========================================================================= - - #[tokio::test] - async fn parity_scenario_api_pagination() -> anyhow::Result<()> { - // Simulate a flow that fetches paginated data - let mut ctx = HashMap::new(); - ctx.insert( - "fetch_result".to_string(), - Arc::new(to_raw_value(&json!({ - "items": [{"id": 1}, {"id": 2}, {"id": 3}], - "nextCursor": "abc123", - "hasMore": true - }))), - ); - ctx.insert("previous_result".to_string(), Arc::new(to_raw_value(&json!({ - "items": [{"id": 1}, {"id": 2}, {"id": 3}], - "nextCursor": "abc123", - "hasMore": true - })))); - - // Iterator for next page - test_parity( - "previous_result.hasMore ? [previous_result.nextCursor] : []", - ctx.clone(), None, None - ).await?; - - // Accumulating results - test_parity( - "fetch_result.items", - ctx.clone(), None, None - ).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_data_transformation_pipeline() -> anyhow::Result<()> { - let mut ctx = HashMap::new(); - - // Step 1: Raw data - ctx.insert( - "raw_data".to_string(), - Arc::new(to_raw_value(&json!({ - "records": [ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-16", "amount": 50, "type": "debit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"}, - {"date": "2024-01-18", "amount": 75, "type": "debit"} - ] - }))), - ); - - // Step 2: Filter credits - ctx.insert( - "credits".to_string(), - Arc::new(to_raw_value(&json!([ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"} - ]))), - ); - - ctx.insert("previous_result".to_string(), Arc::new(to_raw_value(&json!([ - {"date": "2024-01-15", "amount": 100, "type": "credit"}, - {"date": "2024-01-17", "amount": 200, "type": "credit"} - ])))); - - // Filter expression - test_parity( - "raw_data.records.filter(r => r.type === 'credit')", - ctx.clone(), None, None - ).await?; - - // Sum expression - test_parity( - "credits.reduce((sum, r) => sum + r.amount, 0)", - ctx.clone(), None, None - ).await?; - - // Summary - test_parity(r#" - ({ - totalCredits: credits.reduce((sum, r) => sum + r.amount, 0), - count: credits.length, - average: credits.reduce((sum, r) => sum + r.amount, 0) / credits.length - }) - "#, ctx.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_scenario_conditional_workflow() -> anyhow::Result<()> { - // Simulate a workflow with conditional logic - let mut ctx = HashMap::new(); - ctx.insert( - "check_result".to_string(), - Arc::new(to_raw_value(&json!({ - "passed": true, - "score": 95 - }))), - ); - ctx.insert( - "user_data".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "test_user", - "level": "admin" - }))), - ); - - // Branch condition - test_parity( - "check_result.passed && check_result.score > 90", - ctx.clone(), None, None - ).await?; - - // Skip condition - test_parity( - "!check_result.passed || check_result.score < 50", - ctx.clone(), None, None - ).await?; - - // Decision logic - test_parity( - "check_result.passed && user_data.level === 'admin' ? 'approved' : 'pending'", - ctx.clone(), None, None - ).await?; - - Ok(()) - } - - // ========================================================================= - // COMPREHENSIVE OPTIONAL CHAINING TESTS - // ========================================================================= - - #[tokio::test] - async fn parity_optional_chaining_method_calls() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "data": { - "items": [1, 2, 3], - "name": "test" - } - }))), - ); - env.insert("nullObj".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert("undefinedField".to_string(), Arc::new(to_raw_value(&serde_json::Value::Null))); - - // Optional chaining on method calls - test_parity("obj?.data?.items?.map(x => x * 2)", env.clone(), None, None).await?; - test_parity("obj?.data?.items?.filter(x => x > 1)", env.clone(), None, None).await?; - test_parity("obj?.data?.items?.join(',')", env.clone(), None, None).await?; - test_parity("obj?.data?.name?.toUpperCase()", env.clone(), None, None).await?; - test_parity("obj?.data?.name?.split('')", env.clone(), None, None).await?; - - // Optional method calls on null/undefined - test_parity("nullObj?.items?.map(x => x)", env.clone(), None, None).await?; - test_parity("obj?.missing?.method?.()", env.clone(), None, None).await?; - test_parity("undefinedField?.toString?.()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_computed_properties() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "users": { - "user1": {"name": "Alice", "age": 30}, - "user2": {"name": "Bob", "age": 25} - }, - "items": ["a", "b", "c"] - }))), - ); - env.insert("key".to_string(), Arc::new(to_raw_value(&json!("user1")))); - env.insert("index".to_string(), Arc::new(to_raw_value(&json!(1)))); - env.insert("nullData".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining with computed property access - test_parity("data?.users?.[key]", env.clone(), None, None).await?; - test_parity("data?.users?.[key]?.name", env.clone(), None, None).await?; - test_parity("data?.items?.[index]", env.clone(), None, None).await?; - test_parity("data?.users?.['user2']?.age", env.clone(), None, None).await?; - - // Computed access with null/undefined - test_parity("nullData?.users?.[key]", env.clone(), None, None).await?; - test_parity("data?.missing?.[key]", env.clone(), None, None).await?; - test_parity("data?.users?.['nonexistent']?.name", env.clone(), None, None).await?; - - // Dynamic key access - test_parity("data?.users?.[`user${index + 1}`]?.name", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_function_calls() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "config".to_string(), - Arc::new(to_raw_value(&json!({ - "callback": null, - "formatter": null, - "value": 42 - }))), - ); - env.insert("nullConfig".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional function call syntax - test_parity("config?.callback?.()", env.clone(), None, None).await?; - test_parity("config?.formatter?.('test')", env.clone(), None, None).await?; - test_parity("nullConfig?.callback?.()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_deep_nesting() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "response".to_string(), - Arc::new(to_raw_value(&json!({ - "data": { - "result": { - "items": [ - { - "details": { - "metadata": { - "tags": ["tag1", "tag2"] - } - } - } - ] - } - } - }))), - ); - env.insert("emptyResponse".to_string(), Arc::new(to_raw_value(&json!({})))); - - // Deep optional chaining - test_parity("response?.data?.result?.items?.[0]?.details?.metadata?.tags", env.clone(), None, None).await?; - test_parity("response?.data?.result?.items?.[0]?.details?.metadata?.tags?.[0]", env.clone(), None, None).await?; - test_parity("response?.data?.result?.items?.[1]?.details?.metadata?.tags", env.clone(), None, None).await?; - - // Deep chaining with missing intermediate - test_parity("emptyResponse?.data?.result?.items?.[0]", env.clone(), None, None).await?; - test_parity("response?.data?.missing?.items?.[0]?.details", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_with_operators() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "user".to_string(), - Arc::new(to_raw_value(&json!({ - "profile": { - "settings": { - "theme": "dark", - "notifications": true - } - }, - "scores": [85, 90, 78] - }))), - ); - env.insert("nullUser".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining with nullish coalescing - test_parity("user?.profile?.settings?.theme ?? 'light'", env.clone(), None, None).await?; - test_parity("user?.profile?.settings?.language ?? 'en'", env.clone(), None, None).await?; - test_parity("nullUser?.profile?.theme ?? 'default'", env.clone(), None, None).await?; - - // Optional chaining with logical OR - test_parity("user?.profile?.settings?.disabled || false", env.clone(), None, None).await?; - test_parity("user?.name || 'Anonymous'", env.clone(), None, None).await?; - - // Optional chaining with logical AND - test_parity("user?.profile?.settings?.notifications && 'enabled'", env.clone(), None, None).await?; - - // Optional chaining in ternary - test_parity("user?.profile?.settings?.theme === 'dark' ? 'Dark Mode' : 'Light Mode'", env.clone(), None, None).await?; - test_parity("nullUser?.active ? 'yes' : 'no'", env.clone(), None, None).await?; - - // Optional chaining with arithmetic - test_parity("(user?.scores?.[0] ?? 0) + 10", env.clone(), None, None).await?; - test_parity("user?.scores?.length ?? 0", env.clone(), None, None).await?; - - // Optional chaining with comparison - test_parity("user?.scores?.[0] > 80", env.clone(), None, None).await?; - test_parity("nullUser?.scores?.[0] > 80", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_with_array_methods() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "users": [ - {"id": 1, "name": "Alice", "active": true}, - {"id": 2, "name": "Bob", "active": false}, - {"id": 3, "name": "Charlie", "active": true} - ] - }))), - ); - env.insert("emptyData".to_string(), Arc::new(to_raw_value(&json!({})))); - - // Optional chaining before array methods - test_parity("data?.users?.filter(u => u.active)", env.clone(), None, None).await?; - test_parity("data?.users?.map(u => u.name)", env.clone(), None, None).await?; - test_parity("data?.users?.find(u => u.id === 2)?.name", env.clone(), None, None).await?; - test_parity("data?.users?.findIndex(u => u.id === 2)", env.clone(), None, None).await?; - test_parity("data?.users?.some(u => u.active)", env.clone(), None, None).await?; - test_parity("data?.users?.every(u => u.active)", env.clone(), None, None).await?; - test_parity("data?.users?.reduce((acc, u) => acc + (u.active ? 1 : 0), 0)", env.clone(), None, None).await?; - - // Optional chaining on missing arrays - test_parity("emptyData?.users?.filter(u => u.active)", env.clone(), None, None).await?; - test_parity("data?.items?.map(i => i.value)", env.clone(), None, None).await?; - - // Chained optional access on array results - test_parity("data?.users?.filter(u => u.active)?.[0]?.name", env.clone(), None, None).await?; - test_parity("data?.users?.filter(u => u.id > 10)?.[0]?.name ?? 'Not found'", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_in_template_literals() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "person".to_string(), - Arc::new(to_raw_value(&json!({ - "firstName": "John", - "lastName": "Doe", - "address": { - "city": "NYC", - "country": "USA" - } - }))), - ); - env.insert("nullPerson".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Template literals with optional chaining - test_parity("`Hello, ${person?.firstName ?? 'Guest'}!`", env.clone(), None, None).await?; - test_parity("`${person?.firstName} ${person?.lastName}`", env.clone(), None, None).await?; - test_parity("`Location: ${person?.address?.city ?? 'Unknown'}, ${person?.address?.country ?? 'Unknown'}`", env.clone(), None, None).await?; - test_parity("`User: ${nullPerson?.name ?? 'Anonymous'}`", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_flow_context() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Optional chaining on step results - test_parity("a?.toString()", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b?.data?.users?.[0]?.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b?.data?.users?.find(u => u.id === 999)?.name", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("b?.data?.users?.find(u => u.id === 999)?.name ?? 'Not found'", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Optional chaining on missing nested properties (step 'b' exists but nested path may not) - test_parity("b?.missing?.nested?.value ?? 'default'", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Optional chaining on flow_input - test_parity("flow_input?.limit ?? 100", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("flow_input?.missing?.nested?.value ?? 'fallback'", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Optional chaining on previous_result - test_parity("previous_result?.items?.[0]", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("previous_result?.missing ?? []", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_edge_cases() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - env.insert("emptyStr".to_string(), Arc::new(to_raw_value(&json!("")))); - env.insert("falseVal".to_string(), Arc::new(to_raw_value(&json!(false)))); - env.insert("nullVal".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert( - "nested".to_string(), - Arc::new(to_raw_value(&json!({ - "zero": 0, - "empty": "", - "false": false, - "null": null, - "obj": {} - }))), - ); - - // Optional chaining preserves falsy values (except null/undefined) - test_parity("zero?.toString()", env.clone(), None, None).await?; - test_parity("emptyStr?.length", env.clone(), None, None).await?; - test_parity("falseVal?.toString()", env.clone(), None, None).await?; - test_parity("nullVal?.toString()", env.clone(), None, None).await?; - - // Difference between ?. and && - test_parity("nested?.zero", env.clone(), None, None).await?; - test_parity("nested?.empty", env.clone(), None, None).await?; - test_parity("nested?.false", env.clone(), None, None).await?; - test_parity("nested?.null", env.clone(), None, None).await?; - test_parity("nested?.null?.value", env.clone(), None, None).await?; - - // Empty object access - test_parity("nested?.obj?.missing", env.clone(), None, None).await?; - test_parity("nested?.obj?.missing ?? 'not there'", env.clone(), None, None).await?; - - // Chaining after primitives (should return undefined) - test_parity("nested?.zero?.value", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_optional_chaining_typeof() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "a": {"b": {"c": 1}} - }))), - ); - env.insert("nullObj".to_string(), Arc::new(to_raw_value(&json!(null)))); - - // Optional chaining in expressions that return values - test_parity("obj?.a?.b?.c", env.clone(), None, None).await?; - test_parity("obj?.x?.y?.z", env.clone(), None, None).await?; - test_parity("nullObj?.a?.b?.c", env.clone(), None, None).await?; - - // Check that optional chaining works with typeof - test_parity("typeof obj?.a?.b?.c", env.clone(), None, None).await?; - test_parity("typeof obj?.missing?.value", env.clone(), None, None).await?; - test_parity("typeof nullObj?.value", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // EDGE CASE TESTS: Potential breaking changes between engines - // ========================================================================= - - #[tokio::test] - async fn parity_large_integers() -> anyhow::Result<()> { - let mut env = HashMap::new(); - - // i32 boundary values - env.insert( - "i32_max".to_string(), - Arc::new(to_raw_value(&json!(2147483647))), // i32::MAX - ); - env.insert( - "i32_max_plus_1".to_string(), - Arc::new(to_raw_value(&json!(2147483648_i64))), // i32::MAX + 1 - ); - env.insert( - "i32_min".to_string(), - Arc::new(to_raw_value(&json!(-2147483648))), // i32::MIN - ); - env.insert( - "i32_min_minus_1".to_string(), - Arc::new(to_raw_value(&json!(-2147483649_i64))), // i32::MIN - 1 - ); - - // Typical timestamp (milliseconds since epoch) - env.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1704067200000_i64))), // Jan 1, 2024 - ); - - // Near MAX_SAFE_INTEGER - env.insert( - "large_safe".to_string(), - Arc::new(to_raw_value(&json!(9007199254740991_i64))), // MAX_SAFE_INTEGER - ); - - // Basic operations with i32 boundary values - test_parity("i32_max", env.clone(), None, None).await?; - test_parity("i32_max + 1", env.clone(), None, None).await?; - test_parity("i32_max_plus_1", env.clone(), None, None).await?; - test_parity("i32_max_plus_1 + 1", env.clone(), None, None).await?; - test_parity("i32_min", env.clone(), None, None).await?; - test_parity("i32_min - 1", env.clone(), None, None).await?; - test_parity("i32_min_minus_1", env.clone(), None, None).await?; - - // Timestamp arithmetic - test_parity("timestamp", env.clone(), None, None).await?; - test_parity("timestamp + 86400000", env.clone(), None, None).await?; // +1 day - test_parity("timestamp - 3600000", env.clone(), None, None).await?; // -1 hour - - // MAX_SAFE_INTEGER operations - test_parity("large_safe", env.clone(), None, None).await?; - test_parity("large_safe - 1", env.clone(), None, None).await?; - - // Comparisons at boundaries - test_parity("i32_max === 2147483647", env.clone(), None, None).await?; - test_parity("i32_max_plus_1 === 2147483648", env.clone(), None, None).await?; - test_parity("timestamp > 1704067200000", env.clone(), None, None).await?; - test_parity("timestamp === 1704067200000", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_sparse_arrays() -> anyhow::Result<()> { - let mut env = HashMap::new(); - - // Sparse arrays are tricky - we'll simulate them via expressions - // Note: JSON doesn't support sparse arrays directly, so we test via JS - - // Regular array for comparison - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Test array operations that might behave differently with holes - test_parity("[1, 2, 3].length", env.clone(), None, None).await?; - test_parity("[1, 2, 3].map(x => x * 2)", env.clone(), None, None).await?; - test_parity("[1, 2, 3].filter(x => x > 1)", env.clone(), None, None).await?; - test_parity("[1, 2, 3].reduce((a, b) => a + b, 0)", env.clone(), None, None).await?; - - // Array with undefined values (different from holes) - test_parity("[1, undefined, 3].map(x => x ?? 'missing')", env.clone(), None, None).await?; - test_parity("[1, null, 3].map(x => x ?? 'missing')", env.clone(), None, None).await?; - - // Array.from behavior - test_parity("Array.from([1, 2, 3])", env.clone(), None, None).await?; - test_parity("Array.from({length: 3}, (_, i) => i)", env.clone(), None, None).await?; - - // Spread operator - test_parity("[...arr]", env.clone(), None, None).await?; - test_parity("[...arr, 6, 7]", env.clone(), None, None).await?; - test_parity("[0, ...arr]", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_unicode_and_emoji() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "emoji".to_string(), - Arc::new(to_raw_value(&json!("🎉"))), - ); - env.insert( - "text_with_emoji".to_string(), - Arc::new(to_raw_value(&json!("Hello 🌍 World!"))), - ); - env.insert( - "cafe".to_string(), - Arc::new(to_raw_value(&json!("café"))), - ); - env.insert( - "chinese".to_string(), - Arc::new(to_raw_value(&json!("你好世界"))), - ); - env.insert( - "mixed".to_string(), - Arc::new(to_raw_value(&json!("Hello 世界 🌍"))), - ); - - // String length (surrogate pairs count as 2) - test_parity("emoji.length", env.clone(), None, None).await?; - test_parity("text_with_emoji.length", env.clone(), None, None).await?; - test_parity("cafe.length", env.clone(), None, None).await?; - test_parity("chinese.length", env.clone(), None, None).await?; - - // String operations - test_parity("emoji.charCodeAt(0)", env.clone(), None, None).await?; - test_parity("text_with_emoji.indexOf('🌍')", env.clone(), None, None).await?; - test_parity("text_with_emoji.includes('🌍')", env.clone(), None, None).await?; - - // Substring operations - test_parity("text_with_emoji.substring(0, 5)", env.clone(), None, None).await?; - test_parity("text_with_emoji.slice(-1)", env.clone(), None, None).await?; - - // String comparison - test_parity("cafe === 'café'", env.clone(), None, None).await?; - test_parity("'café' === 'café'", env.clone(), None, None).await?; - - // Template literals with unicode - test_parity("`Hello ${emoji}`", env.clone(), None, None).await?; - test_parity("`${chinese} - ${emoji}`", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_special_numeric_values() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("zero".to_string(), Arc::new(to_raw_value(&json!(0)))); - env.insert( - "negative_zero_str".to_string(), - Arc::new(to_raw_value(&json!("-0"))), - ); - env.insert( - "num".to_string(), - Arc::new(to_raw_value(&json!(42))), - ); - - // Basic numeric operations - test_parity("0 === 0", env.clone(), None, None).await?; - test_parity("-0 === 0", env.clone(), None, None).await?; - test_parity("zero === 0", env.clone(), None, None).await?; - - // Division by zero - test_parity("1 / 0", env.clone(), None, None).await?; // Infinity -> null in JSON - test_parity("-1 / 0", env.clone(), None, None).await?; // -Infinity -> null in JSON - test_parity("0 / 0", env.clone(), None, None).await?; // NaN -> null in JSON - - // NaN checks - test_parity("Number.isNaN(0 / 0)", env.clone(), None, None).await?; - test_parity("Number.isFinite(1 / 0)", env.clone(), None, None).await?; - test_parity("Number.isFinite(num)", env.clone(), None, None).await?; - - // Safe integer checks - test_parity("Number.isSafeInteger(42)", env.clone(), None, None).await?; - test_parity("Number.isSafeInteger(9007199254740991)", env.clone(), None, None).await?; - test_parity("Number.isSafeInteger(9007199254740992)", env.clone(), None, None).await?; - - // Number parsing - test_parity("parseInt('42')", env.clone(), None, None).await?; - test_parity("parseFloat('3.14')", env.clone(), None, None).await?; - test_parity("parseInt('0xff', 16)", env.clone(), None, None).await?; - test_parity("parseInt('101', 2)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_object_property_order() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "z": 3, - "a": 1, - "m": 2, - "b": 4 - }))), - ); - env.insert( - "nested".to_string(), - Arc::new(to_raw_value(&json!({ - "outer": { - "z": 1, - "a": 2 - } - }))), - ); - - // Object.keys, Object.values, Object.entries - // Note: Order might differ but we compare as sets - test_parity("Object.keys(obj).sort()", env.clone(), None, None).await?; - test_parity("Object.values(obj).sort((a, b) => a - b)", env.clone(), None, None).await?; - test_parity("Object.entries(obj).sort((a, b) => a[0].localeCompare(b[0]))", env.clone(), None, None).await?; - - // Object spread (order might differ) - test_parity("{...obj, extra: 5}", env.clone(), None, None).await?; - test_parity("{first: 0, ...obj}", env.clone(), None, None).await?; - - // Nested object access - test_parity("nested.outer.z", env.clone(), None, None).await?; - test_parity("Object.keys(nested.outer).sort()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_prototype_methods() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("hello world"))), - ); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), - ); - - // Array methods - test_parity("arr.includes(3)", env.clone(), None, None).await?; - test_parity("arr.indexOf(3)", env.clone(), None, None).await?; - test_parity("arr.lastIndexOf(3)", env.clone(), None, None).await?; - test_parity("arr.find(x => x > 3)", env.clone(), None, None).await?; - test_parity("arr.findIndex(x => x > 3)", env.clone(), None, None).await?; - test_parity("arr.every(x => x > 0)", env.clone(), None, None).await?; - test_parity("arr.some(x => x > 4)", env.clone(), None, None).await?; - test_parity("arr.flat()", env.clone(), None, None).await?; - test_parity("arr.flatMap(x => [x, x * 2])", env.clone(), None, None).await?; - test_parity("arr.fill(0, 1, 3)", env.clone(), None, None).await?; - test_parity("[...arr].reverse()", env.clone(), None, None).await?; - - // String methods - test_parity("str.split(' ')", env.clone(), None, None).await?; - test_parity("str.toUpperCase()", env.clone(), None, None).await?; - test_parity("str.toLowerCase()", env.clone(), None, None).await?; - test_parity("str.trim()", env.clone(), None, None).await?; - test_parity("str.padStart(15, '_')", env.clone(), None, None).await?; - test_parity("str.padEnd(15, '_')", env.clone(), None, None).await?; - test_parity("str.startsWith('hello')", env.clone(), None, None).await?; - test_parity("str.endsWith('world')", env.clone(), None, None).await?; - test_parity("str.repeat(2)", env.clone(), None, None).await?; - - // Object methods - test_parity("Object.keys(obj)", env.clone(), None, None).await?; - test_parity("Object.values(obj)", env.clone(), None, None).await?; - test_parity("Object.entries(obj)", env.clone(), None, None).await?; - test_parity("Object.assign({}, obj, {c: 3})", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_basic() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("Hello123World456"))), - ); - env.insert( - "email".to_string(), - Arc::new(to_raw_value(&json!("test@example.com"))), - ); - - // Basic regex matching - test_parity("/\\d+/.test(text)", env.clone(), None, None).await?; - test_parity("text.match(/\\d+/)", env.clone(), None, None).await?; - test_parity("text.match(/\\d+/g)", env.clone(), None, None).await?; - - // Replace with regex - test_parity("text.replace(/\\d+/, 'X')", env.clone(), None, None).await?; - test_parity("text.replace(/\\d+/g, 'X')", env.clone(), None, None).await?; - - // Split with regex - test_parity("text.split(/\\d+/)", env.clone(), None, None).await?; - - // Case insensitive - test_parity("/hello/i.test(text)", env.clone(), None, None).await?; - test_parity("text.match(/hello/i)", env.clone(), None, None).await?; - - // Email validation (basic pattern) - test_parity("/^[^@]+@[^@]+\\.[^@]+$/.test(email)", env.clone(), None, None).await?; - - // Capturing groups (basic) - test_parity("text.match(/(\\d+)/)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_date_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "timestamp".to_string(), - Arc::new(to_raw_value(&json!(1704067200000_i64))), // 2024-01-01T00:00:00Z - ); - env.insert( - "iso_date".to_string(), - Arc::new(to_raw_value(&json!("2024-01-15T12:30:00.000Z"))), - ); - - // Date parsing - test_parity("Date.parse(iso_date)", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getTime()", env.clone(), None, None).await?; - test_parity("new Date(timestamp).toISOString()", env.clone(), None, None).await?; - - // UTC methods (timezone-independent) - test_parity("new Date(iso_date).getUTCFullYear()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCMonth()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCDate()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCHours()", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getUTCMinutes()", env.clone(), None, None).await?; - - // Date arithmetic - test_parity("new Date(timestamp + 86400000).toISOString()", env.clone(), None, None).await?; - test_parity("new Date(timestamp - 3600000).toISOString()", env.clone(), None, None).await?; - - // Date comparison - test_parity("new Date(iso_date).getTime() > 0", env.clone(), None, None).await?; - test_parity("new Date(iso_date).getTime() === Date.parse(iso_date)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_error_handling() -> anyhow::Result<()> { - let (ctx, fi, fe) = create_multi_step_flow_context(); - - // Try-catch with safe access (using step 'b' which has data) - test_parity( - "(() => { try { return b.data.total; } catch(e) { return 'error'; } })()", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Error from invalid JSON parse (should be caught) - test_parity( - "(() => { try { return JSON.parse('invalid'); } catch(e) { return 'parse_error'; } })()", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - // Typeof for error prevention - test_parity("typeof b.missing === 'undefined'", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("typeof b.data.total === 'number'", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Ternary with type checks - test_parity( - "typeof b === 'object' && b !== null ? b.data.total : 'fallback'", - ctx.clone(), - fi.clone(), - fe.clone(), - ) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_set_and_map() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 2, 3, 3, 3]))), - ); - env.insert( - "pairs".to_string(), - Arc::new(to_raw_value(&json!([["a", 1], ["b", 2], ["c", 3]]))), - ); - - // Set operations (converted back to array for JSON) - test_parity("[...new Set(arr)]", env.clone(), None, None).await?; - test_parity("new Set(arr).size", env.clone(), None, None).await?; - test_parity("new Set(arr).has(2)", env.clone(), None, None).await?; - test_parity("new Set(arr).has(5)", env.clone(), None, None).await?; - - // Map operations (converted back for JSON) - test_parity("new Map(pairs).get('a')", env.clone(), None, None).await?; - test_parity("new Map(pairs).has('b')", env.clone(), None, None).await?; - test_parity("new Map(pairs).size", env.clone(), None, None).await?; - test_parity("[...new Map(pairs).keys()]", env.clone(), None, None).await?; - test_parity("[...new Map(pairs).values()]", env.clone(), None, None).await?; - test_parity("[...new Map(pairs).entries()]", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_computed_property_names() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "key".to_string(), - Arc::new(to_raw_value(&json!("dynamicKey"))), - ); - env.insert( - "prefix".to_string(), - Arc::new(to_raw_value(&json!("item"))), - ); - env.insert( - "index".to_string(), - Arc::new(to_raw_value(&json!(42))), - ); - - // Computed property access - test_parity("({a: 1, b: 2})[key] ?? 'missing'", env.clone(), None, None).await?; - test_parity("({dynamicKey: 'found'})[key]", env.clone(), None, None).await?; - - // Computed property creation - test_parity("({[key]: 'value'})", env.clone(), None, None).await?; - test_parity("({[prefix + '_' + index]: true})", env.clone(), None, None).await?; - test_parity("({[`${prefix}_${index}`]: 'computed'})", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_destructuring_advanced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "data".to_string(), - Arc::new(to_raw_value(&json!({ - "user": {"name": "Alice", "age": 30}, - "items": [1, 2, 3, 4, 5], - "meta": {"count": 5} - }))), - ); - - // Nested destructuring - test_parity("(({user: {name}}) => name)(data)", env.clone(), None, None).await?; - test_parity("(({items: [first, second, ...rest]}) => ({first, second, rest}))(data)", env.clone(), None, None).await?; - - // Default values in destructuring - test_parity("(({missing = 'default'}) => missing)(data)", env.clone(), None, None).await?; - test_parity("(({user: {nickname = 'unknown'}}) => nickname)(data)", env.clone(), None, None).await?; - - // Renaming in destructuring - test_parity("(({user: u}) => u.name)(data)", env.clone(), None, None).await?; - test_parity("(({meta: {count: total}}) => total)(data)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_arrow_functions() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "numbers".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - env.insert( - "users".to_string(), - Arc::new(to_raw_value(&json!([ - {"name": "Alice", "score": 85}, - {"name": "Bob", "score": 92}, - {"name": "Charlie", "score": 78} - ]))), - ); - - // Simple arrow functions - test_parity("numbers.map(x => x * 2)", env.clone(), None, None).await?; - test_parity("numbers.filter(x => x > 2)", env.clone(), None, None).await?; - test_parity("numbers.reduce((a, b) => a + b, 0)", env.clone(), None, None).await?; - - // Arrow functions with objects - test_parity("users.map(u => u.name)", env.clone(), None, None).await?; - test_parity("users.filter(u => u.score >= 80)", env.clone(), None, None).await?; - test_parity("users.find(u => u.name === 'Bob')", env.clone(), None, None).await?; - - // Arrow functions returning objects (note the parentheses) - test_parity("numbers.map(x => ({value: x, doubled: x * 2}))", env.clone(), None, None).await?; - - // Chained arrow function calls - test_parity("numbers.filter(x => x > 1).map(x => x * 10)", env.clone(), None, None).await?; - test_parity("users.filter(u => u.score > 80).map(u => u.name)", env.clone(), None, None).await?; - - // Arrow function with multiple params - test_parity("numbers.reduce((sum, val) => sum + val, 0)", env.clone(), None, None).await?; - test_parity("numbers.map((val, idx) => ({index: idx, value: val}))", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_type_coercion() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("str_num".to_string(), Arc::new(to_raw_value(&json!("42")))); - env.insert("num".to_string(), Arc::new(to_raw_value(&json!(42)))); - env.insert("bool_true".to_string(), Arc::new(to_raw_value(&json!(true)))); - env.insert("bool_false".to_string(), Arc::new(to_raw_value(&json!(false)))); - env.insert("null_val".to_string(), Arc::new(to_raw_value(&json!(null)))); - env.insert("empty_str".to_string(), Arc::new(to_raw_value(&json!("")))); - env.insert("empty_arr".to_string(), Arc::new(to_raw_value(&json!([])))); - env.insert("empty_obj".to_string(), Arc::new(to_raw_value(&json!({})))); - - // String to number - test_parity("Number(str_num)", env.clone(), None, None).await?; - test_parity("+str_num", env.clone(), None, None).await?; - test_parity("parseInt(str_num)", env.clone(), None, None).await?; - - // Number to string - test_parity("String(num)", env.clone(), None, None).await?; - test_parity("num.toString()", env.clone(), None, None).await?; - test_parity("'' + num", env.clone(), None, None).await?; - - // Truthy/falsy checks - test_parity("!!str_num", env.clone(), None, None).await?; - test_parity("!!empty_str", env.clone(), None, None).await?; - test_parity("!!null_val", env.clone(), None, None).await?; - test_parity("!!empty_arr", env.clone(), None, None).await?; - test_parity("!!empty_obj", env.clone(), None, None).await?; - - // Boolean operations - test_parity("bool_true && 'yes'", env.clone(), None, None).await?; - test_parity("bool_false || 'no'", env.clone(), None, None).await?; - test_parity("null_val ?? 'default'", env.clone(), None, None).await?; - test_parity("empty_str || 'fallback'", env.clone(), None, None).await?; - test_parity("empty_str ?? 'wont_use'", env.clone(), None, None).await?; // empty string is not nullish - - // Array coercion - test_parity("Boolean(empty_arr)", env.clone(), None, None).await?; // empty array is truthy - test_parity("empty_arr.length || 'empty'", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_json_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "test", - "values": [1, 2, 3], - "nested": {"key": "value"} - }))), - ); - env.insert( - "json_str".to_string(), - Arc::new(to_raw_value(&json!(r#"{"parsed": true, "count": 42}"#))), - ); - - // JSON.stringify - test_parity("JSON.stringify(obj)", env.clone(), None, None).await?; - test_parity("JSON.stringify(obj, null, 2)", env.clone(), None, None).await?; - test_parity("JSON.stringify([1, 2, 3])", env.clone(), None, None).await?; - test_parity("JSON.stringify(null)", env.clone(), None, None).await?; - test_parity("JSON.stringify('string')", env.clone(), None, None).await?; - - // JSON.parse - test_parity("JSON.parse(json_str)", env.clone(), None, None).await?; - test_parity("JSON.parse(json_str).parsed", env.clone(), None, None).await?; - test_parity("JSON.parse(json_str).count", env.clone(), None, None).await?; - - // Round-trip - test_parity("JSON.parse(JSON.stringify(obj)).name", env.clone(), None, None).await?; - test_parity("JSON.parse(JSON.stringify(obj)).values", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_math_functions() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(16)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(-5.7)))); - env.insert("arr".to_string(), Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5, 9])))); - - // Basic Math functions - test_parity("Math.abs(y)", env.clone(), None, None).await?; - test_parity("Math.sqrt(x)", env.clone(), None, None).await?; - test_parity("Math.pow(2, 10)", env.clone(), None, None).await?; - test_parity("Math.floor(y)", env.clone(), None, None).await?; - test_parity("Math.ceil(y)", env.clone(), None, None).await?; - test_parity("Math.round(y)", env.clone(), None, None).await?; - test_parity("Math.trunc(y)", env.clone(), None, None).await?; - - // Min/Max - test_parity("Math.min(3, 1, 4)", env.clone(), None, None).await?; - test_parity("Math.max(3, 1, 4)", env.clone(), None, None).await?; - test_parity("Math.min(...arr)", env.clone(), None, None).await?; - test_parity("Math.max(...arr)", env.clone(), None, None).await?; - - // Trigonometric (with rounding to avoid precision issues) - test_parity("Math.round(Math.sin(0) * 1000) / 1000", env.clone(), None, None).await?; - test_parity("Math.round(Math.cos(0) * 1000) / 1000", env.clone(), None, None).await?; - - // Logarithmic - test_parity("Math.log(1)", env.clone(), None, None).await?; - test_parity("Math.log10(100)", env.clone(), None, None).await?; - test_parity("Math.log2(8)", env.clone(), None, None).await?; - - // Constants - test_parity("Math.round(Math.PI * 1000) / 1000", env.clone(), None, None).await?; - test_parity("Math.round(Math.E * 1000) / 1000", env.clone(), None, None).await?; - - // Sign and other - test_parity("Math.sign(-5)", env.clone(), None, None).await?; - test_parity("Math.sign(5)", env.clone(), None, None).await?; - test_parity("Math.sign(0)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_bitwise_operations() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("a".to_string(), Arc::new(to_raw_value(&json!(0b1010)))); - env.insert("b".to_string(), Arc::new(to_raw_value(&json!(0b1100)))); - env.insert("neg".to_string(), Arc::new(to_raw_value(&json!(-1)))); - - // Basic bitwise operations - test_parity("a & b", env.clone(), None, None).await?; - test_parity("a | b", env.clone(), None, None).await?; - test_parity("a ^ b", env.clone(), None, None).await?; - test_parity("~a", env.clone(), None, None).await?; - - // Shifts - test_parity("a << 2", env.clone(), None, None).await?; - test_parity("a >> 1", env.clone(), None, None).await?; - test_parity("neg >>> 0", env.clone(), None, None).await?; // unsigned right shift - - // Combined operations - test_parity("(a & b) | 1", env.clone(), None, None).await?; - test_parity("a ^ b ^ a", env.clone(), None, None).await?; // should equal b - - Ok(()) - } - - #[tokio::test] - async fn parity_array_slice_splice() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Slice (non-mutating) - test_parity("arr.slice()", env.clone(), None, None).await?; - test_parity("arr.slice(1)", env.clone(), None, None).await?; - test_parity("arr.slice(1, 3)", env.clone(), None, None).await?; - test_parity("arr.slice(-2)", env.clone(), None, None).await?; - test_parity("arr.slice(-3, -1)", env.clone(), None, None).await?; - test_parity("arr.slice(1, -1)", env.clone(), None, None).await?; - - // Concat (non-mutating) - test_parity("arr.concat([6, 7])", env.clone(), None, None).await?; - test_parity("arr.concat([6], [7, 8])", env.clone(), None, None).await?; - test_parity("[].concat(arr, [6])", env.clone(), None, None).await?; - - // Join - test_parity("arr.join()", env.clone(), None, None).await?; - test_parity("arr.join('-')", env.clone(), None, None).await?; - test_parity("arr.join('')", env.clone(), None, None).await?; - - // Copy and splice (to avoid mutating original) - test_parity("[...arr].splice(1, 2)", env.clone(), None, None).await?; - test_parity("(() => { const a = [...arr]; a.splice(1, 2, 'x'); return a; })()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_flow_error_extraction() -> anyhow::Result<()> { - // Test the specific error extraction logic used in flows - // Use existing context and add parallel results step 'h' - let (mut ctx, fi, fe) = create_multi_step_flow_context(); - - // Simulated parallel results with one error - add as step 'h' - ctx.insert( - "h".to_string(), - Arc::new(to_raw_value(&json!([ - {"success": true, "data": "result1"}, - {"error": {"message": "Something failed", "code": 500}}, - {"success": true, "data": "result3"} - ]))), - ); - - // Find error in step h's results - test_parity("h.find(r => r.error)?.error", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("h.filter(r => r.error).length", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("h.some(r => r.error)", ctx.clone(), fi.clone(), fe.clone()).await?; - test_parity("h.every(r => !r.error)", ctx.clone(), fi.clone(), fe.clone()).await?; - - // Extract all successful results - test_parity("h.filter(r => r.success).map(r => r.data)", ctx.clone(), fi.clone(), fe.clone()).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_string_template_complex() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "user".to_string(), - Arc::new(to_raw_value(&json!({ - "name": "Alice", - "email": "alice@example.com", - "score": 95.5 - }))), - ); - env.insert( - "items".to_string(), - Arc::new(to_raw_value(&json!(["apple", "banana", "cherry"]))), - ); - - // Nested expressions in templates - test_parity("`User: ${user.name} (${user.email})`", env.clone(), None, None).await?; - test_parity("`Score: ${user.score.toFixed(1)}`", env.clone(), None, None).await?; - test_parity("`Items: ${items.join(', ')}`", env.clone(), None, None).await?; - test_parity("`Count: ${items.length}`", env.clone(), None, None).await?; - - // Conditional in template - test_parity("`Status: ${user.score >= 90 ? 'A' : 'B'}`", env.clone(), None, None).await?; - - // Method calls in template - test_parity("`Upper: ${user.name.toUpperCase()}`", env.clone(), None, None).await?; - test_parity("`First item: ${items[0].charAt(0).toUpperCase() + items[0].slice(1)}`", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // ES2022+ METHOD AVAILABILITY TESTS - // These test methods that may not be available in QuickJS - // ========================================================================= - - #[tokio::test] - async fn parity_es2022_array_at() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.at() - ES2022 - test_parity("arr.at(0)", env.clone(), None, None).await?; - test_parity("arr.at(-1)", env.clone(), None, None).await?; - test_parity("arr.at(-2)", env.clone(), None, None).await?; - test_parity("arr.at(10)", env.clone(), None, None).await?; // out of bounds - - Ok(()) - } - - #[tokio::test] - async fn parity_es2022_string_at() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("hello"))), - ); - - // String.prototype.at() - ES2022 - test_parity("str.at(0)", env.clone(), None, None).await?; - test_parity("str.at(-1)", env.clone(), None, None).await?; - test_parity("str.at(-2)", env.clone(), None, None).await?; - test_parity("str.at(10)", env.clone(), None, None).await?; // out of bounds - - Ok(()) - } - - #[tokio::test] - async fn parity_es2022_object_hasown() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "obj".to_string(), - Arc::new(to_raw_value(&json!({"a": 1, "b": 2}))), - ); - - // Object.hasOwn() - ES2022 - test_parity("Object.hasOwn(obj, 'a')", env.clone(), None, None).await?; - test_parity("Object.hasOwn(obj, 'c')", env.clone(), None, None).await?; - test_parity("Object.hasOwn(obj, 'toString')", env.clone(), None, None).await?; // inherited - - Ok(()) - } - - #[tokio::test] - async fn parity_es2021_string_replaceall() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "str".to_string(), - Arc::new(to_raw_value(&json!("foo bar foo baz foo"))), - ); - - // String.prototype.replaceAll() - ES2021 - test_parity("str.replaceAll('foo', 'qux')", env.clone(), None, None).await?; - test_parity("str.replaceAll('x', 'y')", env.clone(), None, None).await?; // no match - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_findlast() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.findLast() - ES2023 - test_parity("arr.findLast(x => x > 2)", env.clone(), None, None).await?; - test_parity("arr.findLast(x => x > 10)", env.clone(), None, None).await?; // no match - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_findlastindex() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.findLastIndex() - ES2023 - test_parity("arr.findLastIndex(x => x > 2)", env.clone(), None, None).await?; - test_parity("arr.findLastIndex(x => x > 10)", env.clone(), None, None).await?; // no match - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_tosorted() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([3, 1, 4, 1, 5]))), - ); - - // Array.prototype.toSorted() - ES2023 (non-mutating sort) - test_parity("arr.toSorted()", env.clone(), None, None).await?; - test_parity("arr.toSorted((a, b) => b - a)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_toreversed() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.toReversed() - ES2023 (non-mutating reverse) - test_parity("arr.toReversed()", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_tospliced() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.toSpliced() - ES2023 (non-mutating splice) - test_parity("arr.toSpliced(1, 2)", env.clone(), None, None).await?; - test_parity("arr.toSpliced(1, 2, 'a', 'b')", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2023_array_with() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - // Array.prototype.with() - ES2023 (non-mutating index assignment) - test_parity("arr.with(2, 99)", env.clone(), None, None).await?; - test_parity("arr.with(-1, 99)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_es2024_object_groupby() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "items".to_string(), - Arc::new(to_raw_value(&json!([ - {"type": "fruit", "name": "apple"}, - {"type": "vegetable", "name": "carrot"}, - {"type": "fruit", "name": "banana"} - ]))), - ); - - // Object.groupBy() - ES2024 - test_parity("Object.groupBy(items, item => item.type)", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // REGEX FEATURE TESTS - Lookbehind and Named Groups - // These may not be available in QuickJS - // ========================================================================= - - #[tokio::test] - async fn parity_regex_lookbehind() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("price: $100, discount: $20"))), - ); - - // Lookbehind assertion - may not work in QuickJS - // This matches numbers that come after a $ - test_parity("text.match(/(?<=\\$)\\d+/g)", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_negative_lookbehind() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("foo123 bar456"))), - ); - - // Negative lookbehind - may not work in QuickJS - test_parity("text.match(/(? anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "date".to_string(), - Arc::new(to_raw_value(&json!("2024-01-15"))), - ); - - // Named capture groups - may not work in QuickJS - test_parity("/(?\\d{4})-(?\\d{2})-(?\\d{2})/.exec(date)?.groups?.year", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_regex_d_flag() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "text".to_string(), - Arc::new(to_raw_value(&json!("hello world"))), - ); - - // d flag (indices) - may not work in QuickJS - test_parity("/world/d.exec(text)?.indices?.[0]", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // BROWSER/RUNTIME API TESTS - These are likely NOT available in QuickJS - // ========================================================================= - - #[tokio::test] - async fn parity_atob_btoa() -> anyhow::Result<()> { - let env = HashMap::new(); - - // Base64 encoding/decoding - browser APIs, likely NOT in QuickJS - test_parity("typeof atob", env.clone(), None, None).await?; - test_parity("typeof btoa", env.clone(), None, None).await?; - // If available, test actual usage - test_parity("typeof btoa === 'function' ? btoa('hello') : 'not_available'", env.clone(), None, None).await?; - test_parity("typeof atob === 'function' ? atob('aGVsbG8=') : 'not_available'", env.clone(), None, None).await?; - - Ok(()) - } - - #[tokio::test] - async fn parity_text_encoder_decoder() -> anyhow::Result<()> { - let env = HashMap::new(); - - // TextEncoder/TextDecoder - browser/Node APIs - test_parity("typeof TextEncoder", env.clone(), None, None).await?; - test_parity("typeof TextDecoder", env.clone(), None, None).await?; - - Ok(()) - } - - // NOTE: This test is EXPECTED to fail - Intl is NOT available in QuickJS - // Deno Core: typeof Intl = "object" - // QuickJS: typeof Intl = "undefined" - // - // BREAKING CHANGE: Any expression using Intl.NumberFormat, Intl.DateTimeFormat, - // or other Intl APIs will fail in QuickJS. - // - // #[tokio::test] - // async fn parity_intl_apis() -> anyhow::Result<()> { - // // Intl APIs are NOT available in QuickJS - this test documents the breaking change - // // Deno Core: typeof Intl = "object" - // // QuickJS: typeof Intl = "undefined" - // } - - #[tokio::test] - async fn parity_url_apis() -> anyhow::Result<()> { - let env = HashMap::new(); - - // URL and URLSearchParams - browser/Node APIs - test_parity("typeof URL", env.clone(), None, None).await?; - test_parity("typeof URLSearchParams", env.clone(), None, None).await?; - - Ok(()) - } - - // ========================================================================= - // NOTE: Non-existent step access via results proxy cannot be tested in unit tests - // because the results proxy is only set up during actual flow execution (requires by_id context). - // See flow_engine_parity.rs for test_flow_results_non_existent_step which tests this behavior. - // - // IMPORTANT: Both Deno Core and QuickJS throw errors when accessing non-existent steps, - // even with optional chaining (results?.nonexistent). This is because: - // 1. results is a Proxy object (not null), so ?. doesn't short-circuit - // 2. The proxy's get handler triggers a backend lookup - // 3. The backend returns "Not found" error - // ========================================================================= -} diff --git a/backend/windmill-worker/src/js_eval_quickjs.rs b/backend/windmill-worker/src/js_eval_quickjs.rs deleted file mode 100644 index 2dc48c753a..0000000000 --- a/backend/windmill-worker/src/js_eval_quickjs.rs +++ /dev/null @@ -1,922 +0,0 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -//! QuickJS-based JavaScript expression evaluation for flow transformations. -//! -//! This module provides an alternative to deno_core for evaluating arbitrary JavaScript -//! expressions in flow transformations. QuickJS offers significantly faster startup times -//! (~200μs vs ~3ms for V8), making it ideal for evaluating many small expressions. -//! -//! ## Performance Characteristics (release mode benchmarks) -//! - **Simple expressions**: ~238μs (QuickJS) vs ~3.05ms (deno_core) = **~13x faster** -//! - **Complex expressions**: ~192μs (QuickJS) vs ~3.09ms (deno_core) = **~16x faster** -//! - **Memory**: ~2.5% of V8's footprint -//! -//! For flow expression evaluation where startup time dominates, QuickJS is significantly -//! faster overall despite being slower for long-running CPU-intensive code. -//! -//! ## Async Operations -//! This implementation uses true async Rust callbacks (similar to deno_core's ops) for -//! `variable()`, `resource()`, and `results.xxx` access. The async functions use -//! rquickjs's `Async>` wrapper which returns JavaScript Promises that are -//! resolved when the Rust async operations complete. No pre-fetching is required. - -use std::collections::HashMap; -use std::sync::Arc; - -use rquickjs::{ - async_with, - prelude::{Async, Func, MutFn}, - AsyncContext, AsyncRuntime, CatchResultExt, FromJs, IntoJs, Object, Value, -}; -use serde_json::value::RawValue; - -use windmill_common::client::AuthedClient; -use windmill_common::flow_status::JobResult; - -use crate::js_eval::{replace_with_await, replace_with_await_result, IdContext}; - -/// Shared state for async operations within QuickJS -#[derive(Clone)] -struct AsyncOpState { - client: AuthedClient, -} - -/// Evaluates a JavaScript expression using QuickJS runtime. -/// -/// This function provides the same interface as `eval_timeout` but uses QuickJS -/// instead of deno_core/V8 for potentially faster startup times. -/// -/// Unlike deno_core, this uses true async Rust callbacks for `variable()`, -/// `resource()`, and `results.xxx` access - no pre-fetching required. -pub async fn eval_timeout_quickjs( - expr: String, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option<&HashMap>>, - authed_client: Option<&AuthedClient>, - by_id: Option<&IdContext>, - ctx: Option>, -) -> anyhow::Result> { - let expr = expr.trim().to_string(); - - tracing::debug!( - "evaluating js eval (quickjs): {} with context {:?}", - expr, - transform_context - ); - - // Clone data for the blocking task - let by_id_clone = by_id.cloned(); - let flow_input_clone = flow_input.clone(); - let flow_env_clone = flow_env.cloned(); - let authed_client_clone = authed_client.cloned(); - - // Determine which context keys are actually used in the expression - let p_ids = by_id.map(|x| { - [ - format!("results.{}", x.previous_id), - format!("results?.{}", x.previous_id), - format!("results[\"{}\"]", x.previous_id), - format!("results?.[\"{}\"]", x.previous_id), - ] - }); - - let mut context_keys: Vec = transform_context - .keys() - .filter(|x| expr.contains(&x.to_string())) - .cloned() - .collect(); - - if !context_keys.contains(&"previous_result".to_string()) - && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) - || expr.contains("error") - { - context_keys.push("previous_result".to_string()); - } - - let has_flow_input = expr.contains("flow_input"); - if has_flow_input { - context_keys.push("flow_input".to_string()) - } - - // Filter transform_context to only include used keys - let filtered_context: HashMap>> = transform_context - .into_iter() - .filter(|(k, _)| context_keys.contains(k)) - .collect(); - - let expr_clone = expr.clone(); - - // Run the QuickJS evaluation with a timeout - tokio::time::timeout( - std::time::Duration::from_millis(10000), - tokio::task::spawn_blocking(move || { - // Create a new tokio runtime for async operations within the blocking context - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - rt.block_on(async move { - eval_quickjs_inner( - &expr_clone, - filtered_context, - flow_input_clone, - flow_env_clone, - authed_client_clone, - by_id_clone, - ctx, - context_keys, - ) - .await - }) - }), - ) - .await - .map_err(|_| { - anyhow::anyhow!( - "The expression evaluation `{expr}` took too long to execute (>10000ms)" - ) - })?? -} - -/// Memory limit for QuickJS runtime (32MB). -/// This is much smaller than deno_core's 128MB limit since flow expressions -/// should be lightweight transformations, not memory-intensive operations. -const QUICKJS_MEMORY_LIMIT: usize = 32 * 1024 * 1024; - -async fn eval_quickjs_inner( - expr: &str, - transform_context: HashMap>>, - flow_input: Option>>>, - flow_env: Option>>, - authed_client: Option, - by_id: Option, - extra_ctx: Option>, - context_keys: Vec, -) -> anyhow::Result> { - let runtime = AsyncRuntime::new()?; - runtime.set_memory_limit(QUICKJS_MEMORY_LIMIT).await; - let context = AsyncContext::full(&runtime).await?; - - // Create shared state for async ops if we have a client - let op_state = authed_client.map(|client| Arc::new(AsyncOpState { client })); - - let op_state_clone = op_state.clone(); - let by_id_clone = by_id.clone(); - - // Transform expression to add await for variable/resource/results access - let expr_with_funcs = ["variable", "resource"] - .into_iter() - .fold(expr.to_string(), replace_with_await); - let transformed_expr = replace_with_await_result(expr_with_funcs); - - async_with!(context => |ctx| { - let globals = ctx.globals(); - - // Set up context variables - for key in &context_keys { - if key == "flow_input" { - if let Some(ref fi) = flow_input { - let json_str = serde_json::to_string(fi.as_ref())?; - let val: serde_json::Value = serde_json::from_str(&json_str)?; - let js_val = json_to_js(&ctx, &val)?; - globals.set(key.as_str(), js_val)?; - } else { - globals.set(key.as_str(), Value::new_null(ctx.clone()))?; - } - } else if let Some(raw_val) = transform_context.get(key) { - let val: serde_json::Value = serde_json::from_str(raw_val.get())?; - let js_val = json_to_js(&ctx, &val)?; - globals.set(key.as_str(), js_val)?; - } - } - - // Set up flow_env if referenced - if expr.contains("flow_env") { - if let Some(ref fe) = flow_env { - let obj = Object::new(ctx.clone())?; - for (k, v) in fe { - let val: serde_json::Value = serde_json::from_str(v.get())?; - let js_val = json_to_js(&ctx, &val)?; - obj.set(k.as_str(), js_val)?; - } - globals.set("flow_env", obj)?; - } else { - globals.set("flow_env", Object::new(ctx.clone())?)?; - } - } - - // Set up additional context variables - if let Some(ctx_vars) = extra_ctx { - for (k, v) in ctx_vars { - globals.set(k.as_str(), v.as_str())?; - } - } - - // Set up error extraction if needed - if expr.contains("error") && context_keys.contains(&"previous_result".to_string()) { - let error_setup = r#" - let error = previous_result?.error; - if (!error) { - if (Array.isArray(previous_result)) { - const errors = previous_result.filter(item => item && typeof item === 'object' && 'error' in item); - if (errors.length === 1) { - error = errors[0].error; - } else if (errors.length > 1) { - error = { - name: 'MultipleErrors', - message: errors.map(({ error: e }, i) => `[${e.step_id || i}] ${e.message || e.name}`).join('; '), - errors: previous_result - }; - } else { - error = { - name: 'MultipleErrors', - message: "Could not parse errors", - errors: previous_result - }; - } - } else { - if (previous_result) { - error = { name: 'UnknownError', message: 'Could not parse the error', error: previous_result }; - } else { - error = { name: 'UnknownError', message: 'No error found' }; - } - } - } - "#; - ctx.eval::<(), _>(error_setup).catch(&ctx).map_err(quickjs_error_to_anyhow)?; - } - - // Set up async functions if we have a client - if let Some(ref state) = op_state_clone { - setup_async_ops(&ctx, &globals, state.clone())?; - } else { - // Set up stub functions that throw errors - setup_stub_functions(&ctx, &globals)?; - } - - // Set up results proxy if we have by_id context - if let Some(ref by_id) = by_id_clone { - setup_results_proxy(&ctx, &globals, by_id, op_state_clone.clone())?; - } - - // Determine if we need to add return statement. - // Wrap with .then((x) => JSON.stringify(x ?? null)) to serialize the result - // using the standard JSON.stringify, matching deno_core's behavior exactly. - let code = if should_add_return_quickjs(&transformed_expr) { - format!("(async function() {{ return {}; }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) - } else { - format!("(async function() {{ {} }})().then((x) => JSON.stringify(x ?? null))", transformed_expr) - }; - - // Evaluate the expression (returns a Promise that resolves to a JSON string) - let promise: rquickjs::Promise = ctx.eval(code).catch(&ctx).map_err(quickjs_error_to_anyhow)?; - - // Await the promise — result is already a JSON string from JSON.stringify - let result: Value = promise.into_future().await.catch(&ctx).map_err(quickjs_error_to_anyhow)?; - - let json_str = String::from_js(&ctx, result) - .unwrap_or_else(|_| "null".to_string()); - - Ok(crate::common::unsafe_raw(json_str)) - }) - .await -} - -/// Set up async variable() and resource() functions using true Rust async callbacks. -/// -/// This uses rquickjs's `Async>` wrapper to create JavaScript functions that -/// return Promises. The Promises are resolved by spawned Rust async operations. -fn setup_async_ops<'js>( - ctx: &rquickjs::Ctx<'js>, - globals: &Object<'js>, - state: Arc, -) -> anyhow::Result<()> { - // Error prefix - must match the JavaScript side - const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; - - // Create variable() function with true async Rust callback - // Returns a JSON string that JavaScript will parse - let state_for_var = state.clone(); - globals.set( - "__fetchVariable", - Func::from(Async(MutFn::new(move |path: String| { - let client = state_for_var.client.clone(); - async move { - match client.get_variable_value(&path).await { - Ok(value) => value, - Err(e) => format!("{}{}", ERR_PREFIX, e), - } - } - }))), - )?; - - // Create resource() function - returns JSON string - let state_for_res = state.clone(); - globals.set( - "__fetchResource", - Func::from(Async(MutFn::new(move |path: String| { - let client = state_for_res.client.clone(); - async move { - match client - .get_resource_value_interpolated::(&path, None) - .await - { - Ok(value) => serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()), - Err(e) => format!("{}{}", ERR_PREFIX, e), - } - } - }))), - )?; - - // Create JavaScript wrappers that parse the JSON results - // We use a unique prefix that's extremely unlikely to appear in real data - let wrapper_code = r#" - const __ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; - - async function variable(path) { - const result = await __fetchVariable(path); - if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { - throw new Error(result.substring(__ERR_PREFIX.length)); - } - return result; - } - - async function resource(path) { - const result = await __fetchResource(path); - if (typeof result === 'string' && result.startsWith(__ERR_PREFIX)) { - throw new Error(result.substring(__ERR_PREFIX.length)); - } - return JSON.parse(result); - } - "#; - - ctx.eval::<(), _>(wrapper_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - - Ok(()) -} - -/// Set up stub functions that throw errors when no client is available -fn setup_stub_functions<'js>( - ctx: &rquickjs::Ctx<'js>, - _globals: &Object<'js>, -) -> anyhow::Result<()> { - let setup_code = r#" - function variable(path) { - return Promise.reject(new Error(`variable() is not available without an authenticated client`)); - } - - function resource(path) { - return Promise.reject(new Error(`resource() is not available without an authenticated client`)); - } - "#; - - ctx.eval::<(), _>(setup_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - - Ok(()) -} - -/// Set up the `results` Proxy object with dynamic access to step results. -/// -/// Uses async Rust callbacks to fetch results on-demand when accessed. -fn setup_results_proxy<'js>( - ctx: &rquickjs::Ctx<'js>, - globals: &Object<'js>, - by_id: &IdContext, - op_state: Option>, -) -> anyhow::Result<()> { - // Store previous_id for the shortcut optimization - globals.set("__previous_id", by_id.previous_id.clone())?; - - // Create async __getResult function that fetches step results via Rust - if let Some(state) = op_state { - let by_id_for_result = by_id.clone(); - globals.set( - "__fetchResult", - Func::from(Async(MutFn::new(move |step_id: String| { - let client = state.client.clone(); - let by_id = by_id_for_result.clone(); - let step_id_clone = step_id.clone(); - - // Look up the job ID(s) for this step from the local cache - let job_result = by_id.steps_results.get(&step_id).cloned(); - let flow_job_id = by_id.flow_job.to_string(); - - async move { - const ERR_PREFIX: &str = "\x00__WINDMILL_ERR__\x00"; - - let result: Result = match job_result { - Some(jr) => { - // Found in local cache, fetch result by job ID - match jr { - JobResult::SingleJob(job_id) => { - client - .get_completed_job_result::(&job_id.to_string(), None) - .await - .map_err(|e| format!("Failed to fetch result for step '{}': {}", step_id_clone, e)) - } - JobResult::ListJob(job_ids) => { - let futs = job_ids.iter().map(|job_id| { - let client = client.clone(); - let job_id_str = job_id.to_string(); - async move { - client - .get_completed_job_result::(&job_id_str, None) - .await - } - }); - let results: Vec<_> = futures::future::join_all(futs).await; - let collected: Result, _> = results.into_iter().collect(); - collected - .map(serde_json::Value::Array) - .map_err(|e| format!("Failed to fetch results for step '{}': {}", step_id_clone, e)) - } - } - } - None => { - // Not in local cache, fallback to querying by flow_job_id and step_id - // This happens for branch modules that need to access parent flow step results - // Use .ok() to match deno_core behavior: return null for non-existent steps - // instead of throwing an error - Ok(client - .get_result_by_id::(&flow_job_id, &step_id_clone, None) - .await - .ok() // Swallow errors, convert to Option - .unwrap_or(serde_json::Value::Null)) // None -> null - } - }; - - match result { - Ok(value) => serde_json::to_string(&value).unwrap_or_else(|_| "null".to_string()), - Err(e) => format!("{}{}", ERR_PREFIX, e), - } - } - }))), - )?; - - // Create JavaScript wrapper that parses the JSON result - let wrapper_code = r#" - const __RESULT_ERR_PREFIX = '\x00__WINDMILL_ERR__\x00'; - async function __getResult(stepId) { - const result = await __fetchResult(stepId); - if (typeof result === 'string' && result.startsWith(__RESULT_ERR_PREFIX)) { - throw new Error(result.substring(__RESULT_ERR_PREFIX.length)); - } - return JSON.parse(result); - } - "#; - ctx.eval::<(), _>(wrapper_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - } else { - // No client - stub function that rejects - let stub_code = r#" - function __getResult(stepId) { - return Promise.reject(new Error('Result fetching not available without authenticated client')); - } - "#; - ctx.eval::<(), _>(stub_code) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - } - - // Create the results proxy that calls __getResult for on-demand fetching - // Matches deno_core behavior: always try to fetch, let backend handle unknown step IDs - let proxy_setup = r#" - const results = new Proxy({}, { - get: function(target, name, receiver) { - // Handle symbol properties (like Symbol.toStringTag) - if (typeof name === 'symbol') { - return undefined; - } - - // Check if it's the previous_id and previous_result exists - if (name === __previous_id && typeof previous_result !== 'undefined') { - return Promise.resolve(previous_result); - } - - // Always try to fetch - let Rust/backend handle unknown step IDs - // This matches deno_core behavior - return __getResult(name); - } - }); - "#; - ctx.eval::<(), _>(proxy_setup) - .catch(ctx) - .map_err(quickjs_error_to_anyhow)?; - - Ok(()) -} - -/// Convert a serde_json::Value to a QuickJS Value -fn json_to_js<'js>( - ctx: &rquickjs::Ctx<'js>, - val: &serde_json::Value, -) -> rquickjs::Result> { - match val { - serde_json::Value::Null => Ok(Value::new_null(ctx.clone())), - serde_json::Value::Bool(b) => Ok(Value::new_bool(ctx.clone(), *b)), - serde_json::Value::Number(n) => { - if let Some(i) = n.as_i64() { - if i >= i32::MIN as i64 && i <= i32::MAX as i64 { - Ok(Value::new_int(ctx.clone(), i as i32)) - } else { - Ok(Value::new_float(ctx.clone(), i as f64)) - } - } else if let Some(f) = n.as_f64() { - Ok(Value::new_float(ctx.clone(), f)) - } else { - Ok(Value::new_float(ctx.clone(), 0.0)) - } - } - serde_json::Value::String(s) => s.clone().into_js(ctx), - serde_json::Value::Array(arr) => { - let js_arr = rquickjs::Array::new(ctx.clone())?; - for (i, item) in arr.iter().enumerate() { - js_arr.set(i, json_to_js(ctx, item)?)?; - } - Ok(js_arr.into_value()) - } - serde_json::Value::Object(obj) => { - let js_obj = Object::new(ctx.clone())?; - for (k, v) in obj { - js_obj.set(k.as_str(), json_to_js(ctx, v)?)?; - } - Ok(js_obj.into_value()) - } - } -} - -/// Convert a QuickJS Value to a serde_json::Value -/// -/// This mimics JavaScript's JSON.stringify behavior: -/// - For objects with a `toJSON` method (like Date), call it and use the result -/// - Arrays are recursively serialized -/// - Plain objects enumerate their own properties - -/// Determines if we should prepend "return" to the expression -fn should_add_return_quickjs(expr: &str) -> bool { - let trimmed = expr.trim(); - - if trimmed.is_empty() { - return true; - } - - if trimmed.starts_with("return ") || trimmed.starts_with("return;") || trimmed == "return" { - return false; - } - - let statement_prefixes = [ - "const ", "let ", "var ", "if ", "if(", "for ", "for(", "while ", "while(", "switch ", - "switch(", "try ", "try{", "throw ", "function ", "class ", "async ", "await ", - ]; - - for prefix in &statement_prefixes { - if trimmed.starts_with(prefix) { - return false; - } - } - - if contains_semicolon_outside_strings(trimmed) { - return false; - } - - true -} - -fn contains_semicolon_outside_strings(expr: &str) -> bool { - let mut in_single_quote = false; - let mut in_double_quote = false; - let mut in_template = false; - let mut prev_char = '\0'; - - for ch in expr.chars() { - match ch { - '\'' if prev_char != '\\' && !in_double_quote && !in_template => { - in_single_quote = !in_single_quote; - } - '"' if prev_char != '\\' && !in_single_quote && !in_template => { - in_double_quote = !in_double_quote; - } - '`' if prev_char != '\\' && !in_single_quote && !in_double_quote => { - in_template = !in_template; - } - ';' if !in_single_quote && !in_double_quote && !in_template => { - return true; - } - _ => {} - } - prev_char = ch; - } - - false -} - -fn quickjs_error_to_anyhow(err: rquickjs::CaughtError<'_>) -> anyhow::Error { - anyhow::anyhow!("QuickJS evaluation error: {}", err) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use windmill_common::worker::to_raw_value; - - #[tokio::test] - async fn test_eval_quickjs_simple() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert("x".to_string(), Arc::new(to_raw_value(&json!(5)))); - env.insert("y".to_string(), Arc::new(to_raw_value(&json!(3)))); - - let result = - eval_timeout_quickjs("x + y".to_string(), env, None, None, None, None, None).await?; - - assert_eq!(result.get(), "8"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_object_access() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "params".to_string(), - Arc::new(to_raw_value(&json!({"test": 42, "nested": {"value": 100}}))), - ); - - let result = eval_timeout_quickjs( - "params.test".to_string(), - env.clone(), - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "42"); - - let result2 = eval_timeout_quickjs( - "params.nested.value".to_string(), - env, - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result2.get(), "100"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_array() -> anyhow::Result<()> { - let mut env = HashMap::new(); - env.insert( - "arr".to_string(), - Arc::new(to_raw_value(&json!([1, 2, 3, 4, 5]))), - ); - - let result = eval_timeout_quickjs( - "arr.map(x => x * 2)".to_string(), - env, - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "[2,4,6,8,10]"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_flow_input() -> anyhow::Result<()> { - let mut flow_input = HashMap::new(); - flow_input.insert("name".to_string(), to_raw_value(&json!("test"))); - flow_input.insert("count".to_string(), to_raw_value(&json!(10))); - - let result = eval_timeout_quickjs( - "flow_input.name".to_string(), - HashMap::new(), - Some(mappable_rc::Marc::new(flow_input)), - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "\"test\""); - Ok(()) - } - - #[test] - fn test_should_add_return_quickjs() { - assert!(should_add_return_quickjs("5")); - assert!(should_add_return_quickjs("x + y")); - assert!(should_add_return_quickjs("foo()")); - - assert!(!should_add_return_quickjs("return 5")); - assert!(!should_add_return_quickjs("return x + y")); - - assert!(!should_add_return_quickjs("const x = 5")); - assert!(!should_add_return_quickjs("let y = 10")); - assert!(!should_add_return_quickjs("if (x > 5) { return x; }")); - - assert!(!should_add_return_quickjs("let x = 5; x + 1")); - } - - #[tokio::test] - async fn test_eval_quickjs_date_serialization() -> anyhow::Result<()> { - // Test that Date objects serialize to ISO strings, matching JSON.stringify behavior - let result = eval_timeout_quickjs( - "new Date('2024-01-15T12:30:00.000Z')".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // Should be an ISO string, not an empty object - assert_eq!(result.get(), "\"2024-01-15T12:30:00.000Z\""); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_date_in_object() -> anyhow::Result<()> { - // Test that Date objects within other objects serialize correctly - let result = eval_timeout_quickjs( - "({ date: new Date('2024-01-15T12:30:00.000Z'), name: 'test' })".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!(value["date"], "2024-01-15T12:30:00.000Z"); - assert_eq!(value["name"], "test"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_date_in_array() -> anyhow::Result<()> { - // Test that Date objects in arrays serialize correctly - let result = eval_timeout_quickjs( - "[new Date('2024-01-15T00:00:00.000Z'), new Date('2024-01-16T00:00:00.000Z')]" - .to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!(value[0], "2024-01-15T00:00:00.000Z"); - assert_eq!(value[1], "2024-01-16T00:00:00.000Z"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_custom_tojson() -> anyhow::Result<()> { - // Test that JSON.stringify handles custom toJSON when returning objects - let result = eval_timeout_quickjs( - r#"({ a: 1, toJSON: () => ({ converted: true }) })"#.to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // JSON.stringify should call toJSON and use that result - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!(value["converted"], true); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_deeply_nested_date() -> anyhow::Result<()> { - // Test that Date objects deep in nested structures are handled - let result = eval_timeout_quickjs( - "({ level1: { level2: { date: new Date('2024-01-15T00:00:00.000Z') } } })".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - let value: serde_json::Value = serde_json::from_str(result.get())?; - assert_eq!( - value["level1"]["level2"]["date"], - "2024-01-15T00:00:00.000Z" - ); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_regexp_serialization() -> anyhow::Result<()> { - // RegExp objects serialize to empty objects in JSON (same as JSON.stringify behavior) - let result = eval_timeout_quickjs( - "/test/gi".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // RegExp doesn't have toJSON, so it serializes to an empty object (same as Deno) - assert_eq!(result.get(), "{}"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_map_serialization() -> anyhow::Result<()> { - // Map objects serialize to empty objects in JSON (same as JSON.stringify behavior) - let result = eval_timeout_quickjs( - "new Map([['key', 'value']])".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // Map doesn't have toJSON, serializes to empty object (same as Deno) - assert_eq!(result.get(), "{}"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_set_serialization() -> anyhow::Result<()> { - // Set objects serialize to empty objects in JSON (same as JSON.stringify behavior) - let result = eval_timeout_quickjs( - "new Set([1, 2, 3])".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - // Set doesn't have toJSON, serializes to empty object (same as Deno) - assert_eq!(result.get(), "{}"); - Ok(()) - } - - #[tokio::test] - async fn test_eval_quickjs_tojson_returning_date() -> anyhow::Result<()> { - // When toJSON returns a Date object, JSON.stringify does NOT call toJSON again - // on the returned value (per spec). Date has no own enumerable properties, - // so it serializes to {}. - let result = eval_timeout_quickjs( - "({ toJSON: () => new Date('2024-01-15T00:00:00.000Z') })".to_string(), - HashMap::new(), - None, - None, - None, - None, - None, - ) - .await?; - - assert_eq!(result.get(), "{}"); - Ok(()) - } -} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index fed4e8d31b..741458c0e0 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -34,18 +34,11 @@ mod global_cache; mod go_executor; mod graphql_executor; mod handle_child; -#[cfg(all(feature = "private", feature = "enterprise"))] -mod otel_tracing_proxy_ee; -mod otel_tracing_proxy_oss; pub mod job_logger; #[cfg(feature = "private")] pub mod job_logger_ee; mod job_logger_oss; mod js_eval; -#[cfg(feature = "quickjs")] -pub mod js_eval_quickjs; -#[cfg(test)] -mod js_eval_parity_tests; pub mod memory_common; #[cfg(feature = "private")] pub mod memory_ee; @@ -59,9 +52,13 @@ mod oracledb_executor; #[cfg(feature = "private")] pub mod otel_ee; mod otel_oss; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod otel_tracing_proxy_ee; +mod otel_tracing_proxy_oss; mod pg_executor; #[cfg(feature = "php")] mod php_executor; +mod prepare_deps; #[cfg(feature = "python")] mod python_executor; #[cfg(feature = "python")] @@ -71,33 +68,23 @@ pub mod result_processor; mod rust_executor; mod sanitized_sql_params; mod schema; -pub mod scoped_dependency_map; pub mod sql_utils; mod universal_pkg_installer; -mod prepare_deps; mod worker; mod worker_flow; mod worker_lockfiles; mod worker_utils; -pub mod workspace_dependencies; -pub use worker::*; -pub use worker_lockfiles::{ - process_relative_imports, trigger_dependents_to_recompute_dependencies, -}; #[cfg(all(feature = "private", feature = "enterprise"))] -pub use otel_tracing_proxy_ee::{ - set_current_job_context, start_jobs_otel_tracing, TRACING_PROXY_PORT, -}; +pub use otel_tracing_proxy_ee::start_jobs_otel_tracing; #[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))] -pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED, OTLP_COLLECTOR_PORT}; - -pub use result_processor::handle_job_error; +pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED}; +pub use worker::*; pub use bun_executor::{ build_loader, compute_bundle_local_and_remote_path, generate_dedicated_worker_wrapper, get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, - BUN_DEDICATED_WORKER_ARGS, LoaderMode, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, + LoaderMode, BUN_DEDICATED_WORKER_ARGS, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, }; pub use deno_executor::generate_deno_lock; pub use prepare_deps::run_prepare_deps_cli; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index d9f1dda928..e836018235 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -46,6 +46,7 @@ struct MssqlDatabase { #[serde(default, deserialize_with = "empty_as_none")] ca_cert: Option, encrypt: Option, + integrated_auth: Option, } #[derive(Debug, Deserialize)] @@ -129,7 +130,23 @@ pub async fn do_mssql( } // Handle authentication based on available credentials - if let Some(token_value) = &database.aad_token { + if database.integrated_auth.unwrap_or(false) { + #[cfg(any(feature = "mssql-kerberos", feature = "mssql-winauth"))] + { + config.authentication(AuthMethod::Integrated); + #[cfg(feature = "mssql-kerberos")] + let logs = format!("\nUsing Integrated Authentication (Kerberos/GSSAPI)"); + #[cfg(feature = "mssql-winauth")] + let logs = format!("\nUsing Integrated Authentication (Windows SSPI)"); + append_logs(&job.id, &job.workspace_id, logs, conn).await; + } + #[cfg(not(any(feature = "mssql-kerberos", feature = "mssql-winauth")))] + { + return Err(Error::BadRequest( + "Integrated authentication is not available in this build. Requires mssql-kerberos (Linux) or mssql-winauth (Windows) feature.".to_string(), + )); + } + } else if let Some(token_value) = &database.aad_token { if let Some(token) = &token_value.token { config.authentication(AuthMethod::aad_token(token)); } else { @@ -141,7 +158,7 @@ pub async fn do_mssql( config.authentication(AuthMethod::sql_server(user.clone(), password.clone())); } else { return Err(Error::BadRequest( - "Neither AAD token nor username/password credentials are set".to_string(), + "No authentication method configured. Set integrated_auth, aad_token, or user/password.".to_string(), )); } diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 954c844209..99f67ed456 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -396,7 +396,10 @@ fn string_date_to_mysql_date(s: &str) -> mysql_async::Value { get_capture_by_index(&caps, 1), get_capture_by_index(&caps, 2), get_capture_by_index(&caps, 3), - 0, 0, 0, 0, + 0, + 0, + 0, + 0, ); } diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 7e17678426..9fc947fc70 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -16,11 +16,11 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - handle_child, get_proxy_envs_for_lang, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + get_proxy_envs_for_lang, handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, TRACING_PROXY_CA_CERT_PATH, }; -use windmill_common::scripts::ScriptLang; use windmill_common::client::AuthedClient; +use windmill_common::scripts::ScriptLang; const NSJAIL_CONFIG_RUN_NU_CONTENT: &str = include_str!("../nsjail/run.nu.config.proto"); lazy_static::lazy_static! { diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 9b2a19fec5..0be37a7ac3 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -27,9 +27,8 @@ use crate::{ read_file_content, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, - TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, + POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV, }; fn val_to_pwsh_param(v: serde_json::Value) -> String { @@ -72,6 +71,7 @@ $ErrorActionPreference = 'Stop' $availableModules = Get-Module -ListAvailable $path = '{path}' $hasPrivateRepo = {has_private_repo} +$hasCredentials = {has_credentials} $jobId = '{job_id}' $privateRepoUrl = '{private_repo_url}' $privateRepoPat = '{private_repo_pat}' @@ -83,10 +83,12 @@ if ($hasPrivateRepo) { $repoName = "windmill-private-$jobId" $repoUri = "$privateRepoUrl" - # Create PSCredential for authentication - $username = "token" - $patToken = ConvertTo-SecureString $privateRepoPat -AsPlainText -Force - $credentials = New-Object System.Management.Automation.PSCredential($username, $patToken) + # Create PSCredential for authentication only if PAT is provided + if ($hasCredentials) { + $username = "token" + $patToken = ConvertTo-SecureString $privateRepoPat -AsPlainText -Force + $credentials = New-Object System.Management.Automation.PSCredential($username, $patToken) + } Write-Host "Registering temporary repository: $repoName" @@ -114,7 +116,8 @@ try { # First try private repository if configured if ($hasPrivateRepo) { - $findParams = @{ Name = $moduleName; Repository = $repoName; ErrorAction = 'SilentlyContinue'; Credential = $credentials } + $findParams = @{ Name = $moduleName; Repository = $repoName; ErrorAction = 'SilentlyContinue' } + if ($credentials) { $findParams.Credential = $credentials } if ($requiredVersion) { $findParams.Version = $requiredVersion } $privateModule = Find-PSResource @findParams @@ -123,7 +126,8 @@ try { $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } Write-Host "Found module $moduleName$versionInfo in private repository, installing from there..." - $saveParams = @{ Name = $moduleName; Path = $path; Repository = $repoName; Credential = $credentials } + $saveParams = @{ Name = $moduleName; Path = $path; Repository = $repoName } + if ($credentials) { $saveParams.Credential = $credentials } if ($requiredVersion) { $saveParams.Version = $requiredVersion } Save-PSResource @saveParams } @@ -265,9 +269,11 @@ pub async fn handle_powershell_job( let value_opt = job_args.and_then(|x| x.get(&arg.name)); // Check if this is a switch parameter (only [switch], not [bool]) - let is_switch = arg.otyp.as_ref().map(|t| { - t.to_lowercase() == "switch" - }).unwrap_or(false); + let is_switch = arg + .otyp + .as_ref() + .map(|t| t.to_lowercase() == "switch") + .unwrap_or(false); if is_switch { // Handle switch parameters: -SwitchName or omit @@ -351,7 +357,8 @@ pub async fn handle_powershell_job( if !modules_to_install.is_empty() { let powershell_repo_url = POWERSHELL_REPO_URL.read().await.clone(); let powershell_repo_pat = POWERSHELL_REPO_PAT.read().await.clone(); - let has_private_repo = powershell_repo_url.is_some() && powershell_repo_pat.is_some(); + let has_private_repo = powershell_repo_url.is_some(); + let has_credentials = powershell_repo_pat.is_some(); let modules_list = modules_to_install .iter() @@ -372,6 +379,7 @@ pub async fn handle_powershell_job( .replace("{path}", POWERSHELL_CACHE_DIR) .replace("{job_id}", &job.id.to_string()) .replace("{has_private_repo}", &format!("${has_private_repo}")) + .replace("{has_credentials}", &format!("${has_credentials}")) .replace( "{private_repo_url}", &powershell_repo_url.unwrap_or_default(), @@ -454,11 +462,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", { format!( "{}\n{}\n{}\n{}\n{}", - param_block, - profile, - strict_termination_start, - remaining_code, - strict_termination_end + param_block, profile, strict_termination_start, remaining_code, strict_termination_end ) } else { format!("{}\n{}", profile, content) @@ -493,8 +497,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .runnable_path .as_ref() .map(|x| { - !x.starts_with(INIT_SCRIPT_PATH_PREFIX) - && !x.starts_with(PERIODIC_SCRIPT_PATH_PREFIX) + !x.starts_with(INIT_SCRIPT_PATH_PREFIX) && !x.starts_with(PERIODIC_SCRIPT_PATH_PREFIX) }) .unwrap_or(true); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 6572568a1d..966a76d90d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1016,6 +1016,21 @@ async fn prepare_wrapper( kwargs[\"{name}\"] = datetime.fromisoformat(kwargs[\"{name}\"])\n", ) } + windmill_parser::Typ::Date => { + let name = &x.name; + format!( + "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n \ + try:\n \ + kwargs[\"{name}\"] = date.fromisoformat(kwargs[\"{name}\"])\n \ + except ValueError:\n \ + for _fmt in (\"%d-%m-%Y\", \"%m/%d/%Y\", \"%d/%m/%Y\", \"%Y/%m/%d\"):\n \ + try:\n \ + kwargs[\"{name}\"] = datetime.strptime(kwargs[\"{name}\"], _fmt).date()\n \ + break\n \ + except ValueError:\n \ + continue\n", + ) + } _ => "".to_string(), }) .collect::>() @@ -1035,14 +1050,19 @@ async fn prepare_wrapper( } else { "" }; - let import_datetime = if init_sig + let has_datetime = init_sig .args .iter() - .any(|x| x.typ == windmill_parser::Typ::Datetime) - { - "from datetime import datetime" - } else { - "" + .any(|x| x.typ == windmill_parser::Typ::Datetime); + let has_date = init_sig + .args + .iter() + .any(|x| x.typ == windmill_parser::Typ::Date); + let import_datetime = match (has_datetime, has_date) { + (true, true) => "from datetime import datetime, date", + (true, false) => "from datetime import datetime", + (false, true) => "from datetime import datetime, date", + (false, false) => "", }; let spread = if sig.star_kwargs { "args = kwargs".to_string() diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index f02f114280..52636808a6 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -21,9 +21,12 @@ use windmill_queue::append_logs; use crate::{ common::{start_child_process, OccupancyMetrics}, handle_child::handle_child, - python_executor::{INDEX_CERT, NATIVE_CERT, PYTHON_PATH, UV_PATH}, - HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, WIN_ENVS, + python_executor::{INDEX_CERT, NATIVE_CERT, PYTHON_PATH}, + HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, UV_CACHE_DIR, + WIN_ENVS, }; +#[cfg(unix)] +use crate::python_executor::UV_PATH; impl From for PyVAlias { fn from(value: PyV) -> Self { @@ -527,9 +530,18 @@ impl PyV { .env("HOME", HOME_ENV.to_string()) .env("PATH", PATH_ENV.to_string()) .envs(PROXY_ENVS.clone()) - .args(["python", "install", &v, "--python-preference=only-managed", "--no-bin"]) + .args([ + "python", + "install", + &v, + "--python-preference=only-managed", + "--no-bin", + ]) // TODO: Do we need these? - .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), ("UV_CACHE_DIR", UV_CACHE_DIR)]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), + ("UV_CACHE_DIR", UV_CACHE_DIR), + ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index c2a2b5a001..6992d6ff74 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -586,7 +586,6 @@ pub async fn process_completed_job( duration, result_columns, preprocessed_args, - has_stream, from_cache, flow_runners, done_tx, @@ -655,7 +654,6 @@ pub async fn process_completed_job( canceled_by.clone(), false, duration, - has_stream.unwrap_or(false), from_cache.unwrap_or(false), ) .await?; diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index f0762a6e0d..9cd656f9e3 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -26,7 +26,8 @@ use crate::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, }, - handle_child::{self}, get_proxy_envs_for_lang, + get_proxy_envs_for_lang, + handle_child::{self}, universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency}, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, TRACING_PROXY_CA_CERT_PATH, @@ -51,7 +52,6 @@ const NSJAIL_CONFIG_DOWNLOAD_RUBY_CONTENT: &str = include_str!("../nsjail/download.ruby.config.proto"); const NSJAIL_CONFIG_LOCK_RUBY_CONTENT: &str = include_str!("../nsjail/lock.ruby.config.proto"); - #[allow(dead_code)] pub(crate) struct JobHandlerInput<'a> { pub base_internal_url: &'a str, diff --git a/backend/windmill-worker/src/runtime.js b/backend/windmill-worker/src/runtime.js deleted file mode 100644 index 0feafb59ec..0000000000 --- a/backend/windmill-worker/src/runtime.js +++ /dev/null @@ -1,212 +0,0 @@ -import * as abortSignal from "ext:deno_web/03_abort_signal.js"; -import * as base64 from "ext:deno_web/05_base64.js"; -import * as console from "ext:deno_console/01_console.js"; -import * as encoding from "ext:deno_web/08_text_encoding.js"; -import * as event from "ext:deno_web/02_event.js"; -import * as fetch from "ext:deno_fetch/26_fetch.js"; -import * as file from "ext:deno_web/09_file.js"; -import * as fileReader from "ext:deno_web/10_filereader.js"; -import * as formData from "ext:deno_fetch/21_formdata.js"; -import * as headers from "ext:deno_fetch/20_headers.js"; -import * as streams from "ext:deno_web/06_streams.js"; -import * as timers from "ext:deno_web/02_timers.js"; -import * as url from "ext:deno_url/00_url.js"; -import * as net from "ext:deno_net/01_net.js"; -import * as tls from "ext:deno_net/02_tls.js"; -import * as urlPattern from "ext:deno_url/01_urlpattern.js"; -import * as webidl from "ext:deno_webidl/00_webidl.js"; -import * as response from "ext:deno_fetch/23_response.js"; -import * as request from "ext:deno_fetch/23_request.js"; -import "ext:deno_web/02_structured_clone.js"; -import "ext:deno_web/04_global_interfaces.js"; -import "ext:deno_web/13_message_port.js"; -import "ext:deno_web/14_compression.js"; -import "ext:deno_web/15_performance.js"; -import "ext:deno_web/16_image_data.js"; -import "ext:deno_fetch/27_eventsource.js"; - -globalThis.atob = base64.atob; -globalThis.btoa = base64.btoa; -globalThis.fetch = fetch.fetch; -globalThis.Request = request.Request; -globalThis.Response = response.Response; -globalThis.Blob = file.Blob; -globalThis.URL = url.URL; -globalThis.FormData = formData.FormData; -globalThis.URLSearchParams = url.URLSearchParams; -globalThis.Headers = headers.Headers; -globalThis.FileReader = fileReader.FileReader; -globalThis.console = new console.Console((msg, level) => - globalThis.Deno.core.ops.op_log(msg) -); -globalThis.AbortController = abortSignal.AbortController; -globalThis.AbortSignal = abortSignal.AbortSignal; - -Object.assign(globalThis, { - clearInterval: timers.clearInterval, - clearTimeout: timers.clearTimeout, - setInterval: timers.setInterval, - setTimeout: timers.setTimeout, -}); - -// Expose bootstrapOtel globally so it can be called from Rust after runtime creation. -// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. -// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] -// consoleConfig: 0=ignore, 1=capture, 2=replace -globalThis.__bootstrapOtel = () => { - import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { - bootstrap([1, 0, 1, 0]); - // Expose enterSpan for setting parent trace context - globalThis.__enterSpan = enterSpan; - }); -}; - -// Object.assign(globalThis, { -// console: nonEnumerable( -// new console.Console((msg, level) => core.print(msg, level > 1)) -// ), - -// // timers - -// // fetch -// Request: nonEnumerable(request.Request), -// Response: nonEnumerable(response.Response), -// Headers: nonEnumerable(headers.Headers), -// fetch: writable(fetch.fetch), - -// // base64 -// atob: writable(base64.atob), -// btoa: writable(base64.btoa), - -// // encoding -// TextDecoder: nonEnumerable(encoding.TextDecoder), -// TextEncoder: nonEnumerable(encoding.TextEncoder), -// TextDecoderStream: nonEnumerable(encoding.TextDecoderStream), -// TextEncoderStream: nonEnumerable(encoding.TextEncoderStream), - -// // url -// URL: nonEnumerable(url.URL), -// URLPattern: nonEnumerable(urlPattern.URLPattern), -// URLSearchParams: nonEnumerable(url.URLSearchParams), - -// // // crypto -// // CryptoKey: nonEnumerable(crypto.CryptoKey), -// // crypto: readOnly(crypto.crypto), -// // Crypto: nonEnumerable(crypto.Crypto), -// // SubtleCrypto: nonEnumerable(crypto.SubtleCrypto), - -// // streams -// ByteLengthQueuingStrategy: nonEnumerable(streams.ByteLengthQueuingStrategy), -// CountQueuingStrategy: nonEnumerable(streams.CountQueuingStrategy), -// ReadableStream: nonEnumerable(streams.ReadableStream), -// ReadableStreamDefaultReader: nonEnumerable( -// streams.ReadableStreamDefaultReader -// ), -// ReadableByteStreamController: nonEnumerable( -// streams.ReadableByteStreamController -// ), -// ReadableStreamBYOBReader: nonEnumerable(streams.ReadableStreamBYOBReader), -// ReadableStreamBYOBRequest: nonEnumerable(streams.ReadableStreamBYOBRequest), -// ReadableStreamDefaultController: nonEnumerable( -// streams.ReadableStreamDefaultController -// ), -// TransformStream: nonEnumerable(streams.TransformStream), -// TransformStreamDefaultController: nonEnumerable( -// streams.TransformStreamDefaultController -// ), -// WritableStream: nonEnumerable(streams.WritableStream), -// WritableStreamDefaultWriter: nonEnumerable( -// streams.WritableStreamDefaultWriter -// ), -// WritableStreamDefaultController: nonEnumerable( -// streams.WritableStreamDefaultController -// ), - -// // event -// CloseEvent: nonEnumerable(event.CloseEvent), -// CustomEvent: nonEnumerable(event.CustomEvent), -// ErrorEvent: nonEnumerable(event.ErrorEvent), -// Event: nonEnumerable(event.Event), -// EventTarget: nonEnumerable(event.EventTarget), -// MessageEvent: nonEnumerable(event.MessageEvent), -// PromiseRejectionEvent: nonEnumerable(event.PromiseRejectionEvent), -// ProgressEvent: nonEnumerable(event.ProgressEvent), -// reportError: writable(event.reportError), -// DOMException: nonEnumerable(DOMException), - -// // file -// Blob: nonEnumerable(file.Blob), -// File: nonEnumerable(file.File), -// FileReader: nonEnumerable(fileReader.FileReader), - -// // form data -// FormData: nonEnumerable(formData.FormData), - -// // abort signal -// AbortController: nonEnumerable(abortSignal.AbortController), -// AbortSignal: nonEnumerable(abortSignal.AbortSignal), - -// // // web sockets -// // WebSocket: nonEnumerable(webSocket.WebSocket), - -// // // performance -// // Performance: nonEnumerable(performance.Performance), -// // PerformanceEntry: nonEnumerable(performance.PerformanceEntry), -// // PerformanceMark: nonEnumerable(performance.PerformanceMark), -// // PerformanceMeasure: nonEnumerable(performance.PerformanceMeasure), -// // performance: writable(performance.performance), - -// // messagePort -// // structuredClone: writable(messagePort.structuredClone), - -// // Branding as a WebIDL object -// [webidl.brand]: nonEnumerable(webidl.brand), -// }); - -// function nonEnumerable(value) { -// return { -// value, -// writable: true, -// enumerable: false, -// configurable: true, -// }; -// } - -// function writable(value) { -// return { -// value, -// writable: true, -// enumerable: true, -// configurable: true, -// }; -// } - -// function readOnly(value) { -// return { -// value, -// enumerable: true, -// writable: false, -// configurable: true, -// }; -// } - -// function getterOnly(getter) { -// return { -// get: getter, -// set() {}, -// enumerable: true, -// configurable: true, -// }; -// } - -// function formatException(error) { -// if (ObjectPrototypeIsPrototypeOf(ErrorPrototype, error)) { -// return null; -// } else if (typeof error == "string") { -// return `Uncaught ${console.inspectArgs([console.quoteString(error)], { -// colors: false, -// })}`; -// } else { -// return `Uncaught ${console.inspectArgs([error], { colors: false })}`; -// } -// } diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 90695145f8..4c195ca24e 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,15 +19,17 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, + get_reserved_variables, read_result, start_child_process, OccupancyMetrics, + DEV_CONF_NSJAIL, }, + get_proxy_envs_for_lang, handle_child::handle_child, - get_proxy_envs_for_lang, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - PROXY_ENVS, RUST_CACHE_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUST_CACHE_DIR, + TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; -use windmill_common::scripts::ScriptLang; use windmill_common::client::AuthedClient; +use windmill_common::scripts::ScriptLang; #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -53,7 +55,6 @@ lazy_static::lazy_static! { static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); } - #[cfg(not(windows))] lazy_static::lazy_static! { static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); diff --git a/backend/windmill-worker/src/schema.rs b/backend/windmill-worker/src/schema.rs index eca9afa72e..3269fa6a50 100644 --- a/backend/windmill-worker/src/schema.rs +++ b/backend/windmill-worker/src/schema.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use windmill_common::schema::{SchemaValidationRule, SchemaValidator}; use windmill_parser::{MainArgSignature, Typ}; - fn make_rules_for_arg_typ(typ: &Typ) -> Vec { let mut rules = vec![]; @@ -35,7 +34,7 @@ fn make_rules_for_arg_typ(typ: &Typ) -> Vec { rules.push(SchemaValidationRule::IsString); rules.push(SchemaValidationRule::IsBytes); } - Typ::Datetime => { + Typ::Datetime | Typ::Date => { rules.push(SchemaValidationRule::IsString); rules.push(SchemaValidationRule::IsDatetime); } @@ -66,7 +65,10 @@ fn make_rules_for_arg_typ(typ: &Typ) -> Vec { for prop in &variant.properties { obj_rules.push((prop.key.to_string(), make_rules_for_arg_typ(&prop.typ))); } - rules_map.insert(variant.label.to_string(), vec![SchemaValidationRule::IsObject(obj_rules)]); + rules_map.insert( + variant.label.to_string(), + vec![SchemaValidationRule::IsObject(obj_rules)], + ); } rules.push(SchemaValidationRule::IsOneOf(rules_map)) @@ -94,4 +96,3 @@ pub fn schema_validator_from_main_arg_sig(sig: &MainArgSignature) -> SchemaValid SchemaValidator { required, rules } } - diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index e1dad165c0..4c68d76135 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -119,11 +119,16 @@ async fn poll_snowflake_async_query( })?; let status = response.status(); - let body = response.text().await.map_err(|e| { - Error::ExecutionErr(format!("error reading poll response body: {}", e)) - })?; + let body = response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("error reading poll response body: {}", e)))?; - tracing::debug!("Snowflake poll response status: {}, body: {}", status, &body[..body.len().min(500)]); + tracing::debug!( + "Snowflake poll response status: {}, body: {}", + status, + &body[..body.len().min(500)] + ); if status == reqwest::StatusCode::ACCEPTED { // Still running, wait and poll again @@ -243,13 +248,14 @@ fn do_snowflake_inner<'a>( let body = raw_response.text().await.map_err(|e| { Error::ExecutionErr(format!("error reading response body: {}", e)) })?; - let async_resp: SnowflakeAsyncResponse = serde_json::from_str(&body).map_err(|e| { - Error::ExecutionErr(format!( - "error decoding async response: {}. Body preview: {}", - e, - &body[..body.len().min(500)] - )) - })?; + let async_resp: SnowflakeAsyncResponse = + serde_json::from_str(&body).map_err(|e| { + Error::ExecutionErr(format!( + "error decoding async response: {}. Body preview: {}", + e, + &body[..body.len().min(500)] + )) + })?; tracing::info!( "Snowflake statement running asynchronously, polling for completion (handle: {})", @@ -273,21 +279,27 @@ fn do_snowflake_inner<'a>( // Handle both sync (200) and async (202) responses let raw_response = handle_snowflake_result(result).await?; let status = raw_response.status(); - let body = raw_response.text().await.map_err(|e| { - Error::ExecutionErr(format!("error reading response body: {}", e)) - })?; + let body = raw_response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("error reading response body: {}", e)))?; - tracing::debug!("Snowflake response status: {}, body: {}", status, &body[..body.len().min(1000)]); + tracing::debug!( + "Snowflake response status: {}, body: {}", + status, + &body[..body.len().min(1000)] + ); let response = if status == reqwest::StatusCode::ACCEPTED { // Async execution - need to poll for results - let async_resp: SnowflakeAsyncResponse = serde_json::from_str(&body).map_err(|e| { - Error::ExecutionErr(format!( - "error decoding async response: {}. Body preview: {}", - e, - &body[..body.len().min(500)] - )) - })?; + let async_resp: SnowflakeAsyncResponse = + serde_json::from_str(&body).map_err(|e| { + Error::ExecutionErr(format!( + "error decoding async response: {}. Body preview: {}", + e, + &body[..body.len().min(500)] + )) + })?; tracing::info!( "Snowflake query running asynchronously, polling for results (handle: {})", diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c8eab55741..f3cdc241e0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -12,7 +12,6 @@ use anyhow::anyhow; use futures::TryFutureExt; use tokio::sync::Mutex; -use tokio::time::sleep; use tokio::time::timeout; use windmill_common::client::AuthedClient; use windmill_common::jobs::WorkerInternalServerInlineUtils; @@ -131,14 +130,13 @@ use crate::{ go_executor::handle_go_job, graphql_executor::do_graphql, handle_child::SLOW_LOGS, - handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, pwsh_executor::handle_powershell_job, - result_processor::{process_result, start_background_processor}, + result_processor::{handle_job_error, process_result, start_background_processor}, schema::schema_validator_from_main_arg_sig, - worker_flow::handle_flow, + worker_flow::{handle_flow, SchedulePushZombieError}, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, }, @@ -246,7 +244,7 @@ pub const DEFAULT_SLEEP_QUEUE: u64 = 50; // only 1 native job so that we don't have to worry about concurrency issues on non dedicated native jobs workers pub const DEFAULT_NATIVE_JOBS: usize = 1; -const VACUUM_PERIOD: u32 = 50000; +const VACUUM_PERIOD: u32 = 10000; // #[cfg(any(target_os = "linux"))] // const DROP_CACHE_PERIOD: u32 = 1000; @@ -1619,6 +1617,11 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] let mut infos = BenchmarkInfo::new(); + #[cfg(feature = "benchmark")] + if let Some(db) = conn.as_sql() { + infos.init_pool_stats(db.size()); + } + let vacuum_shift = rand::rng().random_range(0..VACUUM_PERIOD); IS_READY.store(true, Ordering::Relaxed); @@ -2068,6 +2071,9 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] { add_time!(bench, "sent to dedicated worker"); + if let Some(db) = conn.as_sql() { + infos.sample_pool(db.size(), db.num_idle() as u32); + } infos.add_iter(bench, true); } @@ -2138,6 +2144,9 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] { add_time!(bench, "sent to flow runner"); + if let Some(db) = conn.as_sql() { + infos.sample_pool(db.size(), db.num_idle() as u32); + } infos.add_iter(bench, true); } @@ -2438,6 +2447,9 @@ pub async fn run_worker( { if started { add_time!(bench, "job processed"); + if let Some(db) = conn.as_sql() { + infos.sample_pool(db.size(), db.num_idle() as u32); + } infos.add_iter(bench, true); } } @@ -2466,6 +2478,9 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] { add_time!(bench, "sleep because empty job queue"); + if let Some(db) = conn.as_sql() { + infos.sample_pool(db.size(), db.num_idle() as u32); + } infos.add_iter(bench, false); } #[cfg(feature = "prometheus")] @@ -2964,7 +2979,7 @@ pub async fn handle_queued_job( // Not a preview: fetch from the cache or the database. _ => cache::job::fetch_flow(db, &job.kind, job.runnable_id).await?, }; - Box::pin(handle_flow( + match Box::pin(handle_flow( job, &flow_data, db, @@ -2978,8 +2993,19 @@ pub async fn handle_queued_job( &killpill_rx, )) .warn_after_seconds(10) - .await?; - Ok(true) + .await + { + Err(err) if err.downcast_ref::().is_some() => { + tracing::error!( + "Schedule push zombie: {err}. Leaving flow job in queue for zombie detection to restart." + ); + Ok(true) + } + other => { + other?; + Ok(true) + } + } } else { return Err(Error::internal_err( "Could not handle flow job with agent worker".to_string(), @@ -3015,7 +3041,7 @@ pub async fn handle_queued_job( #[cfg(not(feature = "enterprise"))] if let Connection::Sql(db) = conn { if (job.concurrent_limit.is_some() || - windmill_common::runnable_settings::RunnableSettings::prefetch_cached_from_handle(job.runnable_settings_handle, db).await?.1.concurrent_limit.is_some()) + windmill_common::runnable_settings::prefetch_cached_from_handle(job.runnable_settings_handle, db).await?.1.concurrent_limit.is_some()) && !job.kind.is_dependency() { logs.push_str("---\n"); logs.push_str("WARNING: This job has concurrency limits enabled. Concurrency limits are an EE feature and the setting is ignored.\n"); @@ -3037,7 +3063,7 @@ pub async fn handle_queued_job( .flatten() { tracing::debug!("Debug: {} going to sleep for {}", job.id, dbg_djob_sleep); - sleep(std::time::Duration::from_secs(dbg_djob_sleep as u64)).await; + tokio::time::sleep(std::time::Duration::from_secs(dbg_djob_sleep as u64)).await; } tracing::debug!( diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 42306fbfe5..7b013bda9e 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -21,6 +21,7 @@ use crate::{ use anyhow::Context; use async_once_cell::Lazy; +use backon::{BackoffBuilder, ConstantBuilder, Retryable}; use futures::TryFutureExt; use mappable_rc::Marc; use serde::{Deserialize, Serialize}; @@ -67,8 +68,10 @@ use windmill_common::{ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, - handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy, FlowRunners, - MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + insert_concurrency_key, interpolate_args, + report_error_to_workspace_handler_or_critical_side_channel, try_schedule_next_job, CanceledBy, + FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, + WrappedError, }; use windmill_audit::audit_oss::audit_log; @@ -76,6 +79,17 @@ use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; use windmill_queue::{canceled_job_to_result, push}; +#[derive(Debug)] +pub struct SchedulePushZombieError(pub String); + +impl std::fmt::Display for SchedulePushZombieError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for SchedulePushZombieError {} + /// Helper function to write itered data to separate table /// Returns None if data was written to separate table, Some(itered) if it should be stored in JSONB async fn write_itered_to_db( @@ -1351,35 +1365,31 @@ pub async fn update_flow_status_after_job_completion_internal( let args_hm = args.unwrap_or_default().0; let args = PushArgs::from(&args_hm); if let Some(ck) = concurrency_key { - let mut tx = db.begin().await?; insert_concurrency_key( &flow_job.workspace_id, &args, &flow_job.runnable_path, JobKind::Flow, Some(ck), - &mut tx, + db, flow, ) .await?; - tx.commit().await?; } if let Some(t) = tag { tag = Some(interpolate_args(t, &args, &flow_job.workspace_id)); } } else if concurrent_limit.is_some() { - let mut tx = db.begin().await?; insert_concurrency_key( &flow_job.workspace_id, &PushArgs::from(&HashMap::new()), &flow_job.runnable_path, JobKind::Flow, concurrency_key, - &mut tx, + db, flow, ) .await?; - tx.commit().await?; } // let tag = tag_and_concurrency_key.and_then(|tc| tc.tag.map(|t| interpolate_args(t.clone(), &args, &workspace_id))); @@ -1630,7 +1640,6 @@ pub async fn update_flow_status_after_job_completion_internal( true, None, false, - false, ) .await?; duration @@ -1651,7 +1660,6 @@ pub async fn update_flow_status_after_job_completion_internal( true, None, false, - false, ) .await?; duration @@ -2177,22 +2185,76 @@ pub async fn handle_flow( .await?; if let Some(schedule) = schedule { - if let Err(err) = handle_maybe_scheduled_job( - db, - &MiniCompletedJob::from(flow_job.clone()), - &schedule, - flow_job.runnable_path.as_ref().unwrap(), - &flow_job.workspace_id, - ) - .warn_after_seconds(5) - .await - { - match err { - Error::QuotaExceeded(_) => return Err(err.into()), - // scheduling next job failed and could not disable schedule => make zombie job to retry - _ => return Ok(()), + let mini_job = MiniCompletedJob::from(flow_job.clone()); + let runnable_path = flow_job.runnable_path.as_ref().unwrap().clone(); + let schedule_push_result = (|| async { + let tx = db.begin().warn_after_seconds(5).await + .map_err(|e| Error::internal_err(format!("begin tx for schedule push: {e:#}")))?; + let (tx, schedule_push_err) = try_schedule_next_job( + db, + tx, + &mini_job, + &schedule, + &runnable_path, + ) + .await; + if let Some(err) = schedule_push_err { + return Err(err); } - }; + tx.commit().warn_after_seconds(5).await + .map_err(|e| Error::internal_err(format!("commit schedule push: {e:#}")))?; + Ok::<(), Error>(()) + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(3)) + .with_max_times(10) + .build(), + ) + .when(|err: &Error| !matches!(err, Error::QuotaExceeded(_) | Error::NotFound(_))) + .notify(|err: &Error, dur: std::time::Duration| { + tracing::error!( + "Could not push next scheduled job for flow schedule {}, retrying in {dur:#?}: {err:#?}", + schedule.path + ); + }) + .sleep(tokio::time::sleep) + .await; + + // Non-retryable errors (QuotaExceeded, NotFound) are handled inside + // try_schedule_next_job (schedule disabled, returns None), so they never + // reach here. This handles only transient errors after retry exhaustion. + if let Err(err) = schedule_push_result { + tracing::error!( + "Could not push next scheduled job for {} after retries: {err}. Disabling schedule.", + schedule.path + ); + if let Err(disable_err) = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", + err.to_string(), + &flow_job.workspace_id, + &schedule.path + ) + .execute(db) + .await + { + report_error_to_workspace_handler_or_critical_side_channel( + &mini_job, + db, + format!( + "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + schedule.path, + ), + ) + .await; + return Err(SchedulePushZombieError( + format!( + "Could not push or disable schedule {} after retries", + schedule.path + ), + ).into()); + } + } } else { tracing::error!( "Schedule {schedule_path} in {} not found. Impossible to schedule again", @@ -3189,7 +3251,7 @@ async fn push_next_flow_job( "UPDATE v2_job_runtime SET ping = now() WHERE id = $1 AND ping < now()", flow_job.id, ) - .execute(db) + .execute(&mut *tx) .warn_after_seconds(3) .await?; } @@ -5030,6 +5092,10 @@ pub async fn get_previous_job_result( Some(FlowStatusModule::Success { flow_jobs: Some(flow_jobs), .. }) => { Ok(Some(retrieve_flow_jobs_results(db, w_id, flow_jobs).await?)) } + Some(FlowStatusModule::Success { job, .. }) if *job == Uuid::nil() => { + // Empty branch — no real job was executed, return empty object + Ok(None) + } Some(FlowStatusModule::Success { job, .. }) => Ok(Some( sqlx::query_scalar!( "SELECT result AS \"result!: Json>\" diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 0209eec361..5da446c825 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1,20 +1,16 @@ use std::borrow::Cow; use std::collections::HashMap; use std::fs::{create_dir_all, remove_dir_all}; -use std::path::{Component, Path, PathBuf}; #[cfg(feature = "python")] use crate::ansible_executor::{get_git_repos_lock, AnsibleDependencyLocks}; -use crate::scoped_dependency_map::{DependencyDependent, ScopedDependencyMap}; use async_recursion::async_recursion; -use chrono::{Duration, Utc}; use itertools::Itertools; use serde::Serialize; use serde_json::value::RawValue; use serde_json::{from_value, json, Value}; use sha2::Digest; use sqlx::types::Json; -use tokio::time::timeout; use uuid::Uuid; use windmill_common::assets::{ clear_static_asset_usage, insert_static_asset_usage, AssetUsageKind, @@ -22,17 +18,15 @@ use windmill_common::assets::{ use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; -use windmill_common::jobs::JobPayload; -use windmill_common::runnable_settings::DebouncingSettings; +use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; use windmill_common::scripts::ScriptHash; -use windmill_common::utils::WarnAfterExt; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; -use windmill_common::min_version::MIN_VERSION_SUPPORTS_DEBOUNCING_V2; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; use windmill_common::workspace_dependencies::{ - RawWorkspaceDependencies, WorkspaceDependencies, WorkspaceDependenciesPrefetched, + RawWorkspaceDependencies, WorkspaceDependenciesPrefetched, }; +use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; #[cfg(feature = "python")] use windmill_parser_yaml::AnsibleRequirements; @@ -44,13 +38,12 @@ use windmill_common::{ scripts::ScriptLang, DB, }; +pub use windmill_dep_map::{ + extract_referenced_paths, extract_relative_imports, process_relative_imports, +}; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; -#[cfg(feature = "python")] -use windmill_parser_py_imports::parse_relative_imports; -use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{ - append_logs, CanceledBy, MiniPulledJob, PushIsolationLevel, - WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT, + append_logs, CanceledBy, MiniPulledJob, WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT, }; // TODO: To be removed in future versions @@ -59,9 +52,6 @@ lazy_static::lazy_static! { static ref WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ: bool = std::env::var("WMDEBUG_NO_NEW_FLOW_VERSION_ON_DJ").is_ok(); static ref WMDEBUG_NO_NEW_APP_VERSION_ON_DJ: bool = std::env::var("WMDEBUG_NO_NEW_APP_VERSION_ON_DJ").is_ok(); static ref WMDEBUG_NO_COMPONENTS_TO_RELOCK: bool = std::env::var("WMDEBUG_NO_COMPONENTS_TO_RELOCK").is_ok(); - static ref DEPENDENCY_JOB_DEBOUNCE_DELAY: usize = std::env::var("DEPENDENCY_JOB_DEBOUNCE_DELAY").ok().and_then(|flag| flag.parse().ok()).unwrap_or( - if cfg!(test) { /* if test we want increased debouncing delay */ 15 } else { 5 } - ); } use crate::common::{MaybeLock, OccupancyMetrics}; @@ -84,103 +74,6 @@ use crate::{ go_executor::install_go_dependencies, }; -fn try_normalize(path: &Path) -> Option { - let mut ret = PathBuf::new(); - - for component in path.components() { - match component { - Component::Prefix(..) | Component::RootDir => return None, - Component::CurDir => {} - Component::ParentDir => { - if !ret.pop() { - return None; - } - } - Component::Normal(c) => { - ret.push(c); - } - } - } - - Some(ret) -} - -fn parse_ts_relative_imports(raw_code: &str, script_path: &str) -> error::Result> { - let mut relative_imports = vec![]; - let r = parse_expr_for_imports(raw_code, true)?; - for import in r { - let import = import.trim_end_matches(".ts"); - if import.starts_with("/") { - relative_imports.push(import.trim_start_matches("/").to_string()); - } else if import.starts_with(".") { - let normalized = try_normalize(std::path::Path::new(&format!( - "{}/../{}", - script_path, import - ))); - if let Some(normalized) = normalized { - let normalized = normalized.to_str().unwrap().to_string(); - relative_imports.push(normalized); - } else { - tracing::error!("error canonicalizing path: {script_path} with import {import}"); - } - } - } - - Ok(relative_imports) -} - -pub fn extract_relative_imports( - raw_code: &str, - script_path: &str, - language: &Option, -) -> Option> { - match language { - #[cfg(feature = "python")] - Some(ScriptLang::Python3) => parse_relative_imports(&raw_code, script_path).ok(), - Some(ScriptLang::Bun) | Some(ScriptLang::Bunnative) | Some(ScriptLang::Deno) => { - parse_ts_relative_imports(&raw_code, script_path).ok() - } - _ => None, - } -} - -pub fn extract_referenced_paths( - raw_code: &str, - script_path: &str, - language: Option, -) -> Option> { - let mut referenced_paths = vec![]; - if let Some(wk_deps_refs) = language - .and_then(|l| l.extract_workspace_dependencies_annotated_refs(raw_code, script_path)) - .map(|r| r.external) - { - let l = language.expect("should be some"); - for wk_deps_ref in wk_deps_refs { - if let Some(path) = WorkspaceDependencies::to_path(&Some(wk_deps_ref), l).ok() { - referenced_paths.push(path); - }; - } - } else if let (Some(l), true /* Only if it is not blacklisted */) = ( - language, - WorkspaceDependenciesPrefetched::is_external_references_permitted(script_path), - ) { - // we assume all runnables without annotated dependencies reference default dependencies file. - WorkspaceDependencies::to_path(&None, l) - .ok() - .inspect(|p| referenced_paths.push(p.to_owned())); - } - - if let Some(relative_imports) = extract_relative_imports(raw_code, script_path, &language) { - referenced_paths.extend(relative_imports); - } - - if referenced_paths.is_empty() { - None - } else { - Some(referenced_paths) - } -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_dependency_job( job: &MiniPulledJob, @@ -364,315 +257,6 @@ fn remove_ansi_codes(s: &str) -> String { ANSI_REGEX.replace_all(s, "").to_string() } -pub async fn process_relative_imports( - db: &sqlx::Pool, - _job_id: Option, - args: Option<&Json>>>, - w_id: &str, - script_path: &str, - parent_path: Option, - deployment_message: Option, - code: &str, - script_lang: &Option, - permissioned_as_email: &str, - created_by: &str, - permissioned_as: &str, -) -> error::Result<()> { - // TODO: Should be moved into handle_dependency_job body to be more consistent with how flows and apps are handled - { - let mut tx = db.begin().await?; - let mut dependency_map = ScopedDependencyMap::fetch_maybe_rearranged( - &w_id, - script_path, - "script", - &parent_path, - db, - ) - .await?; - - tx = dependency_map - .patch( - extract_referenced_paths(&code, script_path, *script_lang), - // Ideally should be None, but due to current implementation will use empty string to represent None. - "".into(), - tx, - ) - .await?; - - dependency_map.dissolve(tx).await.commit().await?; - } - - { - let mut already_visited = args - .map(|x| { - x.get("already_visited") - .map(|v| serde_json::from_str::>(v.get()).ok()) - .flatten() - }) - .flatten() - .unwrap_or_default(); - - // TODO: There is a race-condition. - // This can be old version. - - // Check lines of code below, you will find that we get the latest version of the script/app/flow - - // However the latest version does not necessarily mean that it is finalized. - // Instead we assume that this would be the version we would base on. - - // So the script_importers might be behind. Thus some information like nodes_to_relock might be lost. - let importers = crate::scoped_dependency_map::ScopedDependencyMap::get_dependents( - script_path, - w_id, - db, - ) - .await?; - - already_visited.push(script_path.to_string()); - // But currently we will do this extra db call for every script regardless of whether they have relative imports or not - // Script might have no relative imports but still be referenced by someone else. - match timeout( - core::time::Duration::from_secs(60), - Box::pin(trigger_dependents_to_recompute_dependencies( - w_id, - importers, - deployment_message, - parent_path, - permissioned_as_email, - created_by, - permissioned_as, - db, - already_visited, - )), - ) - .warn_after_seconds(10) - .await - { - Ok(Err(e)) => { - tracing::error!(%e, "error triggering dependents to recompute dependencies") - } - Err(e) => { - tracing::error!(%e, "triggering dependents to recompute dependencies has timed out") - } - _ => {} - } - } - - Ok(()) -} - -pub async fn trigger_dependents_to_recompute_dependencies( - w_id: &str, - importers: Vec, - // imported_path: &str, - deployment_message: Option, - parent_path: Option, - email: &str, - created_by: &str, - permissioned_as: &str, - db: &sqlx::Pool, - already_visited: Vec, -) -> error::Result<()> { - tracing::debug!( - "Triggering dependents to recompute dependencies: {}", - importers.iter().map(|dd| &dd.importer_path).join(",") - ); - for DependencyDependent { importer_path, importer_kind, importer_node_ids } in importers.iter() - { - tracing::trace!("Processing dependency: {:?}", importer_path); - if already_visited.contains(importer_path) { - tracing::trace!("Skipping already visited dependency"); - continue; - } - - let mut tx = db.clone().begin().await?; - let mut args: HashMap> = HashMap::new(); - if let Some(ref dm) = deployment_message { - args.insert("deployment_message".to_string(), to_raw_value(&dm)); - } - if let Some(ref p_path) = parent_path { - // NOTE: - // it's not used but maybe one day it will be useful. allows more back-compatibility for the workers when we need it - // also very useful for debugging/observability - // it adds that information to the job args so you can see from the runs page - args.insert("common_dependency_path".to_string(), to_raw_value(&p_path)); - } - - args.insert( - "already_visited".to_string(), - to_raw_value(&already_visited), - ); - - args.insert( - "triggered_by_relative_import".to_string(), - to_raw_value(&true), - ); - - let mut debouncing_settings = DebouncingSettings { - debounce_key: Some(format!("{w_id}:{importer_path}:dependency")), - debounce_delay_s: Some(5), - ..Default::default() - }; - - let job_payload = match importer_kind.as_str() { - // TODO: Make it query only non-archived - // Scripts - "script" => match sqlx::query_scalar!( - "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false ORDER BY created_at DESC LIMIT 1", - importer_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(hash) => { - tracing::debug!("newest hash for {} is: {hash}", importer_path); - - let info = - windmill_common::get_script_info_for_hash(None, db, w_id, hash).await?; - - JobPayload::Dependencies { - path: importer_path.clone(), - hash: ScriptHash(hash), - language: info.language, - dedicated_worker: info.dedicated_worker, - debouncing_settings, - } - } - None => { - ScopedDependencyMap::clear_map_for_item( - importer_path, - w_id, - "script", - tx, - &None, - ) - .await - .commit() - .await?; - continue; - } - }, - - // Flows - "flow" => match sqlx::query_scalar!( - "SELECT id FROM flow_version WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", - importer_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(version) => { - tracing::debug!("Handling flow dependency update for: {}", importer_path); - - args.insert( - "nodes_to_relock".to_string(), - to_raw_value(&importer_node_ids), - ); - - debouncing_settings.debounce_args_to_accumulate = Some(vec!["nodes_to_relock".into()]); - - JobPayload::FlowDependencies { - path: importer_path.clone(), - version, - dedicated_worker: None, - debouncing_settings, - } - } - None => { - ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "flow", tx, &None) - .await - .commit() - .await?; - continue; - } - }, - - // Apps - "app" => match sqlx::query_scalar!( - "SELECT id FROM app_version WHERE app_id = (SELECT id FROM app WHERE path = $1 AND workspace_id = $2) ORDER BY created_at DESC LIMIT 1", - importer_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - { - Some(version) => { - tracing::debug!("Handling app dependency update for: {}", importer_path); - - args.insert( - "components_to_relock".to_string(), - // TODO: unsafe. Importer Node Ids are not checked. They can simply be array of empty strings! - to_raw_value(importer_node_ids), - ); - - debouncing_settings.debounce_args_to_accumulate = Some(vec!["components_to_relock".into()]); - - JobPayload::AppDependencies { path: importer_path.clone(), version, debouncing_settings } - } - None => { - ScopedDependencyMap::clear_map_for_item(importer_path, w_id, "app", tx, &None) - .await - .commit() - .await?; - continue; - } - }, - - _ => { - tracing::error!( - "unexpected importer kind: {kind:?} for path {path}", - kind = importer_kind, - path = importer_path - ); - continue; - } - }; - - tracing::debug!("Pushing dependency job for: {}", importer_path); - let (job_uuid, new_tx) = windmill_queue::push( - db, - PushIsolationLevel::Transaction(tx), - &w_id, - job_payload, - windmill_queue::PushArgs { args: &args, extra: None }, - &created_by, - email, - permissioned_as.to_string(), - Some("trigger.dependents.to.recompute.dependencies"), - // Schedule for future for debouncing. - Some(Utc::now() + Duration::seconds(*DEPENDENCY_JOB_DEBOUNCE_DELAY as i64)), - None, - None, - None, - None, - None, - false, - false, - None, - true, - Some("dependency".into()), - None, - None, - None, - None, - false, - None, - None, - None, - ) - .await?; - - tracing::info!( - "pushed dependency job due to common python path: {job_uuid} for path {path}", - path = importer_path, - ); - new_tx.commit().await?; - } - Ok(()) -} - pub async fn handle_flow_dependency_job( job: MiniPulledJob, preview_data: Option<&RawData>, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d708c06bb8..d398be2c09 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.625.0"; +export const VERSION = "v1.629.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/app/generate_agents.ts b/cli/src/commands/app/generate_agents.ts index b9feebc56d..d2811e0b07 100644 --- a/cli/src/commands/app/generate_agents.ts +++ b/cli/src/commands/app/generate_agents.ts @@ -194,12 +194,15 @@ export async function regenerateAgentDocs( const agentsContent = generateAgentsDocumentation(localData); await Deno.writeTextFile(path.join(targetDir, "AGENTS.md"), agentsContent); + // Generate and write CLAUDE.md referencing AGENTS.md + await Deno.writeTextFile(path.join(targetDir, "CLAUDE.md"), `Instructions are in @AGENTS.md\n`); + // Generate and write DATATABLES.md const datatablesContent = generateDatatablesMarkdown(schemas, localData); await Deno.writeTextFile(path.join(targetDir, "DATATABLES.md"), datatablesContent); if (!silent) { - log.info(colors.green(`✓ Generated AGENTS.md and DATATABLES.md`)); + log.info(colors.green(`✓ Generated AGENTS.md, CLAUDE.md, and DATATABLES.md`)); // Summary const datatableCount = schemas.length; diff --git a/cli/src/commands/app/new.ts b/cli/src/commands/app/new.ts index 7dd5cf7d6e..d79600118d 100644 --- a/cli/src/commands/app/new.ts +++ b/cli/src/commands/app/new.ts @@ -537,6 +537,12 @@ CREATE SCHEMA IF NOT EXISTS ${schemaName}; agentsContent ); + // Create CLAUDE.md referencing AGENTS.md + await Deno.writeTextFile( + path.join(appDir, "CLAUDE.md"), + `Instructions are in @AGENTS.md\n` + ); + // Create DATATABLES.md with the configured data const datatablesContent = generateDatatablesDocumentation(dataForDocs); await Deno.writeTextFile( diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index 471cc858f3..f0ab3f60ac 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -1,11 +1,23 @@ import { colors, Command, log, yamlStringify, Confirm } from "../../../deps.ts"; import { GlobalOptions } from "../../types.ts"; import { readLockfile } from "../../utils/metadata.ts"; -import { SCRIPT_GUIDANCE } from "../../guidance/script_guidance.ts"; -import { FLOW_GUIDANCE } from "../../guidance/flow_guidance.ts"; import { getActiveWorkspaceOrFallback } from "../workspace/workspace.ts"; import { generateRTNamespace } from "../resource-type/resource-type.ts"; -import { CLI_COMMANDS } from "../../guidance/prompts.ts"; +import { SKILLS, SKILL_CONTENT, SCHEMAS, SCHEMA_MAPPINGS } from "../../guidance/skills.ts"; +import { generateAgentsMdContent } from "../../guidance/core.ts"; + +/** + * Format a YAML schema for inclusion in skill markdown files. + */ +function formatSchemaForMarkdown(schemaYaml: string, schemaName: string, filePattern: string): string { + return `## ${schemaName} (\`${filePattern}\`) + +Must be a YAML file that adheres to the following schema: + +\`\`\`yaml +${schemaYaml.trim()} +\`\`\``; +} export interface InitOptions { useDefault?: boolean; @@ -238,37 +250,76 @@ async function initAction(opts: InitOptions) { } } - // Create .cursor/rules directory and files with SCRIPT_GUIDANCE content + // Create guidance files (AGENTS.md, CLAUDE.md, and Claude skills) try { - const scriptGuidanceContent = SCRIPT_GUIDANCE; - const flowGuidanceContent = FLOW_GUIDANCE; - const cliCommandsContent = CLI_COMMANDS; + // Generate skills reference section for AGENTS.md + const skills_base_dir = ".claude/skills"; + const skillsReference = SKILLS.map( + (s) => `- \`${skills_base_dir}/${s.name}/SKILL.md\` - ${s.description}` + ).join("\n"); - // Create AGENTS.md file + // Create AGENTS.md file with minimal instructions if (!(await Deno.stat("AGENTS.md").catch(() => null))) { await Deno.writeTextFile( "AGENTS.md", - ` -You are a helpful assistant that can help with Windmill scripts and flows creation. - -## Script Guidance -${scriptGuidanceContent} - -## Flow Guidance -${flowGuidanceContent} - -## CLI Commands -${cliCommandsContent} -` + generateAgentsMdContent(skillsReference) ); log.info(colors.green("Created AGENTS.md")); } // Create CLAUDE.md file, referencing AGENTS.md if (!(await Deno.stat("CLAUDE.md").catch(() => null))) { - await Deno.writeTextFile("CLAUDE.md", "Instructions are in @AGENTS.md"); + await Deno.writeTextFile( + "CLAUDE.md", + `Instructions are in @AGENTS.md +` + ); log.info(colors.green("Created CLAUDE.md")); } + + // Create .claude/skills/ directory and skill files + try { + await Deno.mkdir(".claude/skills", { recursive: true }); + + await Promise.all( + SKILLS.map(async (skill) => { + const skillDir = `.claude/skills/${skill.name}`; + await Deno.mkdir(skillDir, { recursive: true }); + + let skillContent = SKILL_CONTENT[skill.name]; + if (skillContent) { + // Check if this skill has schemas that need to be appended + const schemaMappings = SCHEMA_MAPPINGS[skill.name]; + if (schemaMappings && schemaMappings.length > 0) { + // Combine base content with schemas + const schemaDocs = schemaMappings + .map((mapping) => { + const schemaYaml = SCHEMAS[mapping.schemaKey]; + if (schemaYaml) { + return formatSchemaForMarkdown(schemaYaml, mapping.name, mapping.filePattern); + } + return null; + }) + .filter((doc): doc is string => doc !== null); + + if (schemaDocs.length > 0) { + skillContent = skillContent + "\n\n" + schemaDocs.join("\n\n"); + } + } + + await Deno.writeTextFile(`${skillDir}/SKILL.md`, skillContent); + } + }) + ); + + log.info(colors.green(`Created .claude/skills/ with ${SKILLS.length} skills`)); + } catch (skillError) { + if (skillError instanceof Error) { + log.warn(`Could not create skills: ${skillError.message}`); + } else { + log.warn(`Could not create skills: ${skillError}`); + } + } } catch (error) { if (error instanceof Error) { log.warn(`Could not create guidance files: ${error.message}`); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 251ef4148b..0af6b725d0 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -317,8 +317,9 @@ export function extractFieldsForRawApps(runnables: Record) { } /** - * Generates AGENTS.md - the main documentation file for AI agents working with raw apps. - * This includes app structure, backend runnables, datatables usage, and all critical rules. + * Generates AGENTS.md - app-specific configuration for AI agents working with raw apps. + * References the raw-app skill for complete documentation and includes instance-specific + * data configuration (datatable, schema, whitelisted tables). */ export function generateAgentsDocumentation(data: { tables?: string[]; @@ -331,146 +332,13 @@ export function generateAgentsDocumentation(data: { return `# AI Agent Instructions -This file contains instructions for AI agents (Claude, GPT, etc.) working with this Windmill raw app. -**Read this file first** before making any changes to the app. +For complete raw app documentation (app structure, backend runnables, datatables, SQL migrations), use the \`raw-app\` skill. -## App Structure - -\`\`\` -my_app.raw_app/ -├── AGENTS.md # This file - read first! -├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) -├── raw_app.yaml # App configuration (summary, path, data settings) -├── index.tsx # Frontend entry point -├── App.tsx # Main React/Svelte/Vue component -├── index.css # Styles -├── package.json # Frontend dependencies -├── wmill.ts # Auto-generated - backend type definitions (DO NOT EDIT) -├── backend/ # Backend runnables (server-side scripts) -│ ├── . # Code file (e.g., get_user.ts) - auto-detected as inline -│ ├── .yaml # Optional: config for fields, or to reference existing scripts -│ └── .lock # Lock file (run 'wmill app generate-locks' to create) -└── sql_to_apply/ # SQL migrations (dev only, not synced) - └── *.sql # SQL files to apply via dev server -\`\`\` - -## Backend Runnables - -Backend runnables are server-side scripts that your frontend can call. They live in the \`backend/\` folder. - -### Creating a Backend Runnable - -The simplest way to create a runnable is to add a code file: - -\`\`\` -backend/. -\`\`\` - -The runnable ID is the filename without extension. For example, \`get_user.ts\` creates a runnable with ID \`get_user\`. - -**Optional:** Add a \`.yaml\` file for additional configuration (fields, static values, etc.). - -### Supported Languages and Extensions - -| Language | Extension | Example | -|-------------|----------------|--------------------| -| TypeScript | \`.ts\` | \`myFunc.ts\` | -| TypeScript (Bun) | \`.bun.ts\` | \`myFunc.bun.ts\` | -| TypeScript (Deno) | \`.deno.ts\` | \`myFunc.deno.ts\` | -| Python | \`.py\` | \`myFunc.py\` | -| Go | \`.go\` | \`myFunc.go\` | -| Bash | \`.sh\` | \`myFunc.sh\` | -| PowerShell | \`.ps1\` | \`myFunc.ps1\` | -| PostgreSQL | \`.pg.sql\` | \`myFunc.pg.sql\` | -| MySQL | \`.my.sql\` | \`myFunc.my.sql\` | -| BigQuery | \`.bq.sql\` | \`myFunc.bq.sql\` | -| Snowflake | \`.sf.sql\` | \`myFunc.sf.sql\` | -| MS SQL | \`.ms.sql\` | \`myFunc.ms.sql\` | -| GraphQL | \`.gql\` | \`myFunc.gql\` | -| PHP | \`.php\` | \`myFunc.php\` | -| Rust | \`.rs\` | \`myFunc.rs\` | -| C# | \`.cs\` | \`myFunc.cs\` | -| Java | \`.java\` | \`myFunc.java\` | - -### Example: Creating a Runnable - -**backend/get_user.ts:** -\`\`\`typescript -import * as wmill from 'windmill-client'; - -export async function main(user_id: string) { - const sql = wmill.datatable(); - const user = await sql\`SELECT * FROM users WHERE id = \${user_id}\`.fetchOne(); - return user; -} -\`\`\` - -That's it! The runnable is automatically detected and ready to use. - -**Generate lock files** for dependency management: -\`\`\`bash -wmill app generate-locks -\`\`\` - -### Optional: YAML Configuration - -Add a \`.yaml\` file to configure fields, static values, or other settings: - -**backend/get_user.yaml:** -\`\`\`yaml -type: inline -fields: - user_id: - type: static - value: "default_user" -\`\`\` - -### Referencing Existing Scripts - -To reference an existing Windmill script instead of inline code, use a different type: - -**backend/existing_script.yaml:** -\`\`\`yaml -type: script -path: f/my_folder/existing_script -\`\`\` - -For flows: -\`\`\`yaml -type: flow -path: f/my_folder/my_flow -\`\`\` - -### Calling Backend Runnables from Frontend - -Import from the auto-generated \`wmill.ts\`: - -\`\`\`typescript -import { backend } from './wmill'; - -// Call a backend runnable -const user = await backend.get_user({ user_id: '123' }); -\`\`\` - -The \`wmill.ts\` file is auto-generated and provides type-safe access to all backend runnables. +This file contains **app-specific configuration** for this raw app instance. --- -## ⚠️ CRITICAL RULES FOR DATA TABLES - -**These rules are mandatory - violating them will cause runtime errors:** - -1. **ONLY USE WHITELISTED TABLES**: You can ONLY query tables listed in \`raw_app.yaml\` → \`data.tables\`. - Tables not in this list are NOT accessible to the app. - -2. **ADD TABLES BEFORE USING**: To use a new table, you MUST first add it to \`data.tables\` in \`raw_app.yaml\`. - -3. **USE CONFIGURED DATATABLE/SCHEMA**: When looking for tables: - - First, check the whitelisted tables below - - If creating new tables, use the default datatable${defaultSchema ? ` and schema` : ''} configured for this app - - See \`DATATABLES.md\` for full schema information - -### Current Data Configuration +## Data Configuration ${defaultDatatable ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ''}` @@ -482,7 +350,7 @@ ${tables.length > 0 ? `These tables are accessible to this app:\n\n${tables.map(t => `- \`${t}\``).join('\n')}` : `**No tables whitelisted.** Add tables to \`data.tables\` in \`raw_app.yaml\`.`} -### Adding a Table to the Whitelist +### Adding a Table Edit \`raw_app.yaml\`: @@ -495,116 +363,31 @@ ${tables.length > 0 ? tables.map(t => ` - ${t}`).join('\n') : ' # Add tabl \`\`\` **Table reference formats:** -- \`\` - All tables in the datatable -- \`/\` - Specific table in public schema +- \`/
\` - Table in public schema - \`/:
\` - Table in specific schema --- -## Querying Data Tables +## Quick Reference -### TypeScript (Bun/Deno) +**Backend runnable:** Add \`backend/.ts\` (or .py, etc.), then run \`wmill app generate-locks\` +**Call from frontend:** \`\`\`typescript -import * as wmill from 'windmill-client'; - -export async function main(user_id: string) { - const sql = wmill.datatable(); // Or: wmill.datatable('other_datatable') - - // Parameterized queries (safe from SQL injection) - const user = await sql\`SELECT * FROM users WHERE id = \${user_id}\`.fetchOne(); - const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); - - // Insert/Update - await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; - await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; - - return user; -} +import { backend } from './wmill'; +const result = await backend.({ arg: 'value' }); \`\`\` -### Python - -\`\`\`python -import wmill - -def main(user_id: str): - db = wmill.datatable() # Or: wmill.datatable('other_datatable') - - # Use $1, $2, etc. for parameters - user = db.query('SELECT * FROM users WHERE id = $1', user_id).fetch_one() - users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() - - # Insert/Update - db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) - db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) - - return user +**Query datatable (TypeScript):** +\`\`\`typescript +const sql = wmill.datatable(); +const rows = await sql\`SELECT * FROM table WHERE id = \${id}\`.fetch(); \`\`\` ---- - -## SQL Migrations (sql_to_apply/) - -The \`sql_to_apply/\` folder is for creating/modifying database tables during development. - -### How It Works - -1. Create \`.sql\` files in \`sql_to_apply/\` -2. Run \`wmill app dev\` - the dev server watches this folder -3. When SQL files change, a modal appears in the browser to confirm execution -4. After creating tables, **add them to \`data.tables\`** in \`raw_app.yaml\` - -### Example Migration - -**sql_to_apply/001_create_users.sql:** -\`\`\`sql -CREATE TABLE IF NOT EXISTS ${defaultSchema ? defaultSchema + '.' : ''}users ( - id SERIAL PRIMARY KEY, - email TEXT NOT NULL UNIQUE, - name TEXT, - created_at TIMESTAMP DEFAULT NOW() -); -\`\`\` - -After applying, add to \`raw_app.yaml\`: -\`\`\`yaml -data: - tables: - - ${defaultDatatable || 'main'}/${defaultSchema ? defaultSchema + ':' : ''}users -\`\`\` - -### Important Notes - -- **This folder is NOT synced** - it's local development only -- **Use idempotent SQL**: \`CREATE TABLE IF NOT EXISTS\`, etc. -- **Number your files**: \`001_\`, \`002_\` for ordering -- **Always whitelist tables after creation** +**SQL migrations:** Add \`.sql\` files to \`sql_to_apply/\`, run \`wmill app dev\`, then whitelist tables --- - -## Commands Reference - -| Command | Description | -|---------|-------------| -| \`wmill app dev\` | Start dev server with live reload | -| \`wmill app generate-agents\` | Refresh AGENTS.md and DATATABLES.md | -| \`wmill app generate-locks\` | Generate lock files for backend runnables | -| \`wmill sync push\` | Deploy app to Windmill | -| \`wmill sync pull\` | Pull latest from Windmill | - ---- - -## Best Practices - -1. **Check DATATABLES.md** for existing tables before creating new ones -2. **Use parameterized queries** - never concatenate user input into SQL -3. **Keep runnables focused** - one function per file -4. **Use descriptive IDs** - \`get_user.ts\` not \`a.ts\` -5. **Always whitelist tables** - add to \`data.tables\` before querying - ---- -*This file is auto-generated. Run \`wmill app new\` or \`wmill sync pull\` to regenerate.* +*Run \`wmill app generate-agents\` to refresh. See \`.claude/skills/raw-app\` skill for full documentation.* `; } diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index fb35ba26fe..4625e589ca 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -398,6 +398,8 @@ async function whoami(_opts: GlobalOptions) { } async function listRemote(_opts: GlobalOptions) { + const { resolveWorkspace } = await import("../../core/context.ts"); + const workspace = await resolveWorkspace(_opts); await requireLogin(_opts); const userWorkspaces = await wmill.listUserWorkspaces(); @@ -415,6 +417,7 @@ async function listRemote(_opts: GlobalOptions) { ) .render(); + log.info(`Remote: ${colors.bold(workspace.remote)}`); log.info(`Logged in as: ${colors.green.bold(userWorkspaces.email)}`); } @@ -541,6 +544,9 @@ const command = new Command() .description("Show the currently active user") .action(whoami as any) .command("list") + .description("List local workspace profiles") + .action(list as any) + .command("list-remote") .description("List workspaces on the remote server that you have access to") .action(listRemote as any) .command("bind") diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts new file mode 100644 index 0000000000..da3b2a146f --- /dev/null +++ b/cli/src/guidance/core.ts @@ -0,0 +1,58 @@ +/** + * Core guidance content for AGENTS.md + * + * This module exports the template for the AGENTS.md file that provides + * AI agent instructions for working with Windmill projects. + */ + +/** + * Generate the AGENTS.md content with the given skills reference. + * @param skillsReference - A formatted list of skills to include in the document + * @returns The complete AGENTS.md content + */ +export function generateAgentsMdContent(skillsReference: string): string { + return `# Windmill AI Agent Instructions + +You are a helpful assistant that can help with Windmill scripts, flows, apps, and resources management. + +## Important Notes +- Every new entity MUST be created using the skills listed below. +- Every modification of an entity MUST be done using the skills listed below. +- User MUST be asked where to create the entity. It can be in its user folder, under u/{user_name} folder, or in a new folder, /f/{folder_name}/. folder_name and user_name must be provided by the user. + +## Script Writing Guide + +You MUST use the \`write-script-\` skill to write or modify scripts in the language specified by the user. Use bun by default. + +## Flow Writing Guide + +You MUST use the \`write-flow\` skill to create or modify flows. + +## Raw App Development + +You MUST use the \`raw-app\` skill to create or modify raw apps. +Whenever a new app needs to be created you MUST ask the user to run \`wmill app new\` in its terminal first. + +## Triggers + +You MUST use the \`triggers\` skill to configure HTTP routes, WebSocket, Kafka, NATS, SQS, MQTT, GCP, or Postgres CDC triggers. + +## Schedules + +You MUST use the \`schedules\` skill to configure cron schedules. + +## Resources + +You MUST use the \`resources\` skill to manage resource types and credentials. + +## CLI Reference + +You MUST use the \`cli-commands\` skill to use the CLI. + +## Skills + +For specific guidance, ALWAYS use the skills listed below. + +${skillsReference} +`; +} diff --git a/cli/src/guidance/flow_guidance.ts b/cli/src/guidance/flow_guidance.ts deleted file mode 100644 index b2b0547bce..0000000000 --- a/cli/src/guidance/flow_guidance.ts +++ /dev/null @@ -1,17 +0,0 @@ -// CLI Flow Guidance - Uses centralized prompts from system_prompts/ -import * as prompts from "./prompts.ts"; - -// CLI-specific introduction -const CLI_INTRO = `You are an expert at creating OpenFlow YAML specifications for Windmill workflows. -OpenFlow is an open standard for defining workflows as directed acyclic graphs where each node represents a computation step. -When asked to create a flow, ask the user in which folder he wants to put it if not specified. Then create a new folder in the specified folder, that ends with \`.flow\`. It should contain a \`.yaml\` file that contains the flow definition. -For rawscript type module in the flow, the content key should start with "!inline" followed by the path of the script containing the code. It should be put in the same folder as the flow. -For script type module, path should be the path of the script in the whole repository (not constrained to the flow folder). -You do not need to create .lock and .yaml files manually. Instead, you should run \`wmill flow generate-locks --yes\` to create them.`; - -// Assemble complete flow guidance -export const FLOW_GUIDANCE = ` -${CLI_INTRO} - -${prompts.FLOW_PROMPT} -`; diff --git a/cli/src/guidance/prompts.ts b/cli/src/guidance/prompts.ts deleted file mode 100644 index c745c205d3..0000000000 --- a/cli/src/guidance/prompts.ts +++ /dev/null @@ -1,2623 +0,0 @@ -// Auto-generated by generate.py - DO NOT EDIT - -export const SCRIPT_PROMPT = `# Windmill Script Writing Guide - -## General Principles - -- Scripts must export a main function (do not call it) -- Libraries are installed automatically - do not show installation instructions -- Credentials and configuration are stored in resources and passed as parameters -- The windmill client (\`wmill\`) provides APIs for interacting with the platform - -## Function Naming - -- Main function: \`main\` (or \`preprocessor\` for preprocessor scripts) -- Must be async for TypeScript variants - -## Return Values - -- Scripts can return any JSON-serializable value -- Return values become available to subsequent flow steps via \`results.step_id\` - -## Preprocessor Scripts - -Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. - -The returned object determines the parameter values passed to the flow. -e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. - -The preprocessor receives a single parameter called \`event\`. - - -# Bash - -## Structure - -Do not include \`#!/bin/bash\`. Arguments are obtained as positional parameters: - -\`\`\`bash -# Get arguments -var1="$1" -var2="$2" - -echo "Processing $var1 and $var2" - -# Return JSON by echoing to stdout -echo "{\\"result\\": \\"$var1\\", \\"count\\": $var2}" -\`\`\` - -**Important:** -- Do not include shebang (\`#!/bin/bash\`) -- Arguments are always strings -- Access with \`$1\`, \`$2\`, etc. - -## Output - -The script output is captured as the result. For structured data, output valid JSON: - -\`\`\`bash -name="$1" -count="$2" - -# Output JSON result -cat << EOF -{ - "name": "$name", - "count": $count, - "timestamp": "$(date -Iseconds)" -} -EOF -\`\`\` - -## Environment Variables - -Environment variables set in Windmill are available: - -\`\`\`bash -# Access environment variable -echo "Workspace: $WM_WORKSPACE" -echo "Job ID: $WM_JOB_ID" -\`\`\` - - -# BigQuery - -Arguments use \`@name\` syntax. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- @name1 (string) --- @name2 (int64) = 0 -SELECT * FROM users WHERE name = @name1 AND age > @name2; -\`\`\` - - -# TypeScript (Bun) - -Bun runtime with full npm ecosystem and fastest execution. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. Libraries are installed automatically. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -\`\`\`typescript -import Stripe from "stripe"; -import { someFunction } from "some-package"; -\`\`\` - -## Windmill Client - -Import the windmill client for platform interactions: - -\`\`\`typescript -import * as wmill from "windmill-client"; -\`\`\` - -See the SDK documentation for available methods. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id, - }; -} -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -### S3Object Type - -The S3Object type represents a file in S3 storage: - -\`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; -\`\`\` - -## TypeScript Operations - -\`\`\`typescript -import * as wmill from "windmill-client"; - -// Load file content from S3 -const content: Uint8Array = await wmill.loadS3File(s3object); - -// Load file as stream -const blob: Blob = await wmill.loadS3FileStream(s3object); - -// Write file to S3 -const result: S3Object = await wmill.writeS3File( - s3object, // Target path (or undefined to auto-generate) - fileContent, // string or Blob - s3ResourcePath // Optional: specific S3 resource to use -); -\`\`\` - - -# TypeScript (Bun Native) - -Native TypeScript execution with fetch only - no external imports allowed. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -**No imports allowed.** Use the globally available \`fetch\` function: - -\`\`\`typescript -export async function main(url: string) { - const response = await fetch(url); - return await response.json(); -} -\`\`\` - -## Windmill Client - -The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id, - }; -} -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -### S3Object Type - -The S3Object type represents a file in S3 storage: - -\`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; -\`\`\` - -## TypeScript Operations - -\`\`\`typescript -import * as wmill from "windmill-client"; - -// Load file content from S3 -const content: Uint8Array = await wmill.loadS3File(s3object); - -// Load file as stream -const blob: Blob = await wmill.loadS3FileStream(s3object); - -// Write file to S3 -const result: S3Object = await wmill.writeS3File( - s3object, // Target path (or undefined to auto-generate) - fileContent, // string or Blob - s3ResourcePath // Optional: specific S3 resource to use -); -\`\`\` - - -# C# - -The script must contain a public static \`Main\` method inside a class: - -\`\`\`csharp -public class Script -{ - public static object Main(string name, int count) - { - return new { Name = name, Count = count }; - } -} -\`\`\` - -**Important:** -- Class name is irrelevant -- Method must be \`public static\` -- Return type can be \`object\` or specific type - -## NuGet Packages - -Add packages using the \`#r\` directive at the top: - -\`\`\`csharp -#r "nuget: Newtonsoft.Json, 13.0.3" -#r "nuget: RestSharp, 110.2.0" - -using Newtonsoft.Json; -using RestSharp; - -public class Script -{ - public static object Main(string url) - { - var client = new RestClient(url); - var request = new RestRequest(); - var response = client.Get(request); - return JsonConvert.DeserializeObject(response.Content); - } -} -\`\`\` - - -# TypeScript (Deno) - -Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. Libraries are installed automatically. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -\`\`\`typescript -// npm packages use npm: prefix -import Stripe from "npm:stripe"; -import { someFunction } from "npm:some-package"; - -// Deno standard library -import { serve } from "https://deno.land/std/http/server.ts"; -\`\`\` - -## Windmill Client - -Import the windmill client for platform interactions: - -\`\`\`typescript -import * as wmill from "windmill-client"; -\`\`\` - -See the SDK documentation for available methods. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id, - }; -} -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -### S3Object Type - -The S3Object type represents a file in S3 storage: - -\`\`\`typescript -type S3Object = { - s3: string; // Path within the bucket -}; -\`\`\` - -## TypeScript Operations - -\`\`\`typescript -import * as wmill from "windmill-client"; - -// Load file content from S3 -const content: Uint8Array = await wmill.loadS3File(s3object); - -// Load file as stream -const blob: Blob = await wmill.loadS3FileStream(s3object); - -// Write file to S3 -const result: S3Object = await wmill.writeS3File( - s3object, // Target path (or undefined to auto-generate) - fileContent, // string or Blob - s3ResourcePath // Optional: specific S3 resource to use -); -\`\`\` - - -# DuckDB - -Arguments are defined with comments and used with \`$name\` syntax: - -\`\`\`sql --- $name (text) = default --- $age (integer) -SELECT * FROM users WHERE name = $name AND age > $age; -\`\`\` - -## Ducklake Integration - -Attach Ducklake for data lake operations: - -\`\`\`sql --- Main ducklake -ATTACH 'ducklake' AS dl; - --- Named ducklake -ATTACH 'ducklake://my_lake' AS dl; - --- Then query -SELECT * FROM dl.schema.table; -\`\`\` - -## External Database Connections - -Connect to external databases using resources: - -\`\`\`sql -ATTACH '$res:path/to/resource' AS db (TYPE postgres); -SELECT * FROM db.schema.table; -\`\`\` - -## S3 File Operations - -Read files from S3 storage: - -\`\`\`sql --- Default storage -SELECT * FROM read_csv('s3:///path/to/file.csv'); - --- Named storage -SELECT * FROM read_csv('s3://storage_name/path/to/file.csv'); - --- Parquet files -SELECT * FROM read_parquet('s3:///path/to/file.parquet'); - --- JSON files -SELECT * FROM read_json('s3:///path/to/file.json'); -\`\`\` - - -# Go - -## Structure - -The file package must be \`inner\` and export a function called \`main\`: - -\`\`\`go -package inner - -func main(param1 string, param2 int) (map[string]interface{}, error) { - return map[string]interface{}{ - "result": param1, - "count": param2, - }, nil -} -\`\`\` - -**Important:** -- Package must be \`inner\` -- Return type must be \`({return_type}, error)\` -- Function name is \`main\` (lowercase) - -## Return Types - -The return type can be any Go type that can be serialized to JSON: - -\`\`\`go -package inner - -type Result struct { - Name string \`json:"name"\` - Count int \`json:"count"\` -} - -func main(name string, count int) (Result, error) { - return Result{ - Name: name, - Count: count, - }, nil -} -\`\`\` - -## Error Handling - -Return errors as the second return value: - -\`\`\`go -package inner - -import "errors" - -func main(value int) (string, error) { - if value < 0 { - return "", errors.New("value must be positive") - } - return "success", nil -} -\`\`\` - - -# GraphQL - -## Structure - -Write GraphQL queries or mutations. Arguments can be added as query parameters: - -\`\`\`graphql -query GetUser($id: ID!) { - user(id: $id) { - id - name - email - } -} -\`\`\` - -## Variables - -Variables are passed as script arguments and automatically bound to the query: - -\`\`\`graphql -query SearchProducts($query: String!, $limit: Int = 10) { - products(search: $query, first: $limit) { - edges { - node { - id - name - price - } - } - } -} -\`\`\` - -## Mutations - -\`\`\`graphql -mutation CreateUser($input: CreateUserInput!) { - createUser(input: $input) { - id - name - createdAt - } -} -\`\`\` - - -# Java - -The script must contain a Main public class with a \`public static main()\` method: - -\`\`\`java -public class Main { - public static Object main(String name, int count) { - java.util.Map result = new java.util.HashMap<>(); - result.put("name", name); - result.put("count", count); - return result; - } -} -\`\`\` - -**Important:** -- Class must be named \`Main\` -- Method must be \`public static Object main(...)\` -- Return type is \`Object\` or \`void\` - -## Maven Dependencies - -Add dependencies using comments at the top: - -\`\`\`java -//requirements: -//com.google.code.gson:gson:2.10.1 -//org.apache.httpcomponents:httpclient:4.5.14 - -import com.google.gson.Gson; - -public class Main { - public static Object main(String input) { - Gson gson = new Gson(); - return gson.fromJson(input, Object.class); - } -} -\`\`\` - - -# Microsoft SQL Server (MSSQL) - -Arguments use \`@P1\`, \`@P2\`, etc. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- @P1 name1 (varchar) --- @P2 name2 (int) = 0 -SELECT * FROM users WHERE name = @P1 AND age > @P2; -\`\`\` - - -# MySQL - -Arguments use \`?\` placeholders. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- ? name1 (text) --- ? name2 (int) = 0 -SELECT * FROM users WHERE name = ? AND age > ?; -\`\`\` - - -# TypeScript (Native) - -Native TypeScript execution with fetch only - no external imports allowed. - -## Structure - -Export a single **async** function called \`main\`: - -\`\`\`typescript -export async function main(param1: string, param2: number) { - // Your code here - return { result: param1, count: param2 }; -} -\`\`\` - -Do not call the main function. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -Use the \`RT\` namespace for resource types: - -\`\`\`typescript -export async function main(stripe: RT.Stripe) { - // stripe contains API key and config from the resource -} -\`\`\` - -Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. - -## Imports - -**No imports allowed.** Use the globally available \`fetch\` function: - -\`\`\`typescript -export async function main(url: string) { - const response = await fetch(url); - return await response.json(); -} -\`\`\` - -## Windmill Client - -The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`typescript -type Event = { - kind: - | "webhook" - | "http" - | "websocket" - | "kafka" - | "email" - | "nats" - | "postgres" - | "sqs" - | "mqtt" - | "gcp"; - body: any; - headers: Record; - query: Record; -}; - -export async function preprocessor(event: Event) { - return { - param1: event.body.field1, - param2: event.query.id - }; -} -\`\`\` - - -# PHP - -## Structure - -The script must start with \` $param1, "count" => $param2]; -} -\`\`\` - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -You need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using \`class_exists\`: - -\`\`\`php - $2::INT; -\`\`\` - - -# PowerShell - -## Structure - -Arguments are obtained by calling the \`param\` function on the first line: - -\`\`\`powershell -param($Name, $Count = 0, [int]$Age) - -# Your code here -Write-Output "Processing $Name, count: $Count, age: $Age" - -# Return object -@{ - name = $Name - count = $Count - age = $Age -} -\`\`\` - -## Parameter Types - -You can specify types for parameters: - -\`\`\`powershell -param( - [string]$Name, - [int]$Count = 0, - [bool]$Enabled = $true, - [array]$Items -) - -@{ - name = $Name - count = $Count - enabled = $Enabled - items = $Items -} -\`\`\` - -## Return Values - -Return values by outputting them at the end of the script: - -\`\`\`powershell -param($Input) - -$result = @{ - processed = $true - data = $Input - timestamp = Get-Date -Format "o" -} - -$result -\`\`\` - - -# Python - -## Structure - -The script must contain at least one function called \`main\`: - -\`\`\`python -def main(param1: str, param2: int): - # Your code here - return {"result": param1, "count": param2} -\`\`\` - -Do not call the main function. Libraries are installed automatically. - -## Resource Types - -On Windmill, credentials and configuration are stored in resources and passed as parameters to main. - -You need to **redefine** the type of the resources that are needed before the main function as TypedDict: - -\`\`\`python -from typing import TypedDict - -class postgresql(TypedDict): - host: str - port: int - user: str - password: str - dbname: str - -def main(db: postgresql): - # db contains the database connection details - pass -\`\`\` - -**Important rules:** - -- The resource type name must be **IN LOWERCASE** -- Only include resource types if they are actually needed -- If an import conflicts with a resource type name, **rename the imported object, not the type name** -- Make sure to import TypedDict from typing **if you're using it** - -## Imports - -Libraries are installed automatically. Do not show installation instructions. - -\`\`\`python -import requests -import pandas as pd -from datetime import datetime -\`\`\` - -If an import name conflicts with a resource type: - -\`\`\`python -# Wrong - don't rename the type -import stripe as stripe_lib -class stripe_type(TypedDict): ... - -# Correct - rename the import -import stripe as stripe_sdk -class stripe(TypedDict): - api_key: str -\`\`\` - -## Windmill Client - -Import the windmill client for platform interactions: - -\`\`\`python -import wmill -\`\`\` - -See the SDK documentation for available methods. - -## Preprocessor Scripts - -For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: - -\`\`\`python -from typing import TypedDict, Literal, Any - -class Event(TypedDict): - kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] - body: Any - headers: dict[str, str] - query: dict[str, str] - -def preprocessor(event: Event): - # Transform the event into flow input parameters - return { - "param1": event["body"]["field1"], - "param2": event["query"]["id"] - } -\`\`\` - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -\`\`\`python -import wmill - -# Load file content from S3 -content: bytes = wmill.load_s3_file(s3object) - -# Load file as stream reader -reader: BufferedReader = wmill.load_s3_file_reader(s3object) - -# Write file to S3 -result: S3Object = wmill.write_s3_file( - s3object, # Target path (or None to auto-generate) - file_content, # bytes or BufferedReader - s3_resource_path, # Optional: specific S3 resource - content_type, # Optional: MIME type - content_disposition # Optional: Content-Disposition header -) -\`\`\` - - -# Rust - -## Structure - -The script must contain a function called \`main\` with proper return type: - -\`\`\`rust -use anyhow::anyhow; -use serde::Serialize; - -#[derive(Serialize, Debug)] -struct ReturnType { - result: String, - count: i32, -} - -fn main(param1: String, param2: i32) -> anyhow::Result { - Ok(ReturnType { - result: param1, - count: param2, - }) -} -\`\`\` - -**Important:** -- Arguments should be owned types -- Return type must be serializable (\`#[derive(Serialize)]\`) -- Return type is \`anyhow::Result\` - -## Dependencies - -Packages must be specified with a partial cargo.toml at the beginning of the script: - -\`\`\`rust -//! \`\`\`cargo -//! [dependencies] -//! anyhow = "1.0.86" -//! reqwest = { version = "0.11", features = ["json"] } -//! tokio = { version = "1", features = ["full"] } -//! \`\`\` - -use anyhow::anyhow; -// ... rest of the code -\`\`\` - -**Note:** Serde is already included, no need to add it again. - -## Async Functions - -If you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside: - -\`\`\`rust -//! \`\`\`cargo -//! [dependencies] -//! anyhow = "1.0.86" -//! tokio = { version = "1", features = ["full"] } -//! reqwest = { version = "0.11", features = ["json"] } -//! \`\`\` - -use anyhow::anyhow; -use serde::Serialize; - -#[derive(Serialize, Debug)] -struct Response { - data: String, -} - -fn main(url: String) -> anyhow::Result { - let rt = tokio::runtime::Runtime::new()?; - rt.block_on(async { - let resp = reqwest::get(&url).await?.text().await?; - Ok(Response { data: resp }) - }) -} -\`\`\` - - -# Snowflake - -Arguments use \`?\` placeholders. - -Name the parameters by adding comments before the statement: - -\`\`\`sql --- ? name1 (text) --- ? name2 (number) = 0 -SELECT * FROM users WHERE name = ? AND age > ?; -\`\`\` - - -# TypeScript SDK (windmill-client) - -Import: import * as wmill from 'windmill-client' - -/** - * Initialize the Windmill client with authentication token and base URL - * @param token - Authentication token (defaults to WM_TOKEN env variable) - * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) - */ -setClient(token?: string, baseUrl?: string): void - -/** - * Create a client configuration from env variables - * @returns client configuration - */ -getWorkspace(): string - -/** - * Get a resource value by path - * @param path path of the resource, default to internal state path - * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error - * @returns resource value - */ -async getResource(path?: string, undefinedIfEmpty?: boolean): Promise - -/** - * Get the true root job id - * @param jobId job id to get the root job id from (default to current job) - * @returns root job id - */ -async getRootJobId(jobId?: string): Promise - -/** - * @deprecated Use runScriptByPath or runScriptByHash instead - */ -async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its path and wait for the result - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Run a script synchronously by its hash and wait for the result - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param verbose - Enable verbose logging - * @returns Script execution result - */ -async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Append a text to the result stream - * @param text text to append to the result stream - */ -appendToResultStream(text: string): void - -/** - * Stream to the result stream - * @param stream stream to stream to the result stream - */ -async streamResult(stream: AsyncIterable): Promise - -/** - * Run a flow synchronously by its path and wait for the result - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param verbose - Enable verbose logging - * @returns Flow execution result - */ -async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise - -/** - * Wait for a job to complete and return its result - * @param jobId - ID of the job to wait for - * @param verbose - Enable verbose logging - * @returns Job result when completed - */ -async waitJob(jobId: string, verbose: boolean = false): Promise - -/** - * Get the result of a completed job - * @param jobId - ID of the completed job - * @returns Job result - */ -async getResult(jobId: string): Promise - -/** - * Get the result of a job if completed, or its current status - * @param jobId - ID of the job - * @returns Object with started, completed, success, and result properties - */ -async getResultMaybe(jobId: string): Promise - -/** - * Wrap a function to execute as a Windmill task within a flow context - * @param f - Function to wrap as a task - * @returns Async wrapper function that executes as a Windmill job - */ -task(f: (_: P) => T): (_: P) => Promise - -/** - * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead - */ -async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its path - * @param path - Script path in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a script asynchronously by its hash - * @param hash_ - Script hash in Windmill - * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @returns Job ID of the created job - */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise - -/** - * Run a flow asynchronously by its path - * @param path - Flow path in Windmill - * @param args - Arguments to pass to the flow - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) - * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) - * @returns Job ID of the created job - */ -async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise - -/** - * Resolve a resource value in case the default value was picked because the input payload was undefined - * @param obj resource value or path of the resource under the format \`$res:path\` - * @returns resource value - */ -async resolveDefaultResource(obj: any): Promise - -/** - * Get the state file path from environment variables - * @returns State path string - */ -getStatePath(): string - -/** - * Set a resource value by path - * @param path path of the resource to set, default to state path - * @param value new value of the resource to set - * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type - */ -async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise - -/** - * Set the state - * @param state state to set - * @deprecated use setState instead - */ -async setInternalState(state: any): Promise - -/** - * Set the state - * @param state state to set - */ -async setState(state: any): Promise - -/** - * Set the progress - * Progress cannot go back and limited to 0% to 99% range - * @param percent Progress to set in % - * @param jobId? Job to set progress for - */ -async setProgress(percent: number, jobId?: any): Promise - -/** - * Get the progress - * @param jobId? Job to get progress from - * @returns Optional clamped between 0 and 100 progress value - */ -async getProgress(jobId?: any): Promise - -/** - * Set a flow user state - * @param key key of the state - * @param value value of the state - */ -async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise - -/** - * Get a flow user state - * @param path path of the variable - */ -async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise - -/** - * Get the internal state - * @deprecated use getState instead - */ -async getInternalState(): Promise - -/** - * Get the state shared across executions - */ -async getState(): Promise - -/** - * Get a variable by path - * @param path path of the variable - * @returns variable value - */ -async getVariable(path: string): Promise - -/** - * Set a variable by path, create if not exist - * @param path path of the variable - * @param value value of the variable - * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) - * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") - */ -async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise - -/** - * Build a PostgreSQL connection URL from a database resource - * @param path - Path to the database resource - * @returns PostgreSQL connection URL string - */ -async databaseUrlFromResource(path: string): Promise - -/** - * Get S3 client settings from a resource or workspace default - * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) - * @returns S3 client configuration settings - */ -async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise - -/** - * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContent = await wmill.loadS3FileContent(inputFile) - * // if the file is a raw text file, it can be decoded and printed directly: - * const text = new TextDecoder().decode(fileContentStream) - * console.log(text); - * \`\`\` - */ -async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * let fileContentBlob = await wmill.loadS3FileStream(inputFile) - * // if the content is plain text, the blob can be read directly: - * console.log(await fileContentBlob.text()); - * \`\`\` - */ -async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise - -/** - * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. - * - * \`\`\`typescript - * const s3object = await writeS3File(s3Object, "Hello Windmill!") - * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') - * console.log(fileContentAsUtf8Str) - * \`\`\` - */ -async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise - -/** - * Sign S3 objects to be used by anonymous users in public apps - * @param s3objects s3 objects to sign - * @returns signed s3 objects - */ -async signS3Objects(s3objects: S3Object[]): Promise - -/** - * Sign S3 object to be used by anonymous users in public apps - * @param s3object s3 object to sign - * @returns signed s3 object - */ -async signS3Object(s3object: S3Object): Promise - -/** - * Generate a presigned public URL for an array of S3 objects. - * If an S3 object is not signed yet, it will be signed first. - * @param s3Objects s3 objects to sign - * @returns list of signed public URLs - */ -async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. - * @param s3Object s3 object to sign - * @returns signed public URL - */ -async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise - -/** - * Get URLs needed for resuming a flow after this step - * @param approver approver name - * @returns approval page UI URL, resume and cancel API URLs for resuming the flow - */ -async getResumeUrls(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * @deprecated use getResumeUrls instead - */ -getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; -}> - -/** - * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) - * @param audience audience of the token - * @param expiresIn Optional number of seconds until the token expires - * @returns jwt token - */ -async getIdToken(audience: string, expiresIn?: number): Promise - -/** - * Convert a base64-encoded string to Uint8Array - * @param data - Base64-encoded string - * @returns Decoded Uint8Array - */ -base64ToUint8Array(data: string): Uint8Array - -/** - * Convert a Uint8Array to base64-encoded string - * @param arrayBuffer - Uint8Array to encode - * @returns Base64-encoded string - */ -uint8ArrayToBase64(arrayBuffer: Uint8Array): string - -/** - * Get email from workspace username - * This method is particularly useful for apps that require the email address of the viewer. - * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. - * @param username - * @returns email address - */ -async usernameToEmail(username: string): Promise - -/** - * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Slack approval request. - * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. - * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Slack approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Slack approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveSlackApproval({ - * slackResourcePath: "/u/alex/my_slack_resource", - * channelId: "admins-slack-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise - -/** - * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. - * - * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** - * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). - * - * @param {Object} options - The configuration options for the Teams approval request. - * @param {string} options.teamName - The Teams team name where the approval request will be sent. - * @param {string} options.channelName - The Teams channel name where the approval request will be sent. - * @param {string} [options.message] - Optional custom message to include in the Teams approval request. - * @param {string} [options.approver] - Optional user ID or name of the approver for the request. - * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. - * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. - * - * @returns {Promise} Resolves when the Teams approval request is successfully sent. - * - * @throws {Error} If the function is not called within a flow or flow preview. - * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. - * - * **Usage Example:** - * \`\`\`typescript - * await requestInteractiveTeamsApproval({ - * teamName: "admins-teams", - * channelName: "admins-teams-channel", - * message: "Please approve this request", - * approver: "approver123", - * defaultArgsJson: { key1: "value1", key2: 42 }, - * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, - * }); - * \`\`\` - * - * **Note:** This function requires execution within a Windmill flow or flow preview. - */ -async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise - -/** - * Parse an S3 object from URI string or record format - * @param s3Object - S3 object as URI string (s3://storage/key) or record - * @returns S3 object record with storage and s3 key - */ -parseS3Object(s3Object: S3Object): S3ObjectRecord - -/** - * Create a SQL template function for PostgreSQL/datatable queries - * @param name - Database/datatable name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.datatable() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age}::int - * \`.fetch() - */ -datatable(name: string = "main"): DatatableSqlTemplateFunction - -/** - * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") - * @returns SQL template function for building parameterized queries - * @example - * let sql = wmill.ducklake() - * let name = 'Robin' - * let age = 21 - * await sql\` - * SELECT * FROM friends - * WHERE name = \${name} AND age = \${age} - * \`.fetch() - */ -ducklake(name: string = "main"): SqlTemplateFunction - -async polarsConnectionSettings(s3_resource_path: string | undefined): Promise - -async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise - - -# Python SDK (wmill) - -Import: import wmill - -def get_mocked_api() -> Optional[dict] - -# Get the HTTP client instance. -# -# Returns: -# Configured httpx.Client for API requests -def get_client() -> httpx.Client - -# Make an HTTP GET request to the Windmill API. -# -# Args: -# endpoint: API endpoint path -# raise_for_status: Whether to raise an exception on HTTP errors -# **kwargs: Additional arguments passed to httpx.get -# -# Returns: -# HTTP response object -def get(endpoint, raise_for_status = True, **kwargs) -> httpx.Response - -# Make an HTTP POST request to the Windmill API. -# -# Args: -# endpoint: API endpoint path -# raise_for_status: Whether to raise an exception on HTTP errors -# **kwargs: Additional arguments passed to httpx.post -# -# Returns: -# HTTP response object -def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response - -# Create a new authentication token. -# -# Args: -# duration: Token validity duration (default: 1 day) -# -# Returns: -# New authentication token string -def create_token(duration = dt.timedelta(days=1)) -> str - -# Create a script job and return its job id. -# -# .. deprecated:: Use run_script_by_path_async or run_script_by_hash_async instead. -def run_script_async(path: str = None, hash_: str = None, args: dict = None, scheduled_in_secs: int = None) -> str - -# Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None) -> str - -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None) -> str - -# Create a flow job and return its job id. -def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True) -> str - -# Run script synchronously and return its result. -# -# .. deprecated:: Use run_script_by_path or run_script_by_hash instead. -def run_script(path: str = None, hash_: str = None, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any - -# Run script by path synchronously and return its result. -def run_script_by_path(path: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any - -# Run script by hash synchronously and return its result. -def run_script_by_hash(hash_: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any - -# Run a script on the current worker without creating a job -def run_inline_script_preview(content: str, language: str, args: dict = None) -> Any - -# Wait for a job to complete and return its result. -# -# Args: -# job_id: ID of the job to wait for -# timeout: Maximum time to wait (seconds or timedelta) -# verbose: Enable verbose logging -# cleanup: Register cleanup handler to cancel job on exit -# assert_result_is_not_none: Raise exception if result is None -# -# Returns: -# Job result when completed -# -# Raises: -# TimeoutError: If timeout is reached -# Exception: If job fails -def wait_job(job_id, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) - -# Cancel a specific job by ID. -# -# Args: -# job_id: UUID of the job to cancel -# reason: Optional reason for cancellation -# -# Returns: -# Response message from the cancel endpoint -def cancel_job(job_id: str, reason: str = None) -> str - -# Cancel currently running executions of the same script. -def cancel_running() -> dict - -# Get job details by ID. -# -# Args: -# job_id: UUID of the job -# -# Returns: -# Job details dictionary -def get_job(job_id: str) -> dict - -# Get the root job ID for a flow hierarchy. -# -# Args: -# job_id: Job ID (defaults to current WM_JOB_ID) -# -# Returns: -# Root job ID -def get_root_job_id(job_id: str | None = None) -> dict - -# Get an OIDC JWT token for authentication to external services. -# -# Args: -# audience: Token audience (e.g., "vault", "aws") -# expires_in: Optional expiration time in seconds -# -# Returns: -# JWT token string -def get_id_token(audience: str, expires_in: int | None = None) -> str - -# Get the status of a job. -# -# Args: -# job_id: UUID of the job -# -# Returns: -# Job status: "RUNNING", "WAITING", or "COMPLETED" -def get_job_status(job_id: str) -> JobStatus - -# Get the result of a completed job. -# -# Args: -# job_id: UUID of the completed job -# assert_result_is_not_none: Raise exception if result is None -# -# Returns: -# Job result -def get_result(job_id: str, assert_result_is_not_none: bool = True) -> Any - -# Get a variable value by path. -# -# Args: -# path: Variable path in Windmill -# -# Returns: -# Variable value as string -def get_variable(path: str) -> str - -# Set a variable value by path, creating it if it doesn't exist. -# -# Args: -# path: Variable path in Windmill -# value: Variable value to set -# is_secret: Whether the variable should be secret (default: False) -def set_variable(path: str, value: str, is_secret: bool = False) -> None - -# Get a resource value by path. -# -# Args: -# path: Resource path in Windmill -# none_if_undefined: Return None instead of raising if not found -# -# Returns: -# Resource value dictionary or None -def get_resource(path: str, none_if_undefined: bool = False) -> dict | None - -# Set a resource value by path, creating it if it doesn't exist. -# -# Args: -# value: Resource value to set -# path: Resource path in Windmill -# resource_type: Resource type for creation -def set_resource(value: Any, path: str, resource_type: str) - -# List resources from Windmill workspace. -# -# Args: -# resource_type: Optional resource type to filter by (e.g., "postgresql", "mysql", "s3") -# page: Optional page number for pagination -# per_page: Optional number of results per page -# -# Returns: -# List of resource dictionaries -def list_resources(resource_type: str = None, page: int = None, per_page: int = None) -> list[dict] - -# Set the workflow state. -# -# Args: -# value: State value to set -def set_state(value: Any) - -# Set job progress percentage (0-99). -# -# Args: -# value: Progress percentage -# job_id: Job ID (defaults to current WM_JOB_ID) -def set_progress(value: int, job_id: Optional[str] = None) - -# Get job progress percentage. -# -# Args: -# job_id: Job ID (defaults to current WM_JOB_ID) -# -# Returns: -# Progress value (0-100) or None if not set -def get_progress(job_id: Optional[str] = None) -> Any - -# Set the user state of a flow at a given key -def set_flow_user_state(key: str, value: Any) -> None - -# Get the user state of a flow at a given key -def get_flow_user_state(key: str) -> Any - -# Get the Windmill server version. -# -# Returns: -# Version string -def version() - -# Convenient helpers that takes an S3 resource as input and returns the settings necessary to -# initiate an S3 connection from DuckDB -def get_duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings | None - -# Convenient helpers that takes an S3 resource as input and returns the settings necessary to -# initiate an S3 connection from Polars -def get_polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings - -# Convenient helpers that takes an S3 resource as input and returns the settings necessary to -# initiate an S3 connection using boto3 -def get_boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings - -# Load a file from the workspace s3 bucket and returns its content as bytes. -# -# '''python -# from wmill import S3Object -# -# s3_obj = S3Object(s3="/path/to/my_file.txt") -# my_obj_content = client.load_s3_file(s3_obj) -# file_content = my_obj_content.decode("utf-8") -# ''' -def load_s3_file(s3object: S3Object | str, s3_resource_path: str | None) -> bytes - -# Load a file from the workspace s3 bucket and returns the bytes stream. -# -# '''python -# from wmill import S3Object -# -# s3_obj = S3Object(s3="/path/to/my_file.txt") -# with wmill.load_s3_file_reader(s3object, s3_resource_path) as file_reader: -# print(file_reader.read()) -# ''' -def load_s3_file_reader(s3object: S3Object | str, s3_resource_path: str | None) -> BufferedReader - -# Write a file to the workspace S3 bucket -# -# '''python -# from wmill import S3Object -# -# s3_obj = S3Object(s3="/path/to/my_file.txt") -# -# # for an in memory bytes array: -# file_content = b'Hello Windmill!' -# client.write_s3_file(s3_obj, file_content) -# -# # for a file: -# with open("my_file.txt", "rb") as my_file: -# client.write_s3_file(s3_obj, my_file) -# ''' -def write_s3_file(s3object: S3Object | str | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, content_type: str | None = None, content_disposition: str | None = None) -> S3Object - -# Sign S3 objects for use by anonymous users in public apps. -# -# Args: -# s3_objects: List of S3 objects to sign -# -# Returns: -# List of signed S3 objects -def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object] - -# Sign a single S3 object for use by anonymous users in public apps. -# -# Args: -# s3_object: S3 object to sign -# -# Returns: -# Signed S3 object -def sign_s3_object(s3_object: S3Object | str) -> S3Object - -# Generate presigned public URLs for an array of S3 objects. -# If an S3 object is not signed yet, it will be signed first. -# -# Args: -# s3_objects: List of S3 objects to sign -# base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL) -# -# Returns: -# List of signed public URLs -# -# Example: -# >>> s3_objs = [S3Object(s3="/path/to/file1.txt"), S3Object(s3="/path/to/file2.txt")] -# >>> urls = client.get_presigned_s3_public_urls(s3_objs) -def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None) -> list[str] - -# Generate a presigned public URL for an S3 object. -# If the S3 object is not signed yet, it will be signed first. -# -# Args: -# s3_object: S3 object to sign -# base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL) -# -# Returns: -# Signed public URL -# -# Example: -# >>> s3_obj = S3Object(s3="/path/to/file.txt") -# >>> url = client.get_presigned_s3_public_url(s3_obj) -def get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None) -> str - -# Get the current user information. -# -# Returns: -# User details dictionary -def whoami() -> dict - -# Get the current user information (alias for whoami). -# -# Returns: -# User details dictionary -def user() -> dict - -# Get the state resource path from environment. -# -# Returns: -# State path string -def state_path() -> str - -# Get the workflow state. -# -# Returns: -# State value or None if not set -def state() -> Any - -# Set the state in the shared folder using pickle -def set_shared_state_pickle(value: Any, path: str = 'state.pickle') -> None - -# Get the state in the shared folder using pickle -def get_shared_state_pickle(path: str = 'state.pickle') -> Any - -# Set the state in the shared folder using pickle -def set_shared_state(value: Any, path: str = 'state.json') -> None - -# Get the state in the shared folder using pickle -def get_shared_state(path: str = 'state.json') -> None - -# Get URLs needed for resuming a flow after suspension. -# -# Args: -# approver: Optional approver name -# -# Returns: -# Dictionary with approvalPage, resume, and cancel URLs -def get_resume_urls(approver: str = None) -> dict - -# Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. -# -# **[Enterprise Edition Only]** To include form fields in the Slack approval request, use the "Advanced -> Suspend -> Form" functionality. -# Learn more at: https://www.windmill.dev/docs/flows/flow_approval#form -# -# :param slack_resource_path: The path to the Slack resource in Windmill. -# :type slack_resource_path: str -# :param channel_id: The Slack channel ID where the approval request will be sent. -# :type channel_id: str -# :param message: Optional custom message to include in the Slack approval request. -# :type message: str, optional -# :param approver: Optional user ID or name of the approver for the request. -# :type approver: str, optional -# :param default_args_json: Optional dictionary defining or overriding the default arguments for form fields. -# :type default_args_json: dict, optional -# :param dynamic_enums_json: Optional dictionary overriding the enum default values of enum form fields. -# :type dynamic_enums_json: dict, optional -# -# :raises Exception: If the function is not called within a flow or flow preview. -# :raises Exception: If the required flow job or flow step environment variables are not set. -# -# :return: None -# -# **Usage Example:** -# >>> client.request_interactive_slack_approval( -# ... slack_resource_path="/u/alex/my_slack_resource", -# ... channel_id="admins-slack-channel", -# ... message="Please approve this request", -# ... approver="approver123", -# ... default_args_json={"key1": "value1", "key2": 42}, -# ... dynamic_enums_json={"foo": ["choice1", "choice2"], "bar": ["optionA", "optionB"]}, -# ... ) -# -# **Notes:** -# - This function must be executed within a Windmill flow or flow preview. -# - The function checks for required environment variables (\`WM_FLOW_JOB_ID\`, \`WM_FLOW_STEP_ID\`) to ensure it is run in the appropriate context. -def request_interactive_slack_approval(slack_resource_path: str, channel_id: str, message: str = None, approver: str = None, default_args_json: dict = None, dynamic_enums_json: dict = None) -> None - -# Get email from workspace username -# This method is particularly useful for apps that require the email address of the viewer. -# Indeed, in the viewer context WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. -def username_to_email(username: str) -> str - -# Send a message to a Microsoft Teams conversation with conversation_id, where success is used to style the message -def send_teams_message(conversation_id: str, text: str, success: bool = True, card_block: dict = None) - -# Get a DataTable client for SQL queries. -# -# Args: -# name: Database name (default: "main") -# -# Returns: -# DataTableClient instance -def datatable(name: str = 'main') - -# Get a DuckLake client for DuckDB queries. -# -# Args: -# name: Database name (default: "main") -# -# Returns: -# DucklakeClient instance -def ducklake(name: str = 'main') - -def init_global_client(f) - -def deprecate(in_favor_of: str) - -# Get the current workspace ID. -# -# Returns: -# Workspace ID string -def get_workspace() -> str - -def get_version() -> str - -# Run a script synchronously by hash and return its result. -# -# Args: -# hash: Script hash -# args: Script arguments -# verbose: Enable verbose logging -# assert_result_is_not_none: Raise exception if result is None -# cleanup: Register cleanup handler to cancel job on exit -# timeout: Maximum time to wait -# -# Returns: -# Script result -def run_script_sync(hash: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None) -> Any - -# Run a script synchronously by path and return its result. -# -# Args: -# path: Script path -# args: Script arguments -# verbose: Enable verbose logging -# assert_result_is_not_none: Raise exception if result is None -# cleanup: Register cleanup handler to cancel job on exit -# timeout: Maximum time to wait -# -# Returns: -# Script result -def run_script_by_path_sync(path: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None) -> Any - -# Convenient helpers that takes an S3 resource as input and returns the settings necessary to -# initiate an S3 connection from DuckDB -def duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings - -# Convenient helpers that takes an S3 resource as input and returns the settings necessary to -# initiate an S3 connection from Polars -def polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings - -# Convenient helpers that takes an S3 resource as input and returns the settings necessary to -# initiate an S3 connection using boto3 -def boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings - -# Get the state -def get_state() -> Any - -# Get the state resource path from environment. -# -# Returns: -# State path string -def get_state_path() -> str - -# Decorator to mark a function as a workflow task. -# -# When executed inside a Windmill job, the decorated function runs as a -# separate workflow step. Outside Windmill, it executes normally. -# -# Args: -# tag: Optional worker tag for execution -# -# Returns: -# Decorated function -def task(*args, **kwargs) - -# Parse resource syntax from string. -def parse_resource_syntax(s: str) -> Optional[str] - -# Parse S3 object from string or S3Object format. -def parse_s3_object(s3_object: S3Object | str) -> S3Object - -# Parse variable syntax from string. -def parse_variable_syntax(s: str) -> Optional[str] - -# Append a text to the result stream. -# -# Args: -# text: text to append to the result stream -def append_to_result_stream(text: str) -> None - -# Stream to the result stream. -# -# Args: -# stream: stream to stream to the result stream -def stream_result(stream) -> None - -# Execute a SQL query against the DataTable. -# -# Args: -# sql: SQL query string with $1, $2, etc. placeholders -# *args: Positional arguments to bind to query placeholders -# -# Returns: -# SqlQuery instance for fetching results -def query(sql: str, *args) - -# Execute query and fetch results. -# -# Args: -# result_collection: Optional result collection mode -# -# Returns: -# Query results -def fetch(result_collection: str | None = None) - -# Execute query and fetch first row of results. -# -# Returns: -# First row of query results -def fetch_one() - -# DuckDB executor requires explicit argument types at declaration -# These types exist in both DuckDB and Postgres -# Check that the types exist if you plan to extend this function for other SQL engines. -def infer_sql_type(value) -> str - -`; - -export const FLOW_PROMPT = `# Windmill Flow Building Guide - -The OpenFlow schema (openflow.openapi.yaml) is the source of truth for flow structure. Refer to OPENFLOW_SCHEMA for the complete type definitions. - -## Reserved Module IDs - -- \`failure\` - Reserved for failure handler module -- \`preprocessor\` - Reserved for preprocessor module -- \`Input\` - Reserved for flow input reference - -## Module ID Rules - -- Must be unique across the entire flow -- Use underscores, not spaces (e.g., \`fetch_data\` not \`fetch data\`) -- Use descriptive names that reflect the step's purpose - -## Common Mistakes to Avoid - -- Missing \`input_transforms\` - Rawscript parameters won't receive values without them -- Referencing future steps - \`results.step_id\` only works for steps that execute before the current one -- Duplicate module IDs - Each module ID must be unique in the flow - -## Data Flow Between Steps - -- \`flow_input.property\` - Access flow input parameters -- \`results.step_id\` - Access output from a previous step -- \`results.step_id.property\` - Access specific property from previous step output -- \`flow_input.iter.value\` - Current item when inside a for-loop -- \`flow_input.iter.index\` - Current index when inside a for-loop - -## Input Transforms - -Every rawscript module needs \`input_transforms\` to map function parameters to values: - -Static transform (fixed value): -{"param_name": {"type": "static", "value": "fixed_string"}} - -JavaScript transform (dynamic expression): -{"param_name": {"type": "javascript", "expr": "results.previous_step.data"}} - -## Resource References - -- For flow inputs: Use type \`"object"\` with format \`"resource-{type}"\` (e.g., \`"resource-postgresql"\`) -- For step inputs: Use static value \`"$res:path/to/resource"\` - -## Failure Handler - -Executes when any step fails. Has access to error details: - -- \`error.message\` - Error message -- \`error.step_id\` - ID of failed step -- \`error.name\` - Error name -- \`error.stack\` - Stack trace - -## S3 Object Operations - -Windmill provides built-in support for S3-compatible storage operations. - -To accept an S3 object as flow input: - -\`\`\`json -{ - "type": "object", - "properties": { - "file": { - "type": "object", - "format": "resource-s3_object", - "description": "File to process" - } - } -} -\`\`\` - -## Using Resources in Flows - -On Windmill, credentials and configuration are stored in resources. Resource types define the format of the resource. - -### As Flow Input - -In the flow schema, set the property type to \`"object"\` with format \`"resource-{type}"\`: - -\`\`\`json -{ - "type": "object", - "properties": { - "database": { - "type": "object", - "format": "resource-postgresql", - "description": "Database connection" - } - } -} -\`\`\` - -### As Step Input (Static Reference) - -Reference a specific resource using \`$res:\` prefix: - -\`\`\`json -{ - "database": { - "type": "static", - "value": "$res:f/folder/my_database" - } -} -\`\`\` - - -## OpenFlow Schema - -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"number","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps","additionalProperties":{"type":"string"}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","description":"Custom error message shown when stopping"}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable"]},"access_type":{"type":"string","description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/InputTransform"},"output_type":{"$ref":"#/components/schemas/InputTransform"},"user_message":{"$ref":"#/components/schemas/InputTransform"},"system_prompt":{"$ref":"#/components/schemas/InputTransform"},"streaming":{"$ref":"#/components/schemas/InputTransform"},"messages_context_length":{"$ref":"#/components/schemas/InputTransform"},"output_schema":{"$ref":"#/components/schemas/InputTransform"},"user_images":{"$ref":"#/components/schemas/InputTransform"},"max_completion_tokens":{"$ref":"#/components/schemas/InputTransform"},"temperature":{"$ref":"#/components/schemas/InputTransform"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; - -export const CLI_COMMANDS = `# Windmill CLI Commands - -The Windmill CLI (\`wmill\`) provides commands for managing scripts, flows, apps, and other resources. - -Current version: 1.591.2 - -## Global Options - -- \`--workspace \` - Specify the target workspace. This overrides the default workspace. -- \`--debug --verbose\` - Show debug/verbose logs -- \`--show-diffs\` - Show diff informations when syncing (may show sensitive informations) -- \`--token \` - Specify an API token. This will override any stored token. -- \`--base-url \` - Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used. -- \`--config-dir \` - Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location. - -## Commands - -### app - -app related commands - -**Subcommands:** - -- \`app push \` - push a local app -- \`app generate-locks [app_folder:string]\` - re-generate the lockfiles for app runnables inline scripts that have changed - - \`--yes\` - Skip confirmation prompt - - \`--dry-run\` - Perform a dry run without making changes - - \`--default-ts \` - Default TypeScript runtime (bun or deno) - -### dependencies - -workspace dependencies related commands - -**Alias:** \`deps\` - -**Subcommands:** - -- \`dependencies push \` - Push workspace dependencies from a local file - - \`--language \` - Programming language (python3, typescript, go, php). If not specified, will be inferred from file extension. - - \`--name \` - Name for the dependencies. If not specified, creates workspace default dependencies. - -### dev - -Launch a dev server that will spawn a webserver with HMR - -**Options:** -- \`--includes \` - Filter paths givena glob pattern or path - -### flow - -flow related commands - -**Options:** -- \`--show-archived\` - Enable archived scripts in output - -**Subcommands:** - -- \`flow push \` - push a local flow spec. This overrides any remote versions. -- \`flow run \` - run a flow by path. - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. -- \`flow generate-locks [flow:file]\` - re-generate the lock files of all inline scripts of all updated flows - - \`--yes\` - Skip confirmation prompt - - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. -- \`flow bootstrap \` - create a new empty flow - - \`--summary \` - script summary - - \`--description \` - script description - -### folder - -folder related commands - -**Subcommands:** - -- \`folder push \` - push a local folder spec. This overrides any remote versions. - -### gitsync-settings - -Manage git-sync settings between local wmill.yaml and Windmill backend - -**Subcommands:** - -- \`gitsync-settings pull\` - Pull git-sync settings from Windmill backend to local wmill.yaml - - \`--replace\` - Replace existing settings (non-interactive mode) - - \`--diff\` - Show differences without applying changes - - \`--json-output\` - Output in JSON format - - \`--yes\` - Skip interactive prompts and use default behavior - - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides -- \`gitsync-settings push\` - Push git-sync settings from local wmill.yaml to Windmill backend - - \`--diff\` - Show what would be pushed without applying changes - - \`--json-output\` - Output in JSON format - - \`--yes\` - Skip interactive prompts and use default behavior - - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides - -### hub - -Hub related commands. EXPERIMENTAL. INTERNAL USE ONLY. - -**Subcommands:** - -- \`hub pull\` - pull any supported definitions. EXPERIMENTAL. - -### init - -Bootstrap a windmill project with a wmill.yaml file - -**Options:** -- \`--use-default\` - Use default settings without checking backend -- \`--use-backend\` - Use backend git-sync settings if available -- \`--repository \` - Specify repository path (e.g., u/user/repo) when using backend settings -- \`--bind-profile\` - Automatically bind active workspace profile to current Git branch -- \`--no-bind-profile\` - Skip workspace profile binding prompt - -### instance - -sync local with a remote instance or the opposite (push or pull) - -**Subcommands:** - -- \`instance add [instance_name:string] [remote:string] [token:string]\` - Add a new instance -- \`instance remove \` - Remove an instance -- \`instance switch \` - Switch the current instance -- \`instance pull\` - Pull instance settings, users, configs, instance groups and overwrite local - - \`--yes\` - Pull without needing confirmation - - \`--dry-run\` - Perform a dry run without making changes - - \`--skip-users\` - Skip pulling users - - \`--skip-settings\` - Skip pulling settings - - \`--skip-configs\` - Skip pulling configs (worker groups and SMTP) - - \`--skip-groups\` - Skip pulling instance groups - - \`--include-workspaces\` - Also pull workspaces - - \`--folder-per-instance\` - Create a folder per instance - - \`--instance \` - Name of the instance to pull from, override the active instance - - \`--prefix \` - Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces - - \`--prefix-settings\` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance -- \`instance push\` - Push instance settings, users, configs, group and overwrite remote - - \`--yes\` - Push without needing confirmation - - \`--dry-run\` - Perform a dry run without making changes - - \`--skip-users\` - Skip pushing users - - \`--skip-settings\` - Skip pushing settings - - \`--skip-configs\` - Skip pushing configs (worker groups and SMTP) - - \`--skip-groups\` - Skip pushing instance groups - - \`--include-workspaces\` - Also push workspaces - - \`--folder-per-instance\` - Create a folder per instance - - \`--instance \` - Name of the instance to push to, override the active instance - - \`--prefix \` - Prefix of the local workspaces folders to push - - \`--prefix-settings\` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance -- \`instance whoami\` - Display information about the currently logged-in user - -### jobs - -Pull completed and queued jobs from workspace - -**Arguments:** \`[workspace:string]\` - -**Options:** -- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) -- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) -- \`--skip-worker-check\` - Skip checking for active workers before export - -### queues - -List all queues with their metrics - -**Arguments:** \`[workspace:string] the optional workspace to filter by (default to all workspaces)\` - -**Options:** -- \`--instance [instance]\` - Name of the instance to push to, override the active instance -- \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance - -### resource - -resource related commands - -**Subcommands:** - -- \`resource push \` - push a local resource spec. This overrides any remote versions. - -### resource-type - -resource type related commands - -**Subcommands:** - -- \`resource-type list\` - list all resource types - - \`--schema\` - Show schema in the output -- \`resource-type push \` - push a local resource spec. This overrides any remote versions. -- \`resource-type generate-namespace\` - Create a TypeScript definition file with the RT namespace generated from the resource types - -### schedule - -schedule related commands - -**Subcommands:** - -- \`schedule push \` - push a local schedule spec. This overrides any remote versions. - -### script - -script related commands - -**Options:** -- \`--show-archived\` - Enable archived scripts in output - -**Subcommands:** - -- \`script push \` - push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh) -- \`script show \` - show a scripts content -- \`script run \` - run a script by path - - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. - - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. -- \`script bootstrap \` - create a new script - - \`--summary \` - script summary - - \`--description \` - script description -- \`script generate-metadata [script:file]\` - re-generate the metadata file updating the lock and the script schema (for flows, use \`wmill flow generate-locks\`) - - \`--yes\` - Skip confirmation prompt - - \`--dry-run\` - Perform a dry run without making changes - - \`--lock-only\` - re-generate only the lock - - \`--schema-only\` - re-generate only script schema - - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. - -### sync - -sync local with a remote workspaces or the opposite (push or pull) - -**Subcommands:** - -- \`sync pull\` - Pull any remote changes and apply them locally. - - \`--yes\` - Pull without needing confirmation - - \`--dry-run\` - Show changes that would be pulled without actually pushing - - \`--plain-secrets\` - Pull secrets as plain text - - \`--json\` - Use JSON instead of YAML - - \`--skip-variables\` - Skip syncing variables (including secrets) - - \`--skip-secrets\` - Skip syncing only secrets variables - - \`--skip-resources\` - Skip syncing resources - - \`--skip-resource-types\` - Skip syncing resource types - - \`--skip-scripts\` - Skip syncing scripts - - \`--skip-flows\` - Skip syncing flows - - \`--skip-apps\` - Skip syncing apps - - \`--skip-folders\` - Skip syncing folders - - \`--skip-workspace-dependencies\` - Skip syncing workspace dependencies - - \`--skip-scripts-metadata\` - Skip syncing scripts metadata, focus solely on logic - - \`--include-schedules\` - Include syncing schedules - - \`--include-triggers\` - Include syncing triggers - - \`--include-users\` - Include syncing users - - \`--include-groups\` - Include syncing groups - - \`--include-settings\` - Include syncing workspace settings - - \`--include-key\` - Include workspace encryption key - - \`--skip-branch-validation\` - Skip git branch validation and prompts - - \`--json-output\` - Output results in JSON format - - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. Overrides wmill.yaml excludes - - \`--repository \` - Specify repository path (e.g., u/user/repo) when multiple repositories exist - - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides -- \`sync push\` - Push any local changes and apply them remotely. - - \`--yes\` - Push without needing confirmation - - \`--dry-run\` - Show changes that would be pushed without actually pushing - - \`--plain-secrets\` - Push secrets as plain text - - \`--json\` - Use JSON instead of YAML - - \`--skip-variables\` - Skip syncing variables (including secrets) - - \`--skip-secrets\` - Skip syncing only secrets variables - - \`--skip-resources\` - Skip syncing resources - - \`--skip-resource-types\` - Skip syncing resource types - - \`--skip-scripts\` - Skip syncing scripts - - \`--skip-flows\` - Skip syncing flows - - \`--skip-apps\` - Skip syncing apps - - \`--skip-folders\` - Skip syncing folders - - \`--skip-workspace-dependencies\` - Skip syncing workspace dependencies - - \`--skip-scripts-metadata\` - Skip syncing scripts metadata, focus solely on logic - - \`--include-schedules\` - Include syncing schedules - - \`--include-triggers\` - Include syncing triggers - - \`--include-users\` - Include syncing users - - \`--include-groups\` - Include syncing groups - - \`--include-settings\` - Include syncing workspace settings - - \`--include-key\` - Include workspace encryption key - - \`--skip-branch-validation\` - Skip git branch validation and prompts - - \`--json-output\` - Output results in JSON format - - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. - - \`--message \` - Include a message that will be added to all scripts/flows/apps updated during this push - - \`--parallel \` - Number of changes to process in parallel - - \`--repository \` - Specify repository path (e.g., u/user/repo) when multiple repositories exist - -### trigger - -trigger related commands - -**Subcommands:** - -- \`trigger push \` - push a local trigger spec. This overrides any remote versions. - -### user - -user related commands - -**Subcommands:** - -- \`user add [password:string]\` - Create a user - - \`--superadmin\` - Specify to make the new user superadmin. - - \`--company \` - Specify to set the company of the new user. - - \`--name \` - Specify to set the name of the new user. -- \`user remove \` - Delete a user -- \`user create-token\` - -### variable - -variable related commands - -**Subcommands:** - -- \`variable push \` - Push a local variable spec. This overrides any remote versions. - - \`--plain-secrets\` - Push secrets as plain text -- \`variable add \` - Create a new variable on the remote. This will update the variable if it already exists. - - \`--plain-secrets\` - Push secrets as plain text - - \`--public\` - Legacy option, use --plain-secrets instead - -### version - -Show version information - -### worker-groups - -display worker groups, pull and push worker groups configs - -**Subcommands:** - -- \`worker-groups pull\` - Pull worker groups (similar to \`wmill instance pull --skip-users --skip-settings --skip-groups\`) - - \`--instance\` - Name of the instance to push to, override the active instance - - \`--base-url\` - Base url to be passed to the instance settings instead of the local one - - \`--yes\` - Pull without needing confirmation -- \`worker-groups push\` - Push instance settings, users, configs, group and overwrite remote - - \`--instance [instance]\` - Name of the instance to push to, override the active instance - - \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance - - \`--yes\` - Push without needing confirmation - -### workers - -List all workers grouped by worker groups - -**Options:** -- \`--instance [instance]\` - Name of the instance to push to, override the active instance -- \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance - -### workspace - -workspace related commands - -**Alias:** \`profile\` - -**Subcommands:** - -- \`workspace switch \` - Switch to another workspace -- \`workspace add [workspace_name:string] [workspace_id:string] [remote:string]\` - Add a workspace - - \`-c --create\` - Create the workspace if it does not exist - - \`--create-workspace-name \` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id. -- \`workspace remove \` - Remove a workspace -- \`workspace whoami\` - Show the currently active user -- \`workspace bind\` - Bind the current Git branch to the active workspace - - \`--branch \` - Specify branch (defaults to current) -- \`workspace unbind\` - Remove workspace binding from the current Git branch - - \`--branch \` - Specify branch (defaults to current) -- \`workspace fork [workspace_name:string] [workspace_id:string]\` - Create a forked workspace - - \`--create-workspace-name \` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id. -- \`workspace delete-fork \` - Delete a forked workspace and git branch - - \`-y --yes\` - Skip confirmation prompt - -`; diff --git a/cli/src/guidance/script_guidance.ts b/cli/src/guidance/script_guidance.ts deleted file mode 100644 index c22ae34b98..0000000000 --- a/cli/src/guidance/script_guidance.ts +++ /dev/null @@ -1,15 +0,0 @@ -// CLI Script Guidance - Uses centralized prompts from system_prompts/ -import * as prompts from "./prompts.ts"; - -// CLI-specific introduction -const CLI_INTRO = `Each script should be placed in a folder. Ask the user in which folder he wants the script to be located at before starting coding. -After writing a script, you do not need to create .lock and .yaml files manually. Instead, you can run \`wmill script generate-metadata\` bash command. This command takes no arguments. After writing the script, you can ask the user if he wants to push the script with \`wmill sync push\`. Both should be run at the root of the repository. - -You can use \`wmill resource-type list --schema\` to list all resource types available. You should use that to know the type of the resource you need to use in your script. You can use grep if the output is too long.`; - -// Assemble complete script guidance -export const SCRIPT_GUIDANCE = ` -${CLI_INTRO} - -${prompts.SCRIPT_PROMPT} -`; diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts new file mode 100644 index 0000000000..19f455fadb --- /dev/null +++ b/cli/src/guidance/skills.ts @@ -0,0 +1,5759 @@ +// Auto-generated by generate.py - DO NOT EDIT + +export interface SkillMetadata { + name: string; + description: string; + languageKey?: string; +} + +export const SKILLS: SkillMetadata[] = [ + { name: "write-script-python3", description: "MUST use when writing Python scripts.", languageKey: "python3" }, + { name: "write-script-bun", description: "MUST use when writing Bun/TypeScript scripts.", languageKey: "bun" }, + { name: "write-script-mysql", description: "MUST use when writing MySQL queries.", languageKey: "mysql" }, + { name: "write-script-powershell", description: "MUST use when writing PowerShell scripts.", languageKey: "powershell" }, + { name: "write-script-snowflake", description: "MUST use when writing Snowflake queries.", languageKey: "snowflake" }, + { name: "write-script-go", description: "MUST use when writing Go scripts.", languageKey: "go" }, + { name: "write-script-deno", description: "MUST use when writing Deno/TypeScript scripts.", languageKey: "deno" }, + { name: "write-script-bash", description: "MUST use when writing Bash scripts.", languageKey: "bash" }, + { name: "write-script-bunnative", description: "MUST use when writing Bun Native scripts.", languageKey: "bunnative" }, + { name: "write-script-duckdb", description: "MUST use when writing DuckDB queries.", languageKey: "duckdb" }, + { name: "write-script-nativets", description: "MUST use when writing Native TypeScript scripts.", languageKey: "nativets" }, + { name: "write-script-bigquery", description: "MUST use when writing BigQuery queries.", languageKey: "bigquery" }, + { name: "write-script-rust", description: "MUST use when writing Rust scripts.", languageKey: "rust" }, + { name: "write-script-php", description: "MUST use when writing PHP scripts.", languageKey: "php" }, + { name: "write-script-mssql", description: "MUST use when writing MS SQL Server queries.", languageKey: "mssql" }, + { name: "write-script-postgresql", description: "MUST use when writing PostgreSQL queries.", languageKey: "postgresql" }, + { name: "write-script-graphql", description: "MUST use when writing GraphQL queries.", languageKey: "graphql" }, + { name: "write-script-csharp", description: "MUST use when writing C# scripts.", languageKey: "csharp" }, + { name: "write-script-java", description: "MUST use when writing Java scripts.", languageKey: "java" }, + { name: "write-flow", description: "MUST use when creating flows." }, + { name: "raw-app", description: "MUST use when creating raw apps." }, + { name: "triggers", description: "MUST use when configuring triggers." }, + { name: "schedules", description: "MUST use when configuring schedules." }, + { name: "resources", description: "MUST use when managing resources." }, + { name: "cli-commands", description: "MUST use when using the CLI." }, +]; + +// Skill content for each skill (loaded inline for bundling) +export const SKILL_CONTENT: Record = { + "write-script-python3": `--- +name: write-script-python3 +description: MUST use when writing Python scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Python + +## Structure + +The script must contain at least one function called \`main\`: + +\`\`\`python +def main(param1: str, param2: int): + # Your code here + return {"result": param1, "count": param2} +\`\`\` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +You need to **redefine** the type of the resources that are needed before the main function as TypedDict: + +\`\`\`python +from typing import TypedDict + +class postgresql(TypedDict): + host: str + port: int + user: str + password: str + dbname: str + +def main(db: postgresql): + # db contains the database connection details + pass +\`\`\` + +**Important rules:** + +- The resource type name must be **IN LOWERCASE** +- Only include resource types if they are actually needed +- If an import conflicts with a resource type name, **rename the imported object, not the type name** +- Make sure to import TypedDict from typing **if you're using it** + +## Imports + +Libraries are installed automatically. Do not show installation instructions. + +\`\`\`python +import requests +import pandas as pd +from datetime import datetime +\`\`\` + +If an import name conflicts with a resource type: + +\`\`\`python +# Wrong - don't rename the type +import stripe as stripe_lib +class stripe_type(TypedDict): ... + +# Correct - rename the import +import stripe as stripe_sdk +class stripe(TypedDict): + api_key: str +\`\`\` + +## Windmill Client + +Import the windmill client for platform interactions: + +\`\`\`python +import wmill +\`\`\` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`python +from typing import TypedDict, Literal, Any + +class Event(TypedDict): + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + body: Any + headers: dict[str, str] + query: dict[str, str] + +def preprocessor(event: Event): + # Transform the event into flow input parameters + return { + "param1": event["body"]["field1"], + "param2": event["query"]["id"] + } +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +\`\`\`python +import wmill + +# Load file content from S3 +content: bytes = wmill.load_s3_file(s3object) + +# Load file as stream reader +reader: BufferedReader = wmill.load_s3_file_reader(s3object) + +# Write file to S3 +result: S3Object = wmill.write_s3_file( + s3object, # Target path (or None to auto-generate) + file_content, # bytes or BufferedReader + s3_resource_path, # Optional: specific S3 resource + content_type, # Optional: MIME type + content_disposition # Optional: Content-Disposition header +) +\`\`\` + + +# Python SDK (wmill) + +Import: import wmill + +def get_mocked_api() -> Optional[dict] + +# Get the HTTP client instance. +# +# Returns: +# Configured httpx.Client for API requests +def get_client() -> httpx.Client + +# Make an HTTP GET request to the Windmill API. +# +# Args: +# endpoint: API endpoint path +# raise_for_status: Whether to raise an exception on HTTP errors +# **kwargs: Additional arguments passed to httpx.get +# +# Returns: +# HTTP response object +def get(endpoint, raise_for_status = True, **kwargs) -> httpx.Response + +# Make an HTTP POST request to the Windmill API. +# +# Args: +# endpoint: API endpoint path +# raise_for_status: Whether to raise an exception on HTTP errors +# **kwargs: Additional arguments passed to httpx.post +# +# Returns: +# HTTP response object +def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response + +# Create a new authentication token. +# +# Args: +# duration: Token validity duration (default: 1 day) +# +# Returns: +# New authentication token string +def create_token(duration = dt.timedelta(days=1)) -> str + +# Create a script job and return its job id. +# +# .. deprecated:: Use run_script_by_path_async or run_script_by_hash_async instead. +def run_script_async(path: str = None, hash_: str = None, args: dict = None, scheduled_in_secs: int = None) -> str + +# Create a script job by path and return its job id. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None) -> str + +# Create a script job by hash and return its job id. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None) -> str + +# Create a flow job and return its job id. +def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True) -> str + +# Run script synchronously and return its result. +# +# .. deprecated:: Use run_script_by_path or run_script_by_hash instead. +def run_script(path: str = None, hash_: str = None, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any + +# Run script by path synchronously and return its result. +def run_script_by_path(path: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any + +# Run script by hash synchronously and return its result. +def run_script_by_hash(hash_: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any + +# Run a script on the current worker without creating a job +def run_inline_script_preview(content: str, language: str, args: dict = None) -> Any + +# Wait for a job to complete and return its result. +# +# Args: +# job_id: ID of the job to wait for +# timeout: Maximum time to wait (seconds or timedelta) +# verbose: Enable verbose logging +# cleanup: Register cleanup handler to cancel job on exit +# assert_result_is_not_none: Raise exception if result is None +# +# Returns: +# Job result when completed +# +# Raises: +# TimeoutError: If timeout is reached +# Exception: If job fails +def wait_job(job_id, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) + +# Cancel a specific job by ID. +# +# Args: +# job_id: UUID of the job to cancel +# reason: Optional reason for cancellation +# +# Returns: +# Response message from the cancel endpoint +def cancel_job(job_id: str, reason: str = None) -> str + +# Cancel currently running executions of the same script. +def cancel_running() -> dict + +# Get job details by ID. +# +# Args: +# job_id: UUID of the job +# +# Returns: +# Job details dictionary +def get_job(job_id: str) -> dict + +# Get the root job ID for a flow hierarchy. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# Root job ID +def get_root_job_id(job_id: str | None = None) -> dict + +# Get an OIDC JWT token for authentication to external services. +# +# Args: +# audience: Token audience (e.g., "vault", "aws") +# expires_in: Optional expiration time in seconds +# +# Returns: +# JWT token string +def get_id_token(audience: str, expires_in: int | None = None) -> str + +# Get the status of a job. +# +# Args: +# job_id: UUID of the job +# +# Returns: +# Job status: "RUNNING", "WAITING", or "COMPLETED" +def get_job_status(job_id: str) -> JobStatus + +# Get the result of a completed job. +# +# Args: +# job_id: UUID of the completed job +# assert_result_is_not_none: Raise exception if result is None +# +# Returns: +# Job result +def get_result(job_id: str, assert_result_is_not_none: bool = True) -> Any + +# Get a variable value by path. +# +# Args: +# path: Variable path in Windmill +# +# Returns: +# Variable value as string +def get_variable(path: str) -> str + +# Set a variable value by path, creating it if it doesn't exist. +# +# Args: +# path: Variable path in Windmill +# value: Variable value to set +# is_secret: Whether the variable should be secret (default: False) +def set_variable(path: str, value: str, is_secret: bool = False) -> None + +# Get a resource value by path. +# +# Args: +# path: Resource path in Windmill +# none_if_undefined: Return None instead of raising if not found +# interpolated: if variables and resources are fully unrolled +# +# Returns: +# Resource value dictionary or None +def get_resource(path: str, none_if_undefined: bool = False, interpolated: bool = True) -> dict | None + +# Set a resource value by path, creating it if it doesn't exist. +# +# Args: +# value: Resource value to set +# path: Resource path in Windmill +# resource_type: Resource type for creation +def set_resource(value: Any, path: str, resource_type: str) + +# List resources from Windmill workspace. +# +# Args: +# resource_type: Optional resource type to filter by (e.g., "postgresql", "mysql", "s3") +# page: Optional page number for pagination +# per_page: Optional number of results per page +# +# Returns: +# List of resource dictionaries +def list_resources(resource_type: str = None, page: int = None, per_page: int = None) -> list[dict] + +# Set the workflow state. +# +# Args: +# value: State value to set +# path: Optional state resource path override. +def set_state(value: Any, path: str | None = None) -> None + +# Get the workflow state. +# +# Args: +# path: Optional state resource path override. +# +# Returns: +# State value or None if not set +def get_state(path: str | None = None) -> Any + +# Set job progress percentage (0-99). +# +# Args: +# value: Progress percentage +# job_id: Job ID (defaults to current WM_JOB_ID) +def set_progress(value: int, job_id: Optional[str] = None) + +# Get job progress percentage. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# Progress value (0-100) or None if not set +def get_progress(job_id: Optional[str] = None) -> Any + +# Set the user state of a flow at a given key +def set_flow_user_state(key: str, value: Any) -> None + +# Get the user state of a flow at a given key +def get_flow_user_state(key: str) -> Any + +# Get the Windmill server version. +# +# Returns: +# Version string +def version() + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from DuckDB +def get_duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings | None + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from Polars +def get_polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection using boto3 +def get_boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings + +# Load a file from the workspace s3 bucket and returns its content as bytes. +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# my_obj_content = client.load_s3_file(s3_obj) +# file_content = my_obj_content.decode("utf-8") +# ''' +def load_s3_file(s3object: S3Object | str, s3_resource_path: str | None) -> bytes + +# Load a file from the workspace s3 bucket and returns the bytes stream. +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# with wmill.load_s3_file_reader(s3object, s3_resource_path) as file_reader: +# print(file_reader.read()) +# ''' +def load_s3_file_reader(s3object: S3Object | str, s3_resource_path: str | None) -> BufferedReader + +# Write a file to the workspace S3 bucket +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# +# # for an in memory bytes array: +# file_content = b'Hello Windmill!' +# client.write_s3_file(s3_obj, file_content) +# +# # for a file: +# with open("my_file.txt", "rb") as my_file: +# client.write_s3_file(s3_obj, my_file) +# ''' +def write_s3_file(s3object: S3Object | str | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, content_type: str | None = None, content_disposition: str | None = None) -> S3Object + +# Sign S3 objects for use by anonymous users in public apps. +# +# Args: +# s3_objects: List of S3 objects to sign +# +# Returns: +# List of signed S3 objects +def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object] + +# Sign a single S3 object for use by anonymous users in public apps. +# +# Args: +# s3_object: S3 object to sign +# +# Returns: +# Signed S3 object +def sign_s3_object(s3_object: S3Object | str) -> S3Object + +# Generate presigned public URLs for an array of S3 objects. +# If an S3 object is not signed yet, it will be signed first. +# +# Args: +# s3_objects: List of S3 objects to sign +# base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL) +# +# Returns: +# List of signed public URLs +# +# Example: +# >>> s3_objs = [S3Object(s3="/path/to/file1.txt"), S3Object(s3="/path/to/file2.txt")] +# >>> urls = client.get_presigned_s3_public_urls(s3_objs) +def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None) -> list[str] + +# Generate a presigned public URL for an S3 object. +# If the S3 object is not signed yet, it will be signed first. +# +# Args: +# s3_object: S3 object to sign +# base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL) +# +# Returns: +# Signed public URL +# +# Example: +# >>> s3_obj = S3Object(s3="/path/to/file.txt") +# >>> url = client.get_presigned_s3_public_url(s3_obj) +def get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None) -> str + +# Get the current user information. +# +# Returns: +# User details dictionary +def whoami() -> dict + +# Get the current user information (alias for whoami). +# +# Returns: +# User details dictionary +def user() -> dict + +# Get the state resource path from environment. +# +# Returns: +# State path string +def state_path() -> str + +# Get the workflow state. +# +# Returns: +# State value or None if not set +def state() -> Any + +# Set the state in the shared folder using pickle +def set_shared_state_pickle(value: Any, path: str = 'state.pickle') -> None + +# Get the state in the shared folder using pickle +def get_shared_state_pickle(path: str = 'state.pickle') -> Any + +# Set the state in the shared folder using pickle +def set_shared_state(value: Any, path: str = 'state.json') -> None + +# Get the state in the shared folder using pickle +def get_shared_state(path: str = 'state.json') -> None + +# Get URLs needed for resuming a flow after suspension. +# +# Args: +# approver: Optional approver name +# flow_level: If True, generate resume URLs for the parent flow instead of the +# specific step. This allows pre-approvals that can be consumed by any later +# suspend step in the same flow. +# +# Returns: +# Dictionary with approvalPage, resume, and cancel URLs +def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict + +# Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. +# +# **[Enterprise Edition Only]** To include form fields in the Slack approval request, use the "Advanced -> Suspend -> Form" functionality. +# Learn more at: https://www.windmill.dev/docs/flows/flow_approval#form +# +# :param slack_resource_path: The path to the Slack resource in Windmill. +# :type slack_resource_path: str +# :param channel_id: The Slack channel ID where the approval request will be sent. +# :type channel_id: str +# :param message: Optional custom message to include in the Slack approval request. +# :type message: str, optional +# :param approver: Optional user ID or name of the approver for the request. +# :type approver: str, optional +# :param default_args_json: Optional dictionary defining or overriding the default arguments for form fields. +# :type default_args_json: dict, optional +# :param dynamic_enums_json: Optional dictionary overriding the enum default values of enum form fields. +# :type dynamic_enums_json: dict, optional +# +# :raises Exception: If the function is not called within a flow or flow preview. +# :raises Exception: If the required flow job or flow step environment variables are not set. +# +# :return: None +# +# **Usage Example:** +# >>> client.request_interactive_slack_approval( +# ... slack_resource_path="/u/alex/my_slack_resource", +# ... channel_id="admins-slack-channel", +# ... message="Please approve this request", +# ... approver="approver123", +# ... default_args_json={"key1": "value1", "key2": 42}, +# ... dynamic_enums_json={"foo": ["choice1", "choice2"], "bar": ["optionA", "optionB"]}, +# ... ) +# +# **Notes:** +# - This function must be executed within a Windmill flow or flow preview. +# - The function checks for required environment variables (\`WM_FLOW_JOB_ID\`, \`WM_FLOW_STEP_ID\`) to ensure it is run in the appropriate context. +def request_interactive_slack_approval(slack_resource_path: str, channel_id: str, message: str = None, approver: str = None, default_args_json: dict = None, dynamic_enums_json: dict = None) -> None + +# Get email from workspace username +# This method is particularly useful for apps that require the email address of the viewer. +# Indeed, in the viewer context WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. +def username_to_email(username: str) -> str + +# Send a message to a Microsoft Teams conversation with conversation_id, where success is used to style the message +def send_teams_message(conversation_id: str, text: str, success: bool = True, card_block: dict = None) + +# Get a DataTable client for SQL queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DataTableClient instance +def datatable(name: str = 'main') + +# Get a DuckLake client for DuckDB queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DucklakeClient instance +def ducklake(name: str = 'main') + +def init_global_client(f) + +def deprecate(in_favor_of: str) + +# Get the current workspace ID. +# +# Returns: +# Workspace ID string +def get_workspace() -> str + +def get_version() -> str + +# Run a script synchronously by hash and return its result. +# +# Args: +# hash: Script hash +# args: Script arguments +# verbose: Enable verbose logging +# assert_result_is_not_none: Raise exception if result is None +# cleanup: Register cleanup handler to cancel job on exit +# timeout: Maximum time to wait +# +# Returns: +# Script result +def run_script_sync(hash: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None) -> Any + +# Run a script synchronously by path and return its result. +# +# Args: +# path: Script path +# args: Script arguments +# verbose: Enable verbose logging +# assert_result_is_not_none: Raise exception if result is None +# cleanup: Register cleanup handler to cancel job on exit +# timeout: Maximum time to wait +# +# Returns: +# Script result +def run_script_by_path_sync(path: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None) -> Any + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from DuckDB +def duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from Polars +def polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection using boto3 +def boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings + +# Get the state resource path from environment. +# +# Returns: +# State path string +def get_state_path() -> str + +# Decorator to mark a function as a workflow task. +# +# When executed inside a Windmill job, the decorated function runs as a +# separate workflow step. Outside Windmill, it executes normally. +# +# Args: +# tag: Optional worker tag for execution +# +# Returns: +# Decorated function +def task(*args, **kwargs) + +# Parse resource syntax from string. +def parse_resource_syntax(s: str) -> Optional[str] + +# Parse S3 object from string or S3Object format. +def parse_s3_object(s3_object: S3Object | str) -> S3Object + +# Parse variable syntax from string. +def parse_variable_syntax(s: str) -> Optional[str] + +# Append a text to the result stream. +# +# Args: +# text: text to append to the result stream +def append_to_result_stream(text: str) -> None + +# Stream to the result stream. +# +# Args: +# stream: stream to stream to the result stream +def stream_result(stream) -> None + +# Execute a SQL query against the DataTable. +# +# Args: +# sql: SQL query string with $1, $2, etc. placeholders +# *args: Positional arguments to bind to query placeholders +# +# Returns: +# SqlQuery instance for fetching results +def query(sql: str, *args) -> SqlQuery + +# Execute query and fetch results. +# +# Args: +# result_collection: Optional result collection mode +# +# Returns: +# Query results +def fetch(result_collection: str | None = None) + +# Execute query and fetch first row of results. +# +# Returns: +# First row of query results +def fetch_one() + +# Execute query and fetch first row of results. Return result as a scalar value. +# +# Returns: +# First row of query result as a scalar value +def fetch_one_scalar() + +# Execute query and don't return any results. +# +def execute() + +# DuckDB executor requires explicit argument types at declaration +# These types exist in both DuckDB and Postgres +# Check that the types exist if you plan to extend this function for other SQL engines. +def infer_sql_type(value) -> str + +def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] + +`, + "write-script-bun": `--- +name: write-script-bun +description: MUST use when writing Bun/TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Bun) + +Bun runtime with full npm ecosystem and fastest execution. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +\`\`\`typescript +import Stripe from "stripe"; +import { someFunction } from "some-package"; +\`\`\` + +## Windmill Client + +Import the windmill client for platform interactions: + +\`\`\`typescript +import * as wmill from "windmill-client"; +\`\`\` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +\`\`\`typescript +type S3Object = { + s3: string; // Path within the bucket +}; +\`\`\` + +## TypeScript Operations + +\`\`\`typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction +`, + "write-script-mysql": `--- +name: write-script-mysql +description: MUST use when writing MySQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# MySQL + +Arguments use \`?\` placeholders. + +Name the parameters by adding comments before the statement: + +\`\`\`sql +-- ? name1 (text) +-- ? name2 (int) = 0 +SELECT * FROM users WHERE name = ? AND age > ?; +\`\`\` +`, + "write-script-powershell": `--- +name: write-script-powershell +description: MUST use when writing PowerShell scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# PowerShell + +## Structure + +Arguments are obtained by calling the \`param\` function on the first line: + +\`\`\`powershell +param($Name, $Count = 0, [int]$Age) + +# Your code here +Write-Output "Processing $Name, count: $Count, age: $Age" + +# Return object +@{ + name = $Name + count = $Count + age = $Age +} +\`\`\` + +## Parameter Types + +You can specify types for parameters: + +\`\`\`powershell +param( + [string]$Name, + [int]$Count = 0, + [bool]$Enabled = $true, + [array]$Items +) + +@{ + name = $Name + count = $Count + enabled = $Enabled + items = $Items +} +\`\`\` + +## Return Values + +Return values by outputting them at the end of the script: + +\`\`\`powershell +param($Input) + +$result = @{ + processed = $true + data = $Input + timestamp = Get-Date -Format "o" +} + +$result +\`\`\` +`, + "write-script-snowflake": `--- +name: write-script-snowflake +description: MUST use when writing Snowflake queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Snowflake + +Arguments use \`?\` placeholders. + +Name the parameters by adding comments before the statement: + +\`\`\`sql +-- ? name1 (text) +-- ? name2 (number) = 0 +SELECT * FROM users WHERE name = ? AND age > ?; +\`\`\` +`, + "write-script-go": `--- +name: write-script-go +description: MUST use when writing Go scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Go + +## Structure + +The file package must be \`inner\` and export a function called \`main\`: + +\`\`\`go +package inner + +func main(param1 string, param2 int) (map[string]interface{}, error) { + return map[string]interface{}{ + "result": param1, + "count": param2, + }, nil +} +\`\`\` + +**Important:** +- Package must be \`inner\` +- Return type must be \`({return_type}, error)\` +- Function name is \`main\` (lowercase) + +## Return Types + +The return type can be any Go type that can be serialized to JSON: + +\`\`\`go +package inner + +type Result struct { + Name string \`json:"name"\` + Count int \`json:"count"\` +} + +func main(name string, count int) (Result, error) { + return Result{ + Name: name, + Count: count, + }, nil +} +\`\`\` + +## Error Handling + +Return errors as the second return value: + +\`\`\`go +package inner + +import "errors" + +func main(value int) (string, error) { + if value < 0 { + return "", errors.New("value must be positive") + } + return "success", nil +} +\`\`\` +`, + "write-script-deno": `--- +name: write-script-deno +description: MUST use when writing Deno/TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Deno) + +Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +\`\`\`typescript +// npm packages use npm: prefix +import Stripe from "npm:stripe"; +import { someFunction } from "npm:some-package"; + +// Deno standard library +import { serve } from "https://deno.land/std/http/server.ts"; +\`\`\` + +## Windmill Client + +Import the windmill client for platform interactions: + +\`\`\`typescript +import * as wmill from "windmill-client"; +\`\`\` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +\`\`\`typescript +type S3Object = { + s3: string; // Path within the bucket +}; +\`\`\` + +## TypeScript Operations + +\`\`\`typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction +`, + "write-script-bash": `--- +name: write-script-bash +description: MUST use when writing Bash scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Bash + +## Structure + +Do not include \`#!/bin/bash\`. Arguments are obtained as positional parameters: + +\`\`\`bash +# Get arguments +var1="$1" +var2="$2" + +echo "Processing $var1 and $var2" + +# Return JSON by echoing to stdout +echo "{\\"result\\": \\"$var1\\", \\"count\\": $var2}" +\`\`\` + +**Important:** +- Do not include shebang (\`#!/bin/bash\`) +- Arguments are always strings +- Access with \`$1\`, \`$2\`, etc. + +## Output + +The script output is captured as the result. For structured data, output valid JSON: + +\`\`\`bash +name="$1" +count="$2" + +# Output JSON result +cat << EOF +{ + "name": "$name", + "count": $count, + "timestamp": "$(date -Iseconds)" +} +EOF +\`\`\` + +## Environment Variables + +Environment variables set in Windmill are available: + +\`\`\`bash +# Access environment variable +echo "Workspace: $WM_WORKSPACE" +echo "Job ID: $WM_JOB_ID" +\`\`\` +`, + "write-script-bunnative": `--- +name: write-script-bunnative +description: MUST use when writing Bun Native scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Bun Native) + +Native TypeScript execution with fetch only - no external imports allowed. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +**No imports allowed.** Use the globally available \`fetch\` function: + +\`\`\`typescript +export async function main(url: string) { + const response = await fetch(url); + return await response.json(); +} +\`\`\` + +## Windmill Client + +The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +\`\`\` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +\`\`\`typescript +type S3Object = { + s3: string; // Path within the bucket +}; +\`\`\` + +## TypeScript Operations + +\`\`\`typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction +`, + "write-script-duckdb": `--- +name: write-script-duckdb +description: MUST use when writing DuckDB queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# DuckDB + +Arguments are defined with comments and used with \`$name\` syntax: + +\`\`\`sql +-- $name (text) = default +-- $age (integer) +SELECT * FROM users WHERE name = $name AND age > $age; +\`\`\` + +## Ducklake Integration + +Attach Ducklake for data lake operations: + +\`\`\`sql +-- Main ducklake +ATTACH 'ducklake' AS dl; + +-- Named ducklake +ATTACH 'ducklake://my_lake' AS dl; + +-- Then query +SELECT * FROM dl.schema.table; +\`\`\` + +## External Database Connections + +Connect to external databases using resources: + +\`\`\`sql +ATTACH '$res:path/to/resource' AS db (TYPE postgres); +SELECT * FROM db.schema.table; +\`\`\` + +## S3 File Operations + +Read files from S3 storage: + +\`\`\`sql +-- Default storage +SELECT * FROM read_csv('s3:///path/to/file.csv'); + +-- Named storage +SELECT * FROM read_csv('s3://storage_name/path/to/file.csv'); + +-- Parquet files +SELECT * FROM read_parquet('s3:///path/to/file.parquet'); + +-- JSON files +SELECT * FROM read_json('s3:///path/to/file.json'); +\`\`\` +`, + "write-script-nativets": `--- +name: write-script-nativets +description: MUST use when writing Native TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# TypeScript (Native) + +Native TypeScript execution with fetch only - no external imports allowed. + +## Structure + +Export a single **async** function called \`main\`: + +\`\`\`typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +\`\`\` + +Do not call the main function. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the \`RT\` namespace for resource types: + +\`\`\`typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +\`\`\` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +**No imports allowed.** Use the globally available \`fetch\` function: + +\`\`\`typescript +export async function main(url: string) { + const response = await fetch(url); + return await response.json(); +} +\`\`\` + +## Windmill Client + +The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named \`preprocessor\` and receives an \`event\` parameter: + +\`\`\`typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id + }; +} +\`\`\` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format \`$res:path\` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to \`getStatePath()\`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * \`\`\` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * \`\`\` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * \`\`\`typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * \`\`\` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getSlackApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the \`JobService.getTeamsApprovalPayload\` call fails. + * + * **Usage Example:** + * \`\`\`typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * \`\`\` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age}::int + * \`.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql\` + * SELECT * FROM friends + * WHERE name = \${name} AND age = \${age} + * \`.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction +`, + "write-script-bigquery": `--- +name: write-script-bigquery +description: MUST use when writing BigQuery queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# BigQuery + +Arguments use \`@name\` syntax. + +Name the parameters by adding comments before the statement: + +\`\`\`sql +-- @name1 (string) +-- @name2 (int64) = 0 +SELECT * FROM users WHERE name = @name1 AND age > @name2; +\`\`\` +`, + "write-script-rust": `--- +name: write-script-rust +description: MUST use when writing Rust scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Rust + +## Structure + +The script must contain a function called \`main\` with proper return type: + +\`\`\`rust +use anyhow::anyhow; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct ReturnType { + result: String, + count: i32, +} + +fn main(param1: String, param2: i32) -> anyhow::Result { + Ok(ReturnType { + result: param1, + count: param2, + }) +} +\`\`\` + +**Important:** +- Arguments should be owned types +- Return type must be serializable (\`#[derive(Serialize)]\`) +- Return type is \`anyhow::Result\` + +## Dependencies + +Packages must be specified with a partial cargo.toml at the beginning of the script: + +\`\`\`rust +//! \`\`\`cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! reqwest = { version = "0.11", features = ["json"] } +//! tokio = { version = "1", features = ["full"] } +//! \`\`\` + +use anyhow::anyhow; +// ... rest of the code +\`\`\` + +**Note:** Serde is already included, no need to add it again. + +## Async Functions + +If you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside: + +\`\`\`rust +//! \`\`\`cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! tokio = { version = "1", features = ["full"] } +//! reqwest = { version = "0.11", features = ["json"] } +//! \`\`\` + +use anyhow::anyhow; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct Response { + data: String, +} + +fn main(url: String) -> anyhow::Result { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { + let resp = reqwest::get(&url).await?.text().await?; + Ok(Response { data: resp }) + }) +} +\`\`\` +`, + "write-script-php": `--- +name: write-script-php +description: MUST use when writing PHP scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# PHP + +## Structure + +The script must start with \` $param1, "count" => $param2]; +} +\`\`\` + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +You need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using \`class_exists\`: + +\`\`\`php + @P2; +\`\`\` +`, + "write-script-postgresql": `--- +name: write-script-postgresql +description: MUST use when writing PostgreSQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# PostgreSQL + +Arguments are obtained directly in the statement with \`$1::{type}\`, \`$2::{type}\`, etc. + +Name the parameters by adding comments at the beginning of the script (without specifying the type): + +\`\`\`sql +-- $1 name1 +-- $2 name2 = default_value +SELECT * FROM users WHERE name = $1::TEXT AND age > $2::INT; +\`\`\` +`, + "write-script-graphql": `--- +name: write-script-graphql +description: MUST use when writing GraphQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# GraphQL + +## Structure + +Write GraphQL queries or mutations. Arguments can be added as query parameters: + +\`\`\`graphql +query GetUser($id: ID!) { + user(id: $id) { + id + name + email + } +} +\`\`\` + +## Variables + +Variables are passed as script arguments and automatically bound to the query: + +\`\`\`graphql +query SearchProducts($query: String!, $limit: Int = 10) { + products(search: $query, first: $limit) { + edges { + node { + id + name + price + } + } + } +} +\`\`\` + +## Mutations + +\`\`\`graphql +mutation CreateUser($input: CreateUserInput!) { + createUser(input: $input) { + id + name + createdAt + } +} +\`\`\` +`, + "write-script-csharp": `--- +name: write-script-csharp +description: MUST use when writing C# scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# C# + +The script must contain a public static \`Main\` method inside a class: + +\`\`\`csharp +public class Script +{ + public static object Main(string name, int count) + { + return new { Name = name, Count = count }; + } +} +\`\`\` + +**Important:** +- Class name is irrelevant +- Method must be \`public static\` +- Return type can be \`object\` or specific type + +## NuGet Packages + +Add packages using the \`#r\` directive at the top: + +\`\`\`csharp +#r "nuget: Newtonsoft.Json, 13.0.3" +#r "nuget: RestSharp, 110.2.0" + +using Newtonsoft.Json; +using RestSharp; + +public class Script +{ + public static object Main(string url) + { + var client = new RestClient(url); + var request = new RestRequest(); + var response = client.Get(request); + return JsonConvert.DeserializeObject(response.Content); + } +} +\`\`\` +`, + "write-script-java": `--- +name: write-script-java +description: MUST use when writing Java scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- \`wmill script generate-metadata\` - Generate .script.yaml and .lock files +- \`wmill sync push\` - Deploy to Windmill + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Java + +The script must contain a Main public class with a \`public static main()\` method: + +\`\`\`java +public class Main { + public static Object main(String name, int count) { + java.util.Map result = new java.util.HashMap<>(); + result.put("name", name); + result.put("count", count); + return result; + } +} +\`\`\` + +**Important:** +- Class must be named \`Main\` +- Method must be \`public static Object main(...)\` +- Return type is \`Object\` or \`void\` + +## Maven Dependencies + +Add dependencies using comments at the top: + +\`\`\`java +//requirements: +//com.google.code.gson:gson:2.10.1 +//org.apache.httpcomponents:httpclient:4.5.14 + +import com.google.gson.Gson; + +public class Main { + public static Object main(String input) { + Gson gson = new Gson(); + return gson.fromJson(input, Object.class); + } +} +\`\`\` +`, + "write-flow": `--- +name: write-flow +description: MUST use when creating flows. +--- + +# Windmill Flow Building Guide + +## CLI Commands + +Create a folder ending with \`.flow\` and add a YAML file with the flow definition. +For rawscript modules, use \`!inline path/to/script.ts\` for the content key. +After writing: +- \`wmill flow generate-locks --yes\` - Generate lock files +- \`wmill sync push\` - Deploy to Windmill + +## OpenFlow Schema + +The OpenFlow schema (openflow.openapi.yaml) is the source of truth for flow structure. Refer to OPENFLOW_SCHEMA for the complete type definitions. + +## Reserved Module IDs + +- \`failure\` - Reserved for failure handler module +- \`preprocessor\` - Reserved for preprocessor module +- \`Input\` - Reserved for flow input reference + +## Module ID Rules + +- Must be unique across the entire flow +- Use underscores, not spaces (e.g., \`fetch_data\` not \`fetch data\`) +- Use descriptive names that reflect the step's purpose + +## Common Mistakes to Avoid + +- Missing \`input_transforms\` - Rawscript parameters won't receive values without them +- Referencing future steps - \`results.step_id\` only works for steps that execute before the current one +- Duplicate module IDs - Each module ID must be unique in the flow + +## Data Flow Between Steps + +- \`flow_input.property\` - Access flow input parameters +- \`results.step_id\` - Access output from a previous step +- \`results.step_id.property\` - Access specific property from previous step output +- \`flow_input.iter.value\` - Current item when inside a for-loop +- \`flow_input.iter.index\` - Current index when inside a for-loop + +## Input Transforms + +Every rawscript module needs \`input_transforms\` to map function parameters to values: + +Static transform (fixed value): +{"param_name": {"type": "static", "value": "fixed_string"}} + +JavaScript transform (dynamic expression): +{"param_name": {"type": "javascript", "expr": "results.previous_step.data"}} + +## Resource References + +- For flow inputs: Use type \`"object"\` with format \`"resource-{type}"\` (e.g., \`"resource-postgresql"\`) +- For step inputs: Use static value \`"$res:path/to/resource"\` + +## Failure Handler + +Executes when any step fails. Has access to error details: + +- \`error.message\` - Error message +- \`error.step_id\` - ID of failed step +- \`error.name\` - Error name +- \`error.stack\` - Stack trace + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +To accept an S3 object as flow input: + +\`\`\`json +{ + "type": "object", + "properties": { + "file": { + "type": "object", + "format": "resource-s3_object", + "description": "File to process" + } + } +} +\`\`\` + +## Using Resources in Flows + +On Windmill, credentials and configuration are stored in resources. Resource types define the format of the resource. + +### As Flow Input + +In the flow schema, set the property type to \`"object"\` with format \`"resource-{type}"\`: + +\`\`\`json +{ + "type": "object", + "properties": { + "database": { + "type": "object", + "format": "resource-postgresql", + "description": "Database connection" + } + } +} +\`\`\` + +### As Step Input (Static Reference) + +Reference a specific resource using \`$res:\` prefix: + +\`\`\`json +{ + "database": { + "type": "static", + "value": "$res:f/folder/my_database" + } +} +\`\`\` + + +## OpenFlow Schema + +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"number","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"number","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"number","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps","additionalProperties":{"type":"string"}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","description":"Custom error message shown when stopping"}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable"]},"access_type":{"type":"string","description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_images":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of image references for vision-capable models.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'images/photo.jpg' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, + "raw-app": `--- +name: raw-app +description: MUST use when creating raw apps. +--- + +# Windmill Raw Apps + +Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. + +## Creating a Raw App + +\`\`\`bash +wmill app new +\`\`\` + +This interactive command creates a complete app structure with your choice of frontend framework (React, Svelte, or Vue). + +## App Structure + +\`\`\` +my_app.raw_app/ +├── AGENTS.md # AI agent instructions (auto-generated) +├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) +├── raw_app.yaml # App configuration (summary, path, data settings) +├── index.tsx # Frontend entry point +├── App.tsx # Main React/Svelte/Vue component +├── index.css # Styles +├── package.json # Frontend dependencies +├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) +├── backend/ # Backend runnables (server-side scripts) +│ ├── . # Code file (e.g., get_user.ts) +│ ├── .yaml # Optional: config for fields, or to reference existing scripts +│ └── .lock # Lock file (run 'wmill app generate-locks' to create) +└── sql_to_apply/ # SQL migrations (dev only, not synced) + └── *.sql # SQL files to apply via dev server +\`\`\` + +## Backend Runnables + +Backend runnables are server-side scripts that your frontend can call. They live in the \`backend/\` folder. + +### Creating a Backend Runnable + +Add a code file to the \`backend/\` folder: + +\`\`\` +backend/. +\`\`\` + +The runnable ID is the filename without extension. For example, \`get_user.ts\` creates a runnable with ID \`get_user\`. + +### Supported Languages + +| Language | Extension | Example | +|------------------|--------------|------------------| +| TypeScript | \`.ts\` | \`myFunc.ts\` | +| TypeScript (Bun) | \`.bun.ts\` | \`myFunc.bun.ts\` | +| TypeScript (Deno)| \`.deno.ts\` | \`myFunc.deno.ts\` | +| Python | \`.py\` | \`myFunc.py\` | +| Go | \`.go\` | \`myFunc.go\` | +| Bash | \`.sh\` | \`myFunc.sh\` | +| PowerShell | \`.ps1\` | \`myFunc.ps1\` | +| PostgreSQL | \`.pg.sql\` | \`myFunc.pg.sql\` | +| MySQL | \`.my.sql\` | \`myFunc.my.sql\` | +| BigQuery | \`.bq.sql\` | \`myFunc.bq.sql\` | +| Snowflake | \`.sf.sql\` | \`myFunc.sf.sql\` | +| MS SQL | \`.ms.sql\` | \`myFunc.ms.sql\` | +| GraphQL | \`.gql\` | \`myFunc.gql\` | +| PHP | \`.php\` | \`myFunc.php\` | +| Rust | \`.rs\` | \`myFunc.rs\` | +| C# | \`.cs\` | \`myFunc.cs\` | +| Java | \`.java\` | \`myFunc.java\` | + +### Example Backend Runnable + +**backend/get_user.ts:** +\`\`\`typescript +import * as wmill from 'windmill-client'; + +export async function main(user_id: string) { + const sql = wmill.datatable(); + const user = await sql\`SELECT * FROM users WHERE id = \${user_id}\`.fetchOne(); + return user; +} +\`\`\` + +After creating, generate lock files: +\`\`\`bash +wmill app generate-locks +\`\`\` + +### Optional YAML Configuration + +Add a \`.yaml\` file to configure fields or static values: + +**backend/get_user.yaml:** +\`\`\`yaml +type: inline +fields: + user_id: + type: static + value: "default_user" +\`\`\` + +### Referencing Existing Scripts + +To use an existing Windmill script instead of inline code: + +**backend/existing_script.yaml:** +\`\`\`yaml +type: script +path: f/my_folder/existing_script +\`\`\` + +For flows: +\`\`\`yaml +type: flow +path: f/my_folder/my_flow +\`\`\` + +### Calling Backend from Frontend + +Import from the auto-generated \`wmill.ts\`: + +\`\`\`typescript +import { backend } from './wmill'; + +// Call a backend runnable +const user = await backend.get_user({ user_id: '123' }); +\`\`\` + +The \`wmill.ts\` file provides type-safe access to all backend runnables. + +## Data Tables + +Raw apps can query Windmill datatables (PostgreSQL databases managed by Windmill). + +### Critical Rules + +1. **ONLY USE WHITELISTED TABLES**: You can ONLY query tables listed in \`raw_app.yaml\` → \`data.tables\`. Tables not in this list are NOT accessible. + +2. **ADD TABLES BEFORE USING**: To use a new table, first add it to \`data.tables\` in \`raw_app.yaml\`. + +3. **USE CONFIGURED DATATABLE/SCHEMA**: Check the app's \`raw_app.yaml\` for the default datatable and schema. + +### Configuration in raw_app.yaml + +\`\`\`yaml +data: + datatable: main # Default datatable + schema: app_schema # Default schema (optional) + tables: + - main/users # Table in public schema + - main/app_schema:items # Table in specific schema +\`\`\` + +**Table reference formats:** +- \`\` - All tables in the datatable +- \`/
\` - Specific table in public schema +- \`/:
\` - Table in specific schema + +### Querying in TypeScript (Bun/Deno) + +\`\`\`typescript +import * as wmill from 'windmill-client'; + +export async function main(user_id: string) { + const sql = wmill.datatable(); // Or: wmill.datatable('other_datatable') + + // Parameterized queries (safe from SQL injection) + const user = await sql\`SELECT * FROM users WHERE id = \${user_id}\`.fetchOne(); + const users = await sql\`SELECT * FROM users WHERE active = \${true}\`.fetch(); + + // Insert/Update + await sql\`INSERT INTO users (name, email) VALUES (\${name}, \${email})\`; + await sql\`UPDATE users SET name = \${newName} WHERE id = \${user_id}\`; + + return user; +} +\`\`\` + +### Querying in Python + +\`\`\`python +import wmill + +def main(user_id: str): + db = wmill.datatable() # Or: wmill.datatable('other_datatable') + + # Use $1, $2, etc. for parameters + user = db.query('SELECT * FROM users WHERE id = $1', user_id).fetch_one() + users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() + + # Insert/Update + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + + return user +\`\`\` + +## SQL Migrations (sql_to_apply/) + +The \`sql_to_apply/\` folder is for creating/modifying database tables during development. + +### Workflow + +1. Create \`.sql\` files in \`sql_to_apply/\` +2. Run \`wmill app dev\` - the dev server watches this folder +3. When SQL files change, a modal appears in the browser to confirm execution +4. After creating tables, **add them to \`data.tables\`** in \`raw_app.yaml\` + +### Example Migration + +**sql_to_apply/001_create_users.sql:** +\`\`\`sql +CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + name TEXT, + created_at TIMESTAMP DEFAULT NOW() +); +\`\`\` + +After applying, add to \`raw_app.yaml\`: +\`\`\`yaml +data: + tables: + - main/users +\`\`\` + +### Migration Best Practices + +- **Use idempotent SQL**: \`CREATE TABLE IF NOT EXISTS\`, etc. +- **Number files**: \`001_\`, \`002_\` for ordering +- **Always whitelist tables** after creation +- This folder is NOT synced - it's for local development only + +## CLI Commands + +| Command | Description | +|---------|-------------| +| \`wmill app new\` | Create a new raw app interactively | +| \`wmill app dev\` | Start dev server with live reload | +| \`wmill app generate-agents\` | Refresh AGENTS.md and DATATABLES.md | +| \`wmill app generate-locks\` | Generate lock files for backend runnables | +| \`wmill sync push\` | Deploy app to Windmill | +| \`wmill sync pull\` | Pull latest from Windmill | + +## Best Practices + +1. **Check DATATABLES.md** for existing tables before creating new ones +2. **Use parameterized queries** - never concatenate user input into SQL +3. **Keep runnables focused** - one function per file +4. **Use descriptive IDs** - \`get_user.ts\` not \`a.ts\` +5. **Always whitelist tables** - add to \`data.tables\` before querying +6. **Generate locks** - run \`wmill app generate-locks\` after adding/modifying backend runnables +`, + "triggers": `--- +name: triggers +description: MUST use when configuring triggers. +--- + +# Windmill Triggers + +Triggers allow external events to invoke your scripts and flows. + +## File Naming + +Trigger configuration files use the pattern: \`{path}.{trigger_type}_trigger.yaml\` + +Examples: +- \`u/user/webhook.http_trigger.yaml\` +- \`f/data/kafka_consumer.kafka_trigger.yaml\` +- \`f/sync/postgres_cdc.postgres_trigger.yaml\` + +## CLI Commands + +\`\`\`bash +# Push trigger configuration +wmill sync push + +# Pull triggers from Windmill +wmill sync pull +\`\`\` +`, + "schedules": `--- +name: schedules +description: MUST use when configuring schedules. +--- + +# Windmill Schedules + +Schedules run scripts and flows automatically on a cron schedule. + +## File Naming + +Schedule files use the pattern: \`{path}.schedule.yaml\` + +Example: \`f/folder/daily_sync.schedule.yaml\` + +Note: The path is derived from the filename, not stored in the file content. + +## Cron Expression Format + +Windmill uses 6-field cron expressions (includes seconds): + +\`\`\` + ┌───────────── second (0-59) + │ ┌───────────── minute (0-59) + │ │ ┌───────────── hour (0-23) + │ │ │ ┌───────────── day of month (1-31) + │ │ │ │ ┌───────────── month (1-12 or jan-dec) + │ │ │ │ │ ┌───────────── day of week (0-6, 0=Sunday, or sun-sat) + │ │ │ │ │ │ + * * * * * * +\`\`\` + +**Common Examples:** +- \`0 0 0 * * *\` - Daily at midnight +- \`0 0 12 * * *\` - Daily at noon +- \`0 */5 * * * *\` - Every 5 minutes +- \`0 0 9 * * 1-5\` - Weekdays at 9 AM +- \`0 0 0 1 * *\` - First day of each month + +## CLI Commands + +\`\`\`bash +# Push schedules to Windmill +wmill sync push + +# Pull schedules from Windmill +wmill sync pull + +# List schedules +wmill schedule +\`\`\` +`, + "resources": `--- +name: resources +description: MUST use when managing resources. +--- + +# Windmill Resources + +Resources store credentials and configuration for external services. + +## File Format + +Resource files use the pattern: \`{path}.resource.json\` + +Example: \`f/databases/postgres_prod.resource.json\` + +## Resource Structure + +\`\`\`json +{ + "value": { + "host": "db.example.com", + "port": 5432, + "user": "admin", + "password": "$var:g/all/db_password", + "dbname": "production" + }, + "description": "Production PostgreSQL database", + "resource_type": "postgresql" +} +\`\`\` + +## Required Fields + +- \`value\` - Object containing the resource configuration +- \`resource_type\` - Name of the resource type (e.g., "postgresql", "slack") + +## Variable References + +Reference variables in resource values: + +\`\`\`json +{ + "value": { + "api_key": "$var:g/all/api_key", + "secret": "$var:u/admin/secret" + } +} +\`\`\` + +**Reference formats:** +- \`$var:g/all/name\` - Global variable +- \`$var:u/username/name\` - User variable +- \`$var:f/folder/name\` - Folder variable + +## Resource References + +Reference other resources: + +\`\`\`json +{ + "value": { + "database": "$res:f/databases/postgres" + } +} +\`\`\` + +## Common Resource Types + +### PostgreSQL +\`\`\`json +{ + "resource_type": "postgresql", + "value": { + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "$var:g/all/pg_password", + "dbname": "windmill", + "sslmode": "prefer" + } +} +\`\`\` + +### MySQL +\`\`\`json +{ + "resource_type": "mysql", + "value": { + "host": "localhost", + "port": 3306, + "user": "root", + "password": "$var:g/all/mysql_password", + "database": "myapp" + } +} +\`\`\` + +### Slack +\`\`\`json +{ + "resource_type": "slack", + "value": { + "token": "$var:g/all/slack_token" + } +} +\`\`\` + +### AWS S3 +\`\`\`json +{ + "resource_type": "s3", + "value": { + "bucket": "my-bucket", + "region": "us-east-1", + "accessKeyId": "$var:g/all/aws_access_key", + "secretAccessKey": "$var:g/all/aws_secret_key" + } +} +\`\`\` + +### HTTP/API +\`\`\`json +{ + "resource_type": "http", + "value": { + "baseUrl": "https://api.example.com", + "headers": { + "Authorization": "Bearer $var:g/all/api_token" + } + } +} +\`\`\` + +### Kafka +\`\`\`json +{ + "resource_type": "kafka", + "value": { + "brokers": "broker1:9092,broker2:9092", + "sasl_mechanism": "PLAIN", + "security_protocol": "SASL_SSL", + "username": "$var:g/all/kafka_user", + "password": "$var:g/all/kafka_password" + } +} +\`\`\` + +### NATS +\`\`\`json +{ + "resource_type": "nats", + "value": { + "servers": ["nats://localhost:4222"], + "user": "$var:g/all/nats_user", + "password": "$var:g/all/nats_password" + } +} +\`\`\` + +### MQTT +\`\`\`json +{ + "resource_type": "mqtt", + "value": { + "host": "mqtt.example.com", + "port": 8883, + "username": "$var:g/all/mqtt_user", + "password": "$var:g/all/mqtt_password", + "tls": true + } +} +\`\`\` + +## Custom Resource Types + +Create custom resource types with JSON Schema: + +\`\`\`json +{ + "name": "custom_api", + "schema": { + "type": "object", + "properties": { + "base_url": {"type": "string", "format": "uri"}, + "api_key": {"type": "string"}, + "timeout": {"type": "integer", "default": 30} + }, + "required": ["base_url", "api_key"] + }, + "description": "Custom API connection" +} +\`\`\` + +Save as: \`custom_api.resource-type.json\` + +## OAuth Resources + +OAuth resources are managed through the Windmill UI and marked: + +\`\`\`json +{ + "is_oauth": true, + "account": 123 +} +\`\`\` + +OAuth tokens are automatically refreshed by Windmill. + +## Using Resources in Scripts + +### TypeScript (Bun/Deno) +\`\`\`typescript +export async function main(db: RT.Postgresql) { + // db contains the resource values + const { host, port, user, password, dbname } = db; +} +\`\`\` + +### Python +\`\`\`python +class postgresql(TypedDict): + host: str + port: int + user: str + password: str + dbname: str + +def main(db: postgresql): + # db contains the resource values + pass +\`\`\` + +## CLI Commands + +\`\`\`bash +# List resources +wmill resource list + +# List resource types with schemas +wmill resource-type list --schema + +# Get specific resource type schema +wmill resource-type get postgresql + +# Push resources +wmill sync push +\`\`\` +`, + "cli-commands": `--- +name: cli-commands +description: MUST use when using the CLI. +--- + +# Windmill CLI Commands + +The Windmill CLI (\`wmill\`) provides commands for managing scripts, flows, apps, and other resources. + +Current version: 1.624.0 + +## Global Options + +- \`--workspace \` - Specify the target workspace. This overrides the default workspace. +- \`--debug --verbose\` - Show debug/verbose logs +- \`--show-diffs\` - Show diff informations when syncing (may show sensitive informations) +- \`--token \` - Specify an API token. This will override any stored token. +- \`--base-url \` - Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used. +- \`--config-dir \` - Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location. + +## Commands + +### app + +app related commands + +**Subcommands:** + +- \`app push \` - push a local app +- \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement + - \`--port \` - Port to run the dev server on (will find next available port if occupied) + - \`--host \` - Host to bind the dev server to + - \`--entry \` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - \`--no-open\` - Don't automatically open the browser +- \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability + - \`--fix\` - Attempt to fix common issues (not implemented yet) +- \`app new\` - create a new raw app from a template +- \`app generate-agents [app_folder:string]\` - regenerate AGENTS.md and DATATABLES.md from remote workspace +- \`app generate-locks [app_folder:string]\` - re-generate the lockfiles for app runnables inline scripts that have changed + - \`--yes\` - Skip confirmation prompt + - \`--dry-run\` - Perform a dry run without making changes + - \`--default-ts \` - Default TypeScript runtime (bun or deno) + +### dependencies + +workspace dependencies related commands + +**Alias:** \`deps\` + +**Subcommands:** + +- \`dependencies push \` - Push workspace dependencies from a local file + +### dev + +Launch a dev server that will spawn a webserver with HMR + +**Options:** +- \`--includes \` - Filter paths givena glob pattern or path + +### flow + +flow related commands + +**Options:** +- \`--show-archived\` - Enable archived scripts in output + +**Subcommands:** + +- \`flow push \` - push a local flow spec. This overrides any remote versions. +- \`flow run \` - run a flow by path. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-s --silent\` - Do not ouput anything other then the final output. Useful for scripting. +- \`flow preview \` - preview a local flow without deploying it. Runs the flow definition from local files. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. +- \`flow generate-locks [flow:file]\` - re-generate the lock files of all inline scripts of all updated flows + - \`--yes\` - Skip confirmation prompt + - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) + - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. +- \`flow bootstrap \` - create a new empty flow + - \`--summary \` - script summary + - \`--description \` - script description + +### folder + +folder related commands + +**Subcommands:** + +- \`folder push \` - push a local folder spec. This overrides any remote versions. + +### gitsync-settings + +Manage git-sync settings between local wmill.yaml and Windmill backend + +**Subcommands:** + +- \`gitsync-settings pull\` - Pull git-sync settings from Windmill backend to local wmill.yaml + - \`--repository \` - Specify repository path (e.g., u/user/repo) + - \`--default\` - Write settings to top-level defaults instead of overrides + - \`--replace\` - Replace existing settings (non-interactive mode) + - \`--override\` - Add branch-specific override (non-interactive mode) + - \`--diff\` - Show differences without applying changes + - \`--json-output\` - Output in JSON format + - \`--with-backend-settings \` - Use provided JSON settings instead of querying backend (for testing) + - \`--yes\` - Skip interactive prompts and use default behavior + - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides +- \`gitsync-settings push\` - Push git-sync settings from local wmill.yaml to Windmill backend + - \`--repository \` - Specify repository path (e.g., u/user/repo) + - \`--diff\` - Show what would be pushed without applying changes + - \`--json-output\` - Output in JSON format + - \`--with-backend-settings \` - Use provided JSON settings instead of querying backend (for testing) + - \`--yes\` - Skip interactive prompts and use default behavior + - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides + +### hub + +Hub related commands. EXPERIMENTAL. INTERNAL USE ONLY. + +**Subcommands:** + +- \`hub pull\` - pull any supported definitions. EXPERIMENTAL. + +### init + +Bootstrap a windmill project with a wmill.yaml file + +**Options:** +- \`--use-default\` - Use default settings without checking backend +- \`--use-backend\` - Use backend git-sync settings if available +- \`--repository \` - Specify repository path (e.g., u/user/repo) when using backend settings +- \`--bind-profile\` - Automatically bind active workspace profile to current Git branch +- \`--no-bind-profile\` - Skip workspace profile binding prompt + +### instance + +sync local with a remote instance or the opposite (push or pull) + +**Subcommands:** + +- \`instance add [instance_name:string] [remote:string] [token:string]\` - Add a new instance +- \`instance remove \` - Remove an instance +- \`instance switch \` - Switch the current instance +- \`instance pull\` - Pull instance settings, users, configs, instance groups and overwrite local + - \`--yes\` - Pull without needing confirmation + - \`--dry-run\` - Perform a dry run without making changes + - \`--skip-users\` - Skip pulling users + - \`--skip-settings\` - Skip pulling settings + - \`--skip-configs\` - Skip pulling configs (worker groups and SMTP) + - \`--skip-groups\` - Skip pulling instance groups + - \`--include-workspaces\` - Also pull workspaces + - \`--folder-per-instance\` - Create a folder per instance + - \`--instance \` - Name of the instance to pull from, override the active instance + - \`--prefix \` - Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces + - \`--prefix-settings\` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance +- \`instance push\` - Push instance settings, users, configs, group and overwrite remote + - \`--yes\` - Push without needing confirmation + - \`--dry-run\` - Perform a dry run without making changes + - \`--skip-users\` - Skip pushing users + - \`--skip-settings\` - Skip pushing settings + - \`--skip-configs\` - Skip pushing configs (worker groups and SMTP) + - \`--skip-groups\` - Skip pushing instance groups + - \`--include-workspaces\` - Also push workspaces + - \`--folder-per-instance\` - Create a folder per instance + - \`--instance \` - Name of the instance to push to, override the active instance + - \`--prefix \` - Prefix of the local workspaces folders to push + - \`--prefix-settings\` - Store instance yamls inside prefixed folders when using --prefix and --folder-per-instance +- \`instance whoami\` - Display information about the currently logged-in user + +### jobs + +Pull completed and queued jobs from workspace + +**Arguments:** \`[workspace:string]\` + +**Options:** +- \`-c, --completed-output \` - Completed jobs output file (default: completed_jobs.json) +- \`-q, --queued-output \` - Queued jobs output file (default: queued_jobs.json) +- \`--skip-worker-check\` - Skip checking for active workers before export + +**Subcommands:** + +- \`jobs pull\` +- \`jobs push\` + +### queues + +List all queues with their metrics + +**Arguments:** \`[workspace:string] the optional workspace to filter by (default to all workspaces)\` + +**Options:** +- \`--instance [instance]\` - Name of the instance to push to, override the active instance +- \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance + +### resource + +resource related commands + +**Subcommands:** + +- \`resource push \` - push a local resource spec. This overrides any remote versions. + +### resource-type + +resource type related commands + +**Subcommands:** + +- \`resource-type list\` - list all resource types + - \`--schema\` - Show schema in the output +- \`resource-type push \` - push a local resource spec. This overrides any remote versions. +- \`resource-type generate-namespace\` - Create a TypeScript definition file with the RT namespace generated from the resource types + +### schedule + +schedule related commands + +**Subcommands:** + +- \`schedule push \` - push a local schedule spec. This overrides any remote versions. + +### script + +script related commands + +**Options:** +- \`--show-archived\` - Enable archived scripts in output + +**Subcommands:** + +- \`script push \` - push a local script spec. This overrides any remote versions. Use the script file (.ts, .js, .py, .sh +- \`script show \` - show a scripts content +- \`script run \` - run a script by path + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-s --silent\` - Do not output anything other then the final output. Useful for scripting. +- \`script preview \` - preview a local script without deploying it. Supports both regular and codebase scripts. + - \`-d --data \` - Inputs specified as a JSON string or a file using @ or stdin using @-. + - \`-s --silent\` - Do not output anything other than the final output. Useful for scripting. +- \`script bootstrap \` - create a new script + - \`--summary \` - script summary + - \`--description \` - script description +- \`script generate-metadata [script:file]\` - re-generate the metadata file updating the lock and the script schema (for flows, use \`wmill flow generate-locks\` + - \`--yes\` - Skip confirmation prompt + - \`--dry-run\` - Perform a dry run without making changes + - \`--lock-only\` - re-generate only the lock + - \`--schema-only\` - re-generate only script schema + - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) + - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. + +### sync + +sync local with a remote workspaces or the opposite (push or pull) + +**Subcommands:** + +- \`sync pull\` - Pull any remote changes and apply them locally. + - \`--yes\` - Pull without needing confirmation + - \`--dry-run\` - Show changes that would be pulled without actually pushing + - \`--plain-secrets\` - Pull secrets as plain text + - \`--json\` - Use JSON instead of YAML + - \`--skip-variables\` - Skip syncing variables (including secrets) + - \`--skip-secrets\` - Skip syncing only secrets variables + - \`--skip-resources\` - Skip syncing resources + - \`--skip-resource-types\` - Skip syncing resource types + - \`--skip-scripts\` - Skip syncing scripts + - \`--skip-flows\` - Skip syncing flows + - \`--skip-apps\` - Skip syncing apps + - \`--skip-folders\` - Skip syncing folders + - \`--skip-workspace-dependencies\` - Skip syncing workspace dependencies + - \`--skip-scripts-metadata\` - Skip syncing scripts metadata, focus solely on logic + - \`--include-schedules\` - Include syncing schedules + - \`--include-triggers\` - Include syncing triggers + - \`--include-users\` - Include syncing users + - \`--include-groups\` - Include syncing groups + - \`--include-settings\` - Include syncing workspace settings + - \`--include-key\` - Include workspace encryption key + - \`--skip-branch-validation\` - Skip git branch validation and prompts + - \`--json-output\` - Output results in JSON format + - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Overrides wmill.yaml includes + - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. Overrides wmill.yaml excludes + - \`--extra-includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy + - \`--repository \` - Specify repository path (e.g., u/user/repo) when multiple repositories exist + - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides + - \`--branch \` - Override the current git branch (works even outside a git repository) +- \`sync push\` - Push any local changes and apply them remotely. + - \`--yes\` - Push without needing confirmation + - \`--dry-run\` - Show changes that would be pushed without actually pushing + - \`--plain-secrets\` - Push secrets as plain text + - \`--json\` - Use JSON instead of YAML + - \`--skip-variables\` - Skip syncing variables (including secrets) + - \`--skip-secrets\` - Skip syncing only secrets variables + - \`--skip-resources\` - Skip syncing resources + - \`--skip-resource-types\` - Skip syncing resource types + - \`--skip-scripts\` - Skip syncing scripts + - \`--skip-flows\` - Skip syncing flows + - \`--skip-apps\` - Skip syncing apps + - \`--skip-folders\` - Skip syncing folders + - \`--skip-workspace-dependencies\` - Skip syncing workspace dependencies + - \`--skip-scripts-metadata\` - Skip syncing scripts metadata, focus solely on logic + - \`--include-schedules\` - Include syncing schedules + - \`--include-triggers\` - Include syncing triggers + - \`--include-users\` - Include syncing users + - \`--include-groups\` - Include syncing groups + - \`--include-settings\` - Include syncing workspace settings + - \`--include-key\` - Include workspace encryption key + - \`--skip-branch-validation\` - Skip git branch validation and prompts + - \`--json-output\` - Output results in JSON format + - \`-i --includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string) + - \`-e --excludes \` - Comma separated patterns to specify which file to NOT take into account. + - \`--extra-includes \` - Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string). Useful to still take wmill.yaml into account and act as a second pattern to satisfy + - \`--message \` - Include a message that will be added to all scripts/flows/apps updated during this push + - \`--parallel \` - Number of changes to process in parallel + - \`--repository \` - Specify repository path (e.g., u/user/repo) when multiple repositories exist + - \`--branch \` - Override the current git branch (works even outside a git repository) + +### trigger + +trigger related commands + +**Subcommands:** + +- \`trigger push \` - push a local trigger spec. This overrides any remote versions. + +### user + +user related commands + +**Subcommands:** + +- \`user add [password:string]\` - Create a user + - \`--superadmin\` - Specify to make the new user superadmin. + - \`--company \` - Specify to set the company of the new user. + - \`--name \` - Specify to set the name of the new user. +- \`user remove \` - Delete a user +- \`user create-token\` + - \`--email \` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either. + - \`--password \` - Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either. + +### variable + +variable related commands + +**Subcommands:** + +- \`variable push \` - Push a local variable spec. This overrides any remote versions. + - \`--plain-secrets\` - Push secrets as plain text +- \`variable add \` - Create a new variable on the remote. This will update the variable if it already exists. + - \`--plain-secrets\` - Push secrets as plain text + - \`--public\` - Legacy option, use --plain-secrets instead + +### version + +Show version information + +### worker-groups + +display worker groups, pull and push worker groups configs + +**Subcommands:** + +- \`worker-groups pull\` - Pull worker groups (similar to \`wmill instance pull --skip-users --skip-settings --skip-groups\`) + - \`--instance\` - Name of the instance to push to, override the active instance + - \`--base-url\` - Base url to be passed to the instance settings instead of the local one + - \`--yes\` - Pull without needing confirmation +- \`worker-groups push\` - Push instance settings, users, configs, group and overwrite remote + - \`--instance [instance]\` - Name of the instance to push to, override the active instance + - \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance + - \`--yes\` - Push without needing confirmation + +### workers + +List all workers grouped by worker groups + +**Options:** +- \`--instance [instance]\` - Name of the instance to push to, override the active instance +- \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance + +### workspace + +workspace related commands + +**Alias:** \`profile\` + +**Subcommands:** + +- \`workspace switch \` - Switch to another workspace +- \`workspace add [workspace_name:string] [workspace_id:string] [remote:string]\` - Add a workspace + - \`-c --create\` - Create the workspace if it does not exist + - \`--create-workspace-name \` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id. + - \`--create-username \` - Specify your own username in the newly created workspace. Ignored if --create is not specified, the workspace already exists or automatic username creation is enabled on the instance. +- \`workspace remove \` - Remove a workspace +- \`workspace whoami\` - Show the currently active user +- \`workspace list\` - List workspaces on the remote server that you have access to +- \`workspace bind\` - Bind the current Git branch to the active workspace + - \`--branch \` - Specify branch (defaults to current) +- \`workspace unbind\` - Remove workspace binding from the current Git branch + - \`--branch \` - Specify branch (defaults to current) +- \`workspace fork [workspace_name:string] [workspace_id:string]\` - Create a forked workspace + - \`--create-workspace-name \` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id. +- \`workspace delete-fork \` - Delete a forked workspace and git branch + - \`-y --yes\` - Skip confirmation prompt + +`, +}; + +// YAML schema content for triggers and schedules +export const SCHEMAS: Record = { + "gcp_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + gcp_resource_path: + type: string + description: Path to the GCP resource containing service account credentials for + authentication. + topic_id: + type: string + description: Google Cloud Pub/Sub topic ID to subscribe to. + subscription_id: + type: string + description: Google Cloud Pub/Sub subscription ID. + delivery_type: + type: string + enum: + - push + - pull + description: Delivery mode for messages. 'push' for HTTP push delivery where messages + are sent to a webhook endpoint, 'pull' for polling where the trigger actively + fetches messages. + delivery_config: + type: object + properties: + audience: + type: string + description: The audience claim for OIDC tokens used in push authentication. + authenticate: + type: boolean + description: If true, push messages will include OIDC authentication tokens. + description: Configuration for push delivery mode. + subscription_mode: + type: string + enum: + - existing + - create_update + description: The mode of subscription. 'existing' means using an existing GCP + subscription, while 'create_update' involves creating or updating a new subscription. + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails. + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- gcp_resource_path +- topic_id +- subscription_id +- delivery_type +- subscription_mode +`, + "http_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + route_path: + type: string + description: The URL route path that will trigger this endpoint (e.g., 'api/myendpoint'). + Must NOT start with a /. + static_asset_config: + type: object + properties: + s3: + type: string + description: S3 bucket path for static assets + storage: + type: string + description: Storage path for static assets + filename: + type: string + description: Filename for the static asset + description: Configuration for serving static assets (s3 bucket, storage path, + filename) + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + authentication_resource_path: + type: string + description: Path to the resource containing authentication configuration (for + api_key, basic_http, custom_script, signature methods) + summary: + type: string + description: Short summary describing the purpose of this trigger + description: + type: string + description: Detailed description of what this trigger does + request_type: + type: string + enum: + - sync + - async + - sync_sse + authentication_method: + type: string + enum: + - none + - windmill + - api_key + - basic_http + - custom_script + - signature + is_static_website: + type: boolean + description: If true, serves static files from S3/storage instead of running a + script + workspaced_route: + type: boolean + description: If true, the route includes the workspace ID in the path + wrap_body: + type: boolean + description: If true, wraps the request body in a 'body' parameter + raw_string: + type: boolean + description: If true, passes the request body as a raw string instead of parsing + as JSON + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- route_path +- request_type +- authentication_method +- http_method +- is_static_website +- workspaced_route +- wrap_body +- raw_string +`, + "kafka_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + kafka_resource_path: + type: string + description: Path to the Kafka resource containing connection configuration + group_id: + type: string + description: Kafka consumer group ID for this trigger + topics: + type: array + items: + type: string + description: Array of Kafka topic names to subscribe to + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- kafka_resource_path +- group_id +- topics +`, + "mqtt_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + mqtt_resource_path: + type: string + description: Path to the MQTT resource containing broker connection configuration + subscribe_topics: + type: array + items: + type: object + description: Array of MQTT topics to subscribe to, each with topic name and QoS + level + v3_config: + type: object + properties: + clean_session: + type: boolean + v5_config: + type: object + properties: + clean_start: + type: boolean + topic_alias_maximum: + type: number + session_expiry_interval: + type: number + client_id: + type: string + description: MQTT client ID for this connection + client_version: + type: string + enum: + - v3 + - v5 + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- subscribe_topics +- mqtt_resource_path +`, + "nats_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + nats_resource_path: + type: string + description: Path to the NATS resource containing connection configuration + use_jetstream: + type: boolean + description: If true, uses NATS JetStream for durable message delivery + stream_name: + type: string + description: JetStream stream name (required when use_jetstream is true) + consumer_name: + type: string + description: JetStream consumer name (required when use_jetstream is true) + subjects: + type: array + items: + type: string + description: Array of NATS subjects to subscribe to + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- nats_resource_path +- use_jetstream +- subjects +`, + "postgres_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + postgres_resource_path: + type: string + description: Path to the PostgreSQL resource containing connection configuration + publication_name: + type: string + description: Name of the PostgreSQL publication to subscribe to for change data + capture + replication_slot_name: + type: string + description: Name of the PostgreSQL logical replication slot to use + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- postgres_resource_path +- replication_slot_name +- publication_name +`, + "schedule": `type: object +properties: + schedule: + type: string + description: Cron expression with 6 fields (seconds, minutes, hours, day of month, + month, day of week). Example '0 0 12 * * *' for daily at noon + timezone: + type: string + description: IANA timezone for the schedule (e.g., 'UTC', 'Europe/Paris', 'America/New_York') + enabled: + type: boolean + description: Whether the schedule is currently active and will trigger jobs + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + args: + type: object + description: The arguments to pass to the script or flow + on_failure: + type: string + description: Path to a script or flow to run when the scheduled job fails + on_failure_times: + type: number + description: Number of consecutive failures before the on_failure handler is triggered + (default 1) + on_failure_exact: + type: boolean + description: If true, trigger on_failure handler only on exactly N failures, not + on every failure after N + on_failure_extra_args: + type: object + description: The arguments to pass to the script or flow + on_recovery: + type: string + description: Path to a script or flow to run when the schedule recovers after + failures + on_recovery_times: + type: number + description: Number of consecutive successes before the on_recovery handler is + triggered (default 1) + on_recovery_extra_args: + type: object + description: The arguments to pass to the script or flow + on_success: + type: string + description: Path to a script or flow to run after each successful execution + on_success_extra_args: + type: object + description: The arguments to pass to the script or flow + ws_error_handler_muted: + type: boolean + description: If true, the workspace-level error handler will not be triggered + for this schedule's failures + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions + summary: + type: string + description: Short summary describing the purpose of this schedule + description: + type: string + description: Detailed description of what this schedule does + no_flow_overlap: + type: boolean + description: If true, skip this schedule's execution if the previous run is still + in progress (prevents concurrent runs) + tag: + type: string + description: Worker tag to route jobs to specific worker groups + paused_until: + type: string + description: ISO 8601 datetime until which the schedule is paused. Schedule resumes + automatically after this time + cron_version: + type: string + description: Cron parser version. Use 'v2' for extended syntax with additional + features + dynamic_skip: + type: string + description: Path to a script that validates scheduled datetimes. Receives scheduled_for + datetime and returns boolean to skip (true) or run (false) +required: +- schedule +- script_path +- timezone +- is_flow +- enabled +`, + "sqs_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + queue_url: + type: string + description: The full URL of the AWS SQS queue to poll for messages + aws_auth_resource_type: + type: string + enum: + - oidc + - credentials + aws_resource_path: + type: string + description: Path to the AWS resource containing credentials or OIDC configuration + message_attributes: + type: array + items: + type: string + description: Array of SQS message attribute names to include with each message + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- queue_url +- aws_resource_path +- aws_auth_resource_type +`, + "websocket_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + url: + type: string + description: The WebSocket URL to connect to (can be a static URL or computed + by a runnable) + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + description: Array of key-value filters to match incoming messages (only matching + messages trigger the script) + initial_messages: + type: array + items: + type: object + description: Messages to send immediately after connecting (can be raw strings + or computed by runnables) + url_runnable_args: + type: object + description: The arguments to pass to the script or flow + can_return_message: + type: boolean + description: If true, the script can return a message to send back through the + WebSocket + can_return_error_result: + type: boolean + description: If true, error results are sent back through the WebSocket + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- is_flow +- url +- filters +- can_return_message +- can_return_error_result +`, +}; + +// Maps skill names to their schema types and file patterns +export interface SchemaMapping { + name: string; + schemaKey: string; + filePattern: string; +} + +export const SCHEMA_MAPPINGS: Record = { + "triggers": [ + { name: "HttpTrigger", schemaKey: "http_trigger", filePattern: "*.http_trigger.yaml" }, + { name: "WebsocketTrigger", schemaKey: "websocket_trigger", filePattern: "*.websocket_trigger.yaml" }, + { name: "KafkaTrigger", schemaKey: "kafka_trigger", filePattern: "*.kafka_trigger.yaml" }, + { name: "NatsTrigger", schemaKey: "nats_trigger", filePattern: "*.nats_trigger.yaml" }, + { name: "PostgresTrigger", schemaKey: "postgres_trigger", filePattern: "*.postgres_trigger.yaml" }, + { name: "MqttTrigger", schemaKey: "mqtt_trigger", filePattern: "*.mqtt_trigger.yaml" }, + { name: "SqsTrigger", schemaKey: "sqs_trigger", filePattern: "*.sqs_trigger.yaml" }, + { name: "GcpTrigger", schemaKey: "gcp_trigger", filePattern: "*.gcp_trigger.yaml" }, + ], + "schedules": [ + { name: "Schedule", schemaKey: "schedule", filePattern: "*.schedule.yaml" }, + ], +}; diff --git a/cli/src/main.ts b/cli/src/main.ts index 583e78a1e9..5a6972f023 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -77,7 +77,7 @@ export { // } // }); -export const VERSION = "1.625.0"; +export const VERSION = "1.629.1"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/cli/test-skills/README.md b/cli/test-skills/README.md new file mode 100644 index 0000000000..7ca6599215 --- /dev/null +++ b/cli/test-skills/README.md @@ -0,0 +1,103 @@ +# Windmill Skill Invocation Tests + +Test suite for verifying that Claude Code correctly invokes Windmill auto-generated skills based on user prompts. + +## Overview + +This framework tests skill invocation behavior by sending prompts through the Claude Agent SDK and verifying that the expected skills are invoked. Users must provide their own `.claude/skills` folder containing auto-generated Windmill skills. + +## Prerequisites + +- [Bun](https://bun.sh/) installed +- `ANTHROPIC_API_KEY` environment variable set +- Auto-generated Windmill skills placed in `.claude/skills/` + +## User Setup + +1. Create a `test-folder` directory inside `cli/test-skills/` and copy your auto-generated Windmill skills into it: + +``` +cli/test-skills/ +└── test-folder/ + └── .claude/ + └── skills/ + ├── write-flow/ + │ └── SKILL.md + ├── write-script-python3/ + │ └── SKILL.md + ├── write-script-bun/ + │ └── SKILL.md + ├── schedules/ + │ └── SKILL.md + └── triggers/ + └── SKILL.md +``` + +2. Set your API key: +```bash +export ANTHROPIC_API_KEY=your-key-here +``` + +3. Install dependencies and run tests: +```bash +cd cli/test-skills +bun install +bun test +``` + +## Expected Skills + +The tests expect the following auto-generated skills to be present: + +| Skill Name | Purpose | +|------------|---------| +| `write-flow` | Creating Windmill flows/workflows | +| `write-script-python3` | Creating Python scripts | +| `write-script-bun` | Creating TypeScript/Bun scripts | +| `schedules` | Configuring schedules and cron jobs | +| `triggers` | Setting up triggers (webhook, Kafka, etc.) | + +## Test Matrix + +| Prompt | Expected Skill | +|--------|----------------| +| "Create a flow to process user data" | `write-flow` | +| "Build a workflow that fetches and transforms data" | `write-flow` | +| "Write a Python script to fetch API data" | `write-script-python3` | +| "Create a Python function to process CSV files" | `write-script-python3` | +| "Write a TypeScript script using Bun" | `write-script-bun` | +| "Create a Bun script to handle webhooks" | `write-script-bun` | +| "Set up a schedule to run this daily at midnight" | `schedules` | +| "Configure a cron job to run every hour" | `schedules` | +| "Set up a webhook trigger for this flow" | `triggers` | +| "Configure a Kafka trigger" | `triggers` | + +## Running Tests + +Run all tests: +```bash +bun test +``` + +Run only skill invocation tests: +```bash +bun test:skills +``` + +## Test Utilities + +The `src/test-utils.ts` module provides: + +- `runPromptAndCapture(prompt, cwd?, maxTurns)` - Runs a prompt and captures tool invocations +- `wasToolUsed(result, toolName)` - Checks if a specific tool was used +- `wasSkillInvoked(result, skillName)` - Checks if a specific skill was invoked +- `getToolInputs(result, toolName)` - Gets all inputs for a specific tool +- `getTestSkillsDir()` - Returns the test-skills directory path + +## Notes + +- Tests have extended timeouts (120 seconds) due to API latency +- Tests run against the actual Claude API, so they consume API credits +- Tests verify skill invocation, not skill execution +- The working directory for tests is `test-folder/` (where `.claude/skills` should be placed) +- Tests will fail with a clear error if `test-folder/` or `test-folder/.claude/skills/` don't exist diff --git a/cli/test-skills/bun.lock b/cli/test-skills/bun.lock new file mode 100644 index 0000000000..9e4443cca8 --- /dev/null +++ b/cli/test-skills/bun.lock @@ -0,0 +1,61 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "claude-code-skill-tests", + "dependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.2.25", + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.0.0", + }, + }, + }, + "packages": { + "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.25", "", { "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.33.5", "@img/sharp-darwin-x64": "^0.33.5", "@img/sharp-linux-arm": "^0.33.5", "@img/sharp-linux-arm64": "^0.33.5", "@img/sharp-linux-x64": "^0.33.5", "@img/sharp-linuxmusl-arm64": "^0.33.5", "@img/sharp-linuxmusl-x64": "^0.33.5", "@img/sharp-win32-x64": "^0.33.5" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-YIP3I40+XSkC3zE1Z8KRQY02VA7UfofFamF1cFrLe7FbtCnjpslyDl9coGBh2DAi9xj2yQcKZZf751jEWpB+dQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.0.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.0.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.0.5", "", { "os": "linux", "cpu": "arm" }, "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.0.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.0.4", "", { "os": "linux", "cpu": "x64" }, "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.0.5" }, "os": "linux", "cpu": "arm" }, "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.0.4" }, "os": "linux", "cpu": "arm64" }, "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.0.4" }, "os": "linux", "cpu": "x64" }, "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" }, "os": "linux", "cpu": "arm64" }, "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.0.4" }, "os": "linux", "cpu": "x64" }, "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.33.5", "", { "os": "win32", "cpu": "x64" }, "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg=="], + + "@types/bun": ["@types/bun@1.3.8", "", { "dependencies": { "bun-types": "1.3.8" } }, "sha512-3LvWJ2q5GerAXYxO2mffLTqOzEu5qnhEAlh48Vnu8WQfnmSwbgagjGZV6BoHKJztENYEDn6QmVd949W4uESRJA=="], + + "@types/node": ["@types/node@25.1.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-t7frlewr6+cbx+9Ohpl0NOTKXZNV9xHRmNOvql47BFJKcEG1CxtxlPEEe+gR9uhVWM4DwhnvTF110mIL4yP9RA=="], + + "bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + + "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + } +} diff --git a/cli/test-skills/package.json b/cli/test-skills/package.json new file mode 100644 index 0000000000..1d839ab23c --- /dev/null +++ b/cli/test-skills/package.json @@ -0,0 +1,16 @@ +{ + "name": "claude-code-skill-tests", + "version": "1.0.0", + "type": "module", + "scripts": { + "test": "bun test", + "test:skills": "bun test src/skill-invocation.test.ts" + }, + "dependencies": { + "@anthropic-ai/claude-agent-sdk": "^0.2.25" + }, + "devDependencies": { + "@types/bun": "latest", + "typescript": "^5.0.0" + } +} diff --git a/cli/test-skills/src/skill-invocation.test.ts b/cli/test-skills/src/skill-invocation.test.ts new file mode 100644 index 0000000000..9633eb2ef5 --- /dev/null +++ b/cli/test-skills/src/skill-invocation.test.ts @@ -0,0 +1,91 @@ +import { describe, test, expect, beforeAll } from "bun:test"; +import { runPromptAndCapture, wasSkillInvoked, wasToolUsed, validateTestFolder } from "./test-utils"; + +describe("Windmill Skill Invocation", () => { + beforeAll(() => { + if (!process.env.ANTHROPIC_API_KEY) { + throw new Error("ANTHROPIC_API_KEY environment variable is required"); + } + validateTestFolder(); + }); + + describe("Flow Creation", () => { + test("'Create a Windmill flow' should invoke write-flow skill", async () => { + const result = await runPromptAndCapture( + "Create a Windmill flow that fetches data from an API and transforms it. Use placeholder URLs.", + undefined, + 3 + ); + + console.log("Tools used:", result.toolsUsed.map(t => t.tool)); + console.log("Skills invoked:", result.skillsInvoked); + + expect(wasToolUsed(result, "Skill")).toBe(true); + expect(wasSkillInvoked(result, "write-flow")).toBe(true); + }, { timeout: 120000 }); + }); + + describe("Python Script Creation", () => { + test("'Write a Windmill Python script' should invoke write-script-python3 skill", async () => { + const result = await runPromptAndCapture( + "Write a Windmill Python script that fetches data from https://api.example.com/users", + undefined, + 3 + ); + + console.log("Tools used:", result.toolsUsed.map(t => t.tool)); + console.log("Skills invoked:", result.skillsInvoked); + + expect(wasToolUsed(result, "Skill")).toBe(true); + expect(wasSkillInvoked(result, "write-script-python3")).toBe(true); + }, { timeout: 120000 }); + }); + + describe("Bun Script Creation", () => { + test("'Write a Windmill Bun/TypeScript script' should invoke write-script-bun skill", async () => { + const result = await runPromptAndCapture( + "Write a Windmill Bun script that processes JSON data", + undefined, + 3 + ); + + console.log("Tools used:", result.toolsUsed.map(t => t.tool)); + console.log("Skills invoked:", result.skillsInvoked); + + expect(wasToolUsed(result, "Skill")).toBe(true); + expect(wasSkillInvoked(result, "write-script-bun")).toBe(true); + }, { timeout: 120000 }); + }); + + describe("Schedule Configuration", () => { + test("'Create a Windmill schedule' should invoke schedules skill", async () => { + const result = await runPromptAndCapture( + "Create a Windmill schedule that runs a script daily at midnight", + undefined, + 3 + ); + + console.log("Tools used:", result.toolsUsed.map(t => t.tool)); + console.log("Skills invoked:", result.skillsInvoked); + + expect(wasToolUsed(result, "Skill")).toBe(true); + expect(wasSkillInvoked(result, "schedules")).toBe(true); + }, { timeout: 120000 }); + }); + + describe("Trigger Configuration", () => { + test("'Set up a Windmill webhook trigger' should invoke triggers skill", async () => { + const result = await runPromptAndCapture( + "Set up a Windmill HTTP trigger for a flow at /api/webhook", + undefined, + 3 + ); + + console.log("Tools used:", result.toolsUsed.map(t => t.tool)); + console.log("Skills invoked:", result.skillsInvoked); + + expect(wasToolUsed(result, "Skill")).toBe(true); + expect(wasSkillInvoked(result, "triggers")).toBe(true); + }, { timeout: 120000 }); + }); +}); diff --git a/cli/test-skills/src/test-utils.ts b/cli/test-skills/src/test-utils.ts new file mode 100644 index 0000000000..dbf2ea4be9 --- /dev/null +++ b/cli/test-skills/src/test-utils.ts @@ -0,0 +1,137 @@ +import { query, type Options } from "@anthropic-ai/claude-agent-sdk"; +import { existsSync } from "fs"; +import { join } from "path"; + +export interface ToolInvocation { + tool: string; + input: Record; + timestamp: number; +} + +export interface TestResult { + toolsUsed: ToolInvocation[]; + skillsInvoked: string[]; + output: string; +} + +/** + * Get the test-skills directory path + */ +export function getTestSkillsDir(): string { + return new URL("..", import.meta.url).pathname; +} + +/** + * Get the test-folder directory path (where user places .claude/skills) + */ +export function getTestFolder(): string { + return join(getTestSkillsDir(), "test-folder"); +} + +/** + * Validate that test-folder exists and has .claude/skills + * Throws an error if validation fails + */ +export function validateTestFolder(): void { + const testFolder = getTestFolder(); + const skillsFolder = join(testFolder, ".claude", "skills"); + + if (!existsSync(testFolder)) { + throw new Error( + `test-folder does not exist at: ${testFolder}\n` + + `Please create it and add your .claude/skills directory inside.` + ); + } + + if (!existsSync(skillsFolder)) { + throw new Error( + `.claude/skills directory not found in test-folder at: ${skillsFolder}\n` + + `Please add your auto-generated Windmill skills to test-folder/.claude/skills/` + ); + } +} + +/** + * Runs a prompt through the Claude Agent SDK and captures tool invocations + * Uses test-folder as cwd where user-provided skills are located + */ +export async function runPromptAndCapture( + prompt: string, + cwd?: string, + maxTurns: number = 3 +): Promise { + const workingDir = cwd ?? getTestFolder(); + const toolsUsed: ToolInvocation[] = []; + const skillsInvoked: string[] = []; + let output = ""; + + const options: Options = { + cwd: workingDir, + model: "haiku", + maxTurns, + settingSources: ["project"], // Required to load Skills from filesystem + allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"], + }; + + for await (const message of query({ prompt, options })) { + if (message.type === "assistant") { + // The assistant message has a BetaMessage which contains content blocks + const content = message.message?.content; + if (Array.isArray(content)) { + for (const block of content) { + if (block.type === "tool_use") { + const toolInvocation: ToolInvocation = { + tool: block.name, + input: block.input as Record, + timestamp: Date.now(), + }; + toolsUsed.push(toolInvocation); + + // Check if this is a Skill tool invocation + if (block.name === "Skill" && typeof block.input === "object" && block.input !== null) { + const skillInput = block.input as { skill?: string }; + if (skillInput.skill) { + skillsInvoked.push(skillInput.skill); + } + } + } else if (block.type === "text") { + output += block.text; + } + } + } + } else if (message.type === "result") { + // Capture final result if available + const resultMessage = message as { result?: string }; + if (typeof resultMessage.result === "string") { + output += resultMessage.result; + } + } + } + + return { + toolsUsed, + skillsInvoked, + output, + }; +} + +/** + * Helper to check if a specific tool was used + */ +export function wasToolUsed(result: TestResult, toolName: string): boolean { + return result.toolsUsed.some((t) => t.tool === toolName); +} + +/** + * Helper to check if a specific skill was invoked + */ +export function wasSkillInvoked(result: TestResult, skillName: string): boolean { + return result.skillsInvoked.some((s) => s === skillName || s.includes(skillName)); +} + +/** + * Helper to get all tool inputs for a specific tool + */ +export function getToolInputs(result: TestResult, toolName: string): Record[] { + return result.toolsUsed.filter((t) => t.tool === toolName).map((t) => t.input); +} diff --git a/cli/test-skills/tsconfig.json b/cli/test-skills/tsconfig.json new file mode 100644 index 0000000000..45f0069307 --- /dev/null +++ b/cli/test-skills/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "types": ["bun-types"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/cli/windmill-utils-internal/package-lock.json b/cli/windmill-utils-internal/package-lock.json index ddf3437925..6188fc4652 100644 --- a/cli/windmill-utils-internal/package-lock.json +++ b/cli/windmill-utils-internal/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-utils-internal", - "version": "1.3.2", + "version": "1.3.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-utils-internal", - "version": "1.3.2", + "version": "1.3.4", "license": "Apache 2.0", "devDependencies": { "@types/node": "^24.2.0", diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index e6463c8e58..8ad4fa6e10 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.3.3", + "version": "1.3.4", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/cli/windmill-utils-internal/src/parse/parse-schema.ts b/cli/windmill-utils-internal/src/parse/parse-schema.ts index bb2e2078af..1887ef2d40 100644 --- a/cli/windmill-utils-internal/src/parse/parse-schema.ts +++ b/cli/windmill-utils-internal/src/parse/parse-schema.ts @@ -108,6 +108,9 @@ export function argSigToJsonSchemaType( } else if (t === "datetime") { newS.type = "string"; newS.format = "date-time"; + } else if (t === "date") { + newS.type = "string"; + newS.format = "date"; } else if (typeof t !== "string" && "oneof" in t) { newS.type = "object"; if (t.oneof) { diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index 4b67e06abf..3fedbd8d37 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -34,6 +34,7 @@ export const LANGUAGE_EXTENSIONS: Record = { java: "java", duckdb: "duckdb.sql", bunnative: "ts", + ruby: "rb", // for related places search: ADD_NEW_LANG }; @@ -82,6 +83,7 @@ export const EXTENSION_TO_LANGUAGE: Record = { "playbook.yml": "ansible", "java": "java", "duckdb.sql": "duckdb", + "rb": "ruby", // Plain .ts defaults to bun (will be overridden by defaultTs setting) "ts": "bun", }; diff --git a/docker/RHEL8/Dockerfile b/docker/RHEL8/Dockerfile index eaaa01ea87..6832e88625 100644 --- a/docker/RHEL8/Dockerfile +++ b/docker/RHEL8/Dockerfile @@ -16,6 +16,8 @@ RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version ^0.1 WORKDIR /windmill ENV SQLX_OFFLINE=true +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=cc +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS="" # ENV CARGO_INCREMENTAL=1 FROM node:20-alpine as frontend @@ -62,7 +64,7 @@ RUN --mount=type=secret,id=rh_username \ RUN subscription-manager repos --enable codeready-builder-for-rhel-8-$(arch)-rpms RUN yum update -y && \ - yum install -y perl-interpreter perl-IPC-Cmd perl-Time-Piece libxml2-devel xmlsec1-devel xmlsec1-openssl-devel clang llvm-devel cmake libtool-ltdl-devel + yum install -y perl-interpreter perl-IPC-Cmd perl-Time-Piece libxml2-devel xmlsec1-devel xmlsec1-openssl-devel krb5-devel cyrus-sasl-devel cyrus-sasl-gssapi libcurl-devel clang llvm-devel cmake libtool-ltdl-devel # RUN --mount=type=cache,target=/usr/local/cargo/registry \ # CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index ebcba5ecc4..7bfeb48eff 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -16,6 +16,8 @@ RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version ^0.1 WORKDIR /windmill ENV SQLX_OFFLINE=true +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=cc +ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS="" # ENV CARGO_INCREMENTAL=1 FROM node:20-alpine as frontend @@ -62,7 +64,7 @@ RUN --mount=type=secret,id=rh_username \ RUN subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms RUN yum update -y && \ - yum install -y perl-FindBin perl-IPC-Cmd perl-Time-Piece libxml2-devel xmlsec1-devel xmlsec1-openssl-devel clang llvm-devel cmake libtool-ltdl-devel + yum install -y perl-FindBin perl-IPC-Cmd perl-Time-Piece libxml2-devel xmlsec1-devel xmlsec1-openssl-devel krb5-devel cyrus-sasl-devel cyrus-sasl-gssapi libcurl-devel clang llvm-devel cmake libtool-ltdl-devel # RUN --mount=type=cache,target=/usr/local/cargo/registry \ # CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json diff --git a/ephemeral-backends/.env.template b/ephemeral-backends/.env.template new file mode 100644 index 0000000000..68d522b862 --- /dev/null +++ b/ephemeral-backends/.env.template @@ -0,0 +1,15 @@ +# Windmill Ephemeral Backend Manager - Environment Variables +# Copy this file to /etc/windmill-ephemeral-manager/.env and fill in the values + +# Manager Authentication Token +# Required for protecting the manager endpoints from unauthorized access +# Generate a secure random token: openssl rand -hex 32 +MANAGER_AUTH_TOKEN=your-secure-random-token-here + +# GitHub Personal Access Token with 'secrets' scope (Write access) +GITHUB_TOKEN=github_pat_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + +# Path to the SSH deploy key for accessing windmill-ee-private repository +# This should be a read-only deploy key +# The file should be owned by the 'sandbox' user with 600 permissions +GIT_EE_DEPLOY_KEY_FILE=/home/sandbox/.ssh/windmill_ee_deploy_key diff --git a/ephemeral-backends/.gitignore b/ephemeral-backends/.gitignore new file mode 100644 index 0000000000..7f5ed9b3af --- /dev/null +++ b/ephemeral-backends/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +dist/ +*.log +.DS_Store +.env diff --git a/ephemeral-backends/DEPLOYMENT.md b/ephemeral-backends/DEPLOYMENT.md new file mode 100644 index 0000000000..e0b6884450 --- /dev/null +++ b/ephemeral-backends/DEPLOYMENT.md @@ -0,0 +1,35 @@ +# Deploying the Ephemeral Backend Manager + +This guide explains how to deploy the Windmill Ephemeral Backend Manager as a systemd service on a Linux machine. + +## Prerequisites + +Before running the installation script, ensure you have: + +1. **Linux machine** with systemd (Ubuntu 20.04+, Debian 11+, etc.) +2. **User `sandbox`** created with appropriate permissions +3. **Repository cloned** to `/home/sandbox/ephemeral-backend/windmill` +4. **Required tools installed**: + - Git + - Docker (with `sandbox` user having access) + - Bun (installed for the `sandbox` user) + - Rust/Cargo (for building backends) + - Bubblewrap (will be installed by the script if missing) + - Cloudflared (will be installed by the script if missing) + +## Architecture + +The service runs as user `sandbox` and: + +- Listens on port 8001 for HTTP requests +- Creates a Cloudflare tunnel for external access +- Updates GitHub Actions secrets with the tunnel URL +- Manages a pool of git worktrees for ephemeral backends +- Spawns ephemeral backends on-demand +- Automatically cleans up resources on shutdown + +The worktree pool enables: + +- Fast incremental compilation (reuses target directories) +- Efficient resource usage (no constant creation/deletion) +- Automatic discovery of existing worktrees on restart diff --git a/ephemeral-backends/install.sh b/ephemeral-backends/install.sh new file mode 100755 index 0000000000..9508ee5419 --- /dev/null +++ b/ephemeral-backends/install.sh @@ -0,0 +1,167 @@ +#!/bin/bash +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "🎛️ Windmill Ephemeral Backend Manager - Installation Script" +echo "==========================================================" +echo "" + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + echo -e "${RED}❌ This script must be run as root (use sudo)${NC}" + exit 1 +fi + +# Get the actual user who ran sudo (if applicable) +ACTUAL_USER="${SUDO_USER:-$(whoami)}" + +# Configuration +SERVICE_USER="sandbox" +SERVICE_NAME="windmill-ephemeral-manager" +REPO_DIR="/home/$SERVICE_USER/ephemeral-backend/windmill" +ENV_DIR="/etc/$SERVICE_NAME" +ENV_FILE="$ENV_DIR/.env" + +echo "📋 Configuration:" +echo " Service user: $SERVICE_USER" +echo " Repository directory: $REPO_DIR" +echo " Environment file: $ENV_FILE" +echo "" + +# Check if sandbox user exists +if ! id "$SERVICE_USER" &>/dev/null; then + echo -e "${RED}❌ User '$SERVICE_USER' does not exist${NC}" + echo " Create the user first: sudo adduser --system --group $SERVICE_USER" + exit 1 +fi + +# Check if repository directory exists +if [ ! -d "$REPO_DIR" ]; then + echo -e "${RED}❌ Repository directory does not exist: $REPO_DIR${NC}" + echo " Clone the repository first as user $SERVICE_USER" + exit 1 +fi + +# Check if bun is installed for the service user +if ! sudo -u "$SERVICE_USER" bash -c "command -v bun" &>/dev/null; then + echo -e "${RED}❌ Bun is not installed for user $SERVICE_USER${NC}" + echo " Install bun: curl -fsSL https://bun.sh/install | bash" + exit 1 +fi + +# Check if bubblewrap is installed +if ! command -v bwrap &>/dev/null; then + echo -e "${YELLOW}⚠️ bubblewrap is not installed${NC}" + echo " Installing bubblewrap..." + apt-get update -qq + apt-get install -y -qq bubblewrap + echo -e "${GREEN}✓ bubblewrap installed${NC}" +fi + +# Check if cloudflared is installed +if ! command -v cloudflared &>/dev/null; then + echo -e "${YELLOW}⚠️ cloudflared is not installed${NC}" + echo " Installing cloudflared..." + + # Detect architecture + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb + dpkg -i cloudflared-linux-amd64.deb + rm cloudflared-linux-amd64.deb + elif [ "$ARCH" = "aarch64" ]; then + wget -q https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-arm64.deb + dpkg -i cloudflared-linux-arm64.deb + rm cloudflared-linux-arm64.deb + else + echo -e "${RED}❌ Unsupported architecture: $ARCH${NC}" + exit 1 + fi + + echo -e "${GREEN}✓ cloudflared installed${NC}" +fi + +# Check if docker is installed and service user has access +if ! command -v docker &>/dev/null; then + echo -e "${RED}❌ Docker is not installed${NC}" + echo " Install docker: https://docs.docker.com/engine/install/" + exit 1 +fi + +if ! sudo -u "$SERVICE_USER" docker ps &>/dev/null; then + echo -e "${YELLOW}⚠️ User $SERVICE_USER cannot access Docker${NC}" + echo " Adding $SERVICE_USER to docker group..." + usermod -aG docker "$SERVICE_USER" + echo -e "${GREEN}✓ User added to docker group (may require logout/login)${NC}" +fi + +# Create environment directory +echo "" +echo "📁 Setting up environment..." +mkdir -p "$ENV_DIR" +chmod 755 "$ENV_DIR" + +# Copy or create environment file +if [ ! -f "$ENV_FILE" ]; then + if [ -f "$REPO_DIR/ephemeral-backends/.env.template" ]; then + cp "$REPO_DIR/ephemeral-backends/.env.template" "$ENV_FILE" + echo -e "${YELLOW}⚠️ Environment file created from template: $ENV_FILE${NC}" + echo " ${RED}IMPORTANT: Edit this file and fill in the required values!${NC}" + else + echo -e "${RED}❌ Template file not found: $REPO_DIR/ephemeral-backends/.env.template${NC}" + exit 1 + fi +else + echo -e "${GREEN}✓ Environment file already exists: $ENV_FILE${NC}" +fi + +chmod 600 "$ENV_FILE" + +# Install systemd service +echo "" +echo "🔧 Installing systemd service..." +cp "$REPO_DIR/ephemeral-backends/windmill-ephemeral-manager.service" "/etc/systemd/system/$SERVICE_NAME.service" +chmod 644 "/etc/systemd/system/$SERVICE_NAME.service" +echo -e "${GREEN}✓ Service file installed${NC}" + +# Reload systemd +echo "" +echo "🔄 Reloading systemd..." +systemctl daemon-reload +echo -e "${GREEN}✓ Systemd reloaded${NC}" + +# Enable service +echo "" +echo "✅ Enabling service..." +systemctl enable "$SERVICE_NAME.service" +echo -e "${GREEN}✓ Service enabled (will start on boot)${NC}" + +echo "" +echo "==========================================================" +echo -e "${GREEN}✅ Installation complete!${NC}" +echo "" +echo "📝 Next steps:" +echo " 1. Edit the environment file: sudo nano $ENV_FILE" +echo " 2. Fill in the required values:" +echo " - MANAGER_AUTH_TOKEN (generate with: openssl rand -hex 32)" +echo " - GITHUB_TOKEN (GitHub personal access token)" +echo " - GIT_EE_DEPLOY_KEY_FILE (path to SSH deploy key)" +echo " 3. Ensure the SSH deploy key exists and is readable by $SERVICE_USER" +echo " 4. Add MANAGER_AUTH_TOKEN to GitHub repository secrets" +echo " 5. Start the service: sudo systemctl start $SERVICE_NAME" +echo " 6. Check status: sudo systemctl status $SERVICE_NAME" +echo " 7. View logs: sudo journalctl -u $SERVICE_NAME -f" +echo "" +echo "🔍 Useful commands:" +echo " sudo systemctl start $SERVICE_NAME # Start the service" +echo " sudo systemctl stop $SERVICE_NAME # Stop the service" +echo " sudo systemctl restart $SERVICE_NAME # Restart the service" +echo " sudo systemctl status $SERVICE_NAME # Check service status" +echo " sudo journalctl -u $SERVICE_NAME -f # Follow logs" +echo " sudo journalctl -u $SERVICE_NAME -n 100 # Last 100 log lines" +echo "" diff --git a/ephemeral-backends/logger.ts b/ephemeral-backends/logger.ts new file mode 100644 index 0000000000..661d0ca6d8 --- /dev/null +++ b/ephemeral-backends/logger.ts @@ -0,0 +1,131 @@ +import { + appendFileSync, + mkdirSync, + unlinkSync, + readdirSync, + statSync, +} from "fs"; +import path from "path"; + +const LOG_DIR = "/tmp/windmill-ephemeral-logs"; +const LOG_RETENTION_MS = 24 * 60 * 60 * 1000; // 24 hours + +export class Logger { + private logFilePath: string; + private prefix: string; + + constructor(identifier: string, prefix: string = "") { + // Sanitize identifier to prevent path traversal + const sanitizedId = identifier.replace(/[^a-zA-Z0-9-_]/g, "_"); + + // Ensure log directory exists + try { + mkdirSync(LOG_DIR, { recursive: true }); + } catch (error) { + // Directory might already exist + } + + this.logFilePath = path.join(LOG_DIR, `${sanitizedId}.log`); + this.prefix = prefix; + + // Write initial log entry + const startTime = new Date().toISOString(); + this.log(`=== Log started at ${startTime} ===`); + } + + private formatMessage(level: string, message: string): string { + const timestamp = new Date().toISOString(); + const prefixStr = this.prefix ? `[${this.prefix}] ` : ""; + return `${timestamp} ${level} ${prefixStr}${message}\n`; + } + + log(message: string): void { + const formatted = this.formatMessage("stdin ", message); + try { + appendFileSync(this.logFilePath, formatted); + } catch (error) { + // Fallback to console if file write fails + console.error("Failed to write to log file:", error); + console.log(formatted); + } + // Also log to stdout for systemd journal + process.stdout.write(formatted); + } + + error(message: string): void { + const formatted = this.formatMessage("stderr", message); + try { + appendFileSync(this.logFilePath, formatted); + } catch (error) { + console.error("Failed to write to log file:", error); + console.error(formatted); + } + // Also log to stderr for systemd journal + process.stderr.write(formatted); + } + + warn(message: string): void { + const formatted = this.formatMessage("warn ", message); + try { + appendFileSync(this.logFilePath, formatted); + } catch (error) { + console.error("Failed to write to log file:", error); + console.warn(formatted); + } + process.stdout.write(formatted); + } + + getLogFilePath(): string { + return this.logFilePath; + } + static getLogFilePathForCommit(commitHash: string): string { + // Sanitize commit hash to prevent path traversal + const sanitizedHash = commitHash.replace(/[^a-f0-9]/g, ""); + if (sanitizedHash.length < 7 || sanitizedHash.length > 40) { + throw new Error("Invalid commit hash"); + } + return path.join(LOG_DIR, `${sanitizedHash}.log`); + } + + /** + * Clean up old log files (older than 24 hours) + */ + static cleanupOldLogs(): void { + try { + const files = readdirSync(LOG_DIR); + const now = Date.now(); + let deletedCount = 0; + + for (const file of files) { + if (!file.endsWith(".log")) continue; + + const filePath = path.join(LOG_DIR, file); + try { + const stats = statSync(filePath); + const ageMs = now - stats.mtimeMs; + + if (ageMs > LOG_RETENTION_MS) { + unlinkSync(filePath); + deletedCount++; + console.log( + `Deleted old log file: ${file} (age: ${Math.floor( + ageMs / 1000 / 60 / 60 + )}h)` + ); + } + } catch (error) { + // Skip files we can't stat or delete + console.error(`Failed to process log file ${file}:`, error); + } + } + + if (deletedCount > 0) { + console.log( + `Cleanup complete: deleted ${deletedCount} old log file(s)` + ); + } + } catch (error) { + console.error("Failed to cleanup old logs:", error); + } + } +} diff --git a/ephemeral-backends/manager.ts b/ephemeral-backends/manager.ts new file mode 100644 index 0000000000..7f8033b1dc --- /dev/null +++ b/ephemeral-backends/manager.ts @@ -0,0 +1,687 @@ +#!/usr/bin/env node + +import { spawn } from "child_process"; +import * as readline from "readline"; +import sodium from "libsodium-wrappers-sumo"; +import { EphemeralBackend } from "./spawn"; +import { WorktreePool } from "./worktree-pool"; +import { Logger } from "./logger"; +import { readFileSync, existsSync } from "fs"; + +process.on("unhandledRejection", (err) => { + console.error("UNHANDLED PROMISE:", err); +}); + +process.on("uncaughtException", (err) => { + console.error("UNCAUGHT EXCEPTION:", err); +}); + +const githubToken = process.env.GITHUB_TOKEN; +if (!githubToken) { + console.log("⚠️ GITHUB_TOKEN environment variable not set"); + console.log("\n📝 Set a GitHub token with 'secrets' scope:"); + console.log(" export GITHUB_TOKEN=github_pat_..."); + process.exit(1); +} + +const managerAuthToken = process.env.MANAGER_AUTH_TOKEN; +if (!managerAuthToken) { + console.log("⚠️ MANAGER_AUTH_TOKEN environment variable not set"); + console.log("\n📝 Set a secure random token for API authentication:"); + console.log(" export MANAGER_AUTH_TOKEN=$(openssl rand -hex 32)"); + process.exit(1); +} + +if (!process.env.GIT_EE_DEPLOY_KEY_FILE) { + console.log("⚠️ GIT_EE_DEPLOY_KEY_FILE environment variable not set"); + console.log("\n📝 Set a read-only SSH deploy key:"); + console.log(" export GIT_EE_DEPLOY_KEY_FILE=/home/..."); + process.exit(1); +} + +const MANAGER_PORT = 8001; +const BACKEND_TIMEOUT_MS = 120 * 60 * 1000; // 2 hours in milliseconds + +interface BackendInfo { + backend: EphemeralBackend; + timeoutId: NodeJS.Timeout; + createdAt: Date; +} + +interface ManagerResources { + cloudflaredProcess?: any; + tunnelUrl?: string; + ephemeralBackends: Map; + worktreePool?: WorktreePool; + cleanupInterval?: NodeJS.Timeout; +} + +class EphemeralBackendManager { + private resources: ManagerResources = { + ephemeralBackends: new Map(), + }; + private server?: any; + + async start(): Promise { + // Setup cleanup handlers early + process.on("SIGINT", () => this.cleanup()); + process.on("SIGTERM", () => this.cleanup()); + + try { + console.log("🎛️ Starting Ephemeral Backend Manager..."); + console.log(`📊 Manager port: ${MANAGER_PORT}`); + + // Initialize the worktree pool + this.resources.worktreePool = new WorktreePool(); + await this.resources.worktreePool.initialize(); + + // Set up periodic log cleanup (every 6 hours) + this.resources.cleanupInterval = setInterval(() => { + console.log("\n🧹 Running periodic log cleanup..."); + Logger.cleanupOldLogs(); + }, 6 * 60 * 60 * 1000); + + await this.startHttpServer(); + if (!process.env.SKIP_CLOUDFLARED) await this.startCloudflared(); + if (!process.env.SKIP_SET_GH_SECRET) await this.updateGitHubSecret(); + + console.log("\n✅ Manager is ready!"); + console.log(`📍 Tunnel URL: ${this.resources.tunnelUrl}`); + + console.log("\n💡 Press Ctrl+C to stop..."); + + // Keep the process running indefinitely + await new Promise(() => {}); // Never resolves + } catch (error) { + console.error("❌ Error starting manager:", error); + await this.cleanup(); + process.exit(1); + } + } + + private async startHttpServer(): Promise { + const self = this; + console.log("\n🌐 Starting HTTP server..."); + + return new Promise((resolve) => { + // Use Bun's built-in HTTP server + this.server = Bun.serve({ + port: MANAGER_PORT, + idleTimeout: 30, + async fetch(req) { + const url = new URL(req.url); + + // CORS headers for app.windmill.dev + const origin = req.headers.get("origin"); + const corsHeaders: Record = {}; + + if (origin === "https://app.windmill.dev") { + corsHeaders["Access-Control-Allow-Origin"] = origin; + corsHeaders["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS"; + corsHeaders["Access-Control-Allow-Headers"] = + "Content-Type, Authorization"; + corsHeaders["Access-Control-Max-Age"] = "86400"; + } + + // Handle preflight requests + if (req.method === "OPTIONS") { + return new Response(null, { + status: 204, + headers: corsHeaders, + }); + } + + // Authentication check function + const checkAuth = (): boolean => { + const authHeader = req.headers.get("authorization"); + return authHeader === `Bearer ${managerAuthToken}`; + }; + + // Unauthorized response helper + const unauthorizedResponse = () => { + return new Response( + JSON.stringify({ + error: "Unauthorized", + message: "Valid Bearer token required in Authorization header", + }), + { + status: 401, + headers: { + ...corsHeaders, + "Content-Type": "application/json", + "WWW-Authenticate": 'Bearer realm="Manager API"', + }, + } + ); + }; + + // Health check endpoint + if (url.pathname === "/health") { + return new Response( + JSON.stringify({ + status: "ok", + timestamp: new Date().toISOString(), + }), + { + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // Status endpoint - shows all running backends and worktree pool stats + if (url.pathname === "/status") { + if (!checkAuth()) { + return unauthorizedResponse(); + } + + const backends = Array.from( + self.resources.ephemeralBackends.entries() + ).map(([commitHash, backendInfo]) => { + const now = new Date(); + const timeoutAt = new Date( + backendInfo.createdAt.getTime() + BACKEND_TIMEOUT_MS + ); + const timeRemainingMs = timeoutAt.getTime() - now.getTime(); + const timeRemainingMinutes = Math.floor( + timeRemainingMs / 1000 / 60 + ); + + return { + commitHash, + shortHash: commitHash.substring(0, 8), + createdAt: backendInfo.createdAt.toISOString(), + timeoutAt: timeoutAt.toISOString(), + timeRemainingMinutes, + serverPort: backendInfo.backend.getServerPort(), + dbPort: backendInfo.backend.getDbPort(), + }; + }); + + const worktreePoolStats = + self.resources.worktreePool?.getStats() || { + total: 0, + inUse: 0, + available: 0, + }; + + return new Response( + JSON.stringify({ + activeBackends: backends.length, + backends, + worktreePool: worktreePoolStats, + timestamp: new Date().toISOString(), + }), + { + headers: { ...corsHeaders, "Content-Type": "application/json" }, + } + ); + } + + // Match /logs/{commit_hash} - serve log files + const logsMatch = url.pathname.match(/^\/logs\/([a-f0-9]+)$/); + if (logsMatch && req.method === "GET") { + if (!checkAuth()) { + return unauthorizedResponse(); + } + + const commitHash = logsMatch[1]; + + // Validate commit hash format (7-40 hex characters) + if (commitHash.length < 7 || commitHash.length > 40) { + return new Response("Invalid commit hash", { status: 400 }); + } + + try { + // Use the Logger class's secure path resolution + const logFilePath = Logger.getLogFilePathForCommit(commitHash); + + // Check if file exists + if (!existsSync(logFilePath)) { + return new Response("Log file not found", { + status: 404, + headers: corsHeaders, + }); + } + + // Read the log file + const logContent = readFileSync(logFilePath, "utf-8"); + + return new Response(logContent, { + headers: { + ...corsHeaders, + "Content-Type": "text/plain; charset=utf-8", + "X-Commit-Hash": commitHash, + }, + }); + } catch (error: any) { + console.error(`Error reading log file for ${commitHash}:`, error); + return new Response(`Error reading log file: ${error.message}`, { + status: 500, + headers: corsHeaders, + }); + } + } + + // Match /spawn/{commit_hash} + const spawnMatch = url.pathname.match(/^\/spawn\/([a-f0-9]+)$/); + if (spawnMatch && req.method === "POST") { + if (!checkAuth()) { + return unauthorizedResponse(); + } + + let body = await req.json(); + if (typeof body !== "object") + return new Response("Invalid JSON body", { status: 400 }); + let resumeUrl = body?.resume_url; + if (typeof resumeUrl !== "string") + return new Response("Invalid resume_url", { status: 400 }); + let cancelUrl = body?.cancel_url; + if (typeof cancelUrl !== "string") + return new Response("Invalid cancel_url", { status: 400 }); + const commitHash = spawnMatch[1]; + console.log( + `\n🔹 Received request to spawn ephemeral backend for commit: ${commitHash}` + ); + if (self.resources.ephemeralBackends.has(commitHash)) { + throw new Error(`Backend ${commitHash} is already running`); + } + + if (!self.resources.worktreePool) { + throw new Error("Worktree pool not initialized"); + } + + const tunnelUrl = await new Promise((res, err) => { + const adminRandomPwd = Math.random() + .toString(36) + .substring(2, 15); + const ephemeralBackend = new EphemeralBackend({ + dbPort: self.findFreeDbPorts(), + serverPort: self.findFreeServerPorts(), + skipBuild: !!process.env.SKIP_BACKEND_BUILD, + commitHash: commitHash, + worktreePool: self.resources.worktreePool!, + adminPwd: adminRandomPwd, + onCloudflaredUrl: (url) => (res(url), clearTimeout(timeout)), + onCleanup: () => { + const backendInfo = + self.resources.ephemeralBackends.get(commitHash); + if (backendInfo) { + clearTimeout(backendInfo.timeoutId); + self.resources.ephemeralBackends.delete(commitHash); + } + }, + }); + function onError(e: any) { + ephemeralBackend.cleanup().catch(() => { + console.error( + `Failed to cleanup backend for commit ${commitHash}` + ); + }); + clearTimeout(timeout); + fetch(resumeUrl, { + // Cancel URL doesn't show any relevant info, use resume URL + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: "error", + commitHash, + error: e.message, + }), + }).catch((e) => { + console.error( + `Failed to notify cancel URL for commit ${commitHash}:`, + e + ); + }); + } + const timeout = setTimeout(() => { + onError(new Error("Timeout waiting for backend URL")); + }, 20000); + try { + ephemeralBackend + .spawn() + .then(({ tunnelUrl }) => { + if (resumeUrl) { + fetch(resumeUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: "ready", + timeoutAt: + (self.resources.ephemeralBackends + .get(commitHash) + ?.createdAt.getTime() ?? 0) + BACKEND_TIMEOUT_MS, + commitHash, + adminPassword: adminRandomPwd, + tunnelUrl, + }), + }).catch((e) => { + onError(e); + }); + } + }) + .catch((e) => onError(e)); + + // Set up 1-hour timeout for automatic cleanup + const cleanupTimeoutId = setTimeout(async () => { + console.log( + `\n⏰ Backend ${commitHash} has reached 1-hour timeout, cleaning up...` + ); + try { + await ephemeralBackend.cleanup(); + self.resources.ephemeralBackends.delete(commitHash); + console.log( + `✓ Backend ${commitHash} cleaned up after timeout` + ); + } catch (error) { + console.error( + `❌ Failed to cleanup backend ${commitHash} after timeout:`, + error + ); + } + }, BACKEND_TIMEOUT_MS); + + self.resources.ephemeralBackends.set(commitHash, { + backend: ephemeralBackend, + timeoutId: cleanupTimeoutId, + createdAt: new Date(), + }); + } catch (e) { + onError(e); + } + }); + + return new Response( + JSON.stringify({ + tunnelUrl, + timestamp: new Date().toISOString(), + }), + { + headers: { ...corsHeaders, "Content-Type": "application/json" }, + status: 202, + } + ); + } + + // Default 404 + return new Response("Not Found", { + status: 404, + headers: corsHeaders, + }); + }, + }); + + console.log(`✓ HTTP server listening on port ${MANAGER_PORT}`); + resolve(); + }); + } + + private async startCloudflared(): Promise { + console.log("\n🌐 Starting Cloudflare tunnel for manager..."); + + return new Promise((resolve, reject) => { + this.resources.cloudflaredProcess = spawn("cloudflared", [ + "tunnel", + "--url", + `http://localhost:${MANAGER_PORT}`, + "--config", + "/dev/null", + ]); + + const rl = readline.createInterface({ + input: this.resources.cloudflaredProcess.stdout, + }); + + rl.on("line", (line: string) => { + console.log(`[cloudflared] ${line}`); + }); + + this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => { + process.stderr.write(`[cloudflared] ${data}`); + const line = data.toString(); + const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/); + if (match) { + this.resources.tunnelUrl = match[1]; + console.log(`✓ Tunnel URL extracted: ${this.resources.tunnelUrl}`); + resolve(); + } + }); + + this.resources.cloudflaredProcess.on("close", (code: number) => { + console.log(`Cloudflared process exited with code ${code}`); + }); + + // Timeout if we can't find the URL in 30 seconds + setTimeout(() => { + if (!this.resources.tunnelUrl) { + reject(new Error("Failed to extract Cloudflare tunnel URL")); + } + }, 30000); + }); + } + + private async updateGitHubSecret(): Promise { + console.log("\n🔐 Updating GitHub Actions secret..."); + + if (!this.resources.tunnelUrl) { + console.error("❌ No tunnel URL available to update secret"); + return; + } + + const fullUrl = `https://${this.resources.tunnelUrl}`; + const repo = "windmill-labs/windmill"; + const secretName = "EPHEMERAL_BACKEND_QUEUE_URL"; + + try { + // First, get the repository public key for encrypting the secret + console.log(" Fetching repository public key..."); + const keyResponse = await fetch( + `https://api.github.com/repos/${repo}/actions/secrets/public-key`, + { + headers: { + Authorization: `Bearer ${githubToken}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + } + ); + + if (!keyResponse.ok) { + throw new Error( + `Failed to fetch public key: ${keyResponse.statusText}` + ); + } + + const { key, key_id } = await keyResponse.json(); + + // Encrypt the secret using libsodium (via tweetnacl for Bun compatibility) + console.log(" Encrypting secret value..."); + await sodium.ready; + + const messageBytes = new TextEncoder().encode(fullUrl); + const keyBytes = sodium.from_base64(key, sodium.base64_variants.ORIGINAL); + const encryptedBytes = sodium.crypto_box_seal(messageBytes, keyBytes); + const encryptedValue = sodium.to_base64( + encryptedBytes, + sodium.base64_variants.ORIGINAL + ); + + // Update the secret + console.log(" Updating secret..."); + const updateResponse = await fetch( + `https://api.github.com/repos/${repo}/actions/secrets/${secretName}`, + { + method: "PUT", + headers: { + Authorization: `Bearer ${githubToken}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + }, + body: JSON.stringify({ + encrypted_value: encryptedValue, + key_id: key_id, + }), + } + ); + + if (!updateResponse.ok) { + const errorText = await updateResponse.text(); + throw new Error( + `Failed to update secret: ${updateResponse.statusText} - ${errorText}` + ); + } + + console.log(`✓ GitHub secret updated successfully!`); + console.log(` Repository: ${repo}`); + console.log(` Secret: ${secretName}`); + console.log(` Value: ${fullUrl}`); + } catch (error: any) { + console.error("❌ Failed to update GitHub secret:", error.message); + } + } + + private async deleteGitHubSecret(): Promise { + const repo = "windmill-labs/windmill"; + const secretName = "EPHEMERAL_BACKEND_QUEUE_URL"; + + try { + console.log(" Deleting GitHub Actions secret..."); + const deleteResponse = await fetch( + `https://api.github.com/repos/${repo}/actions/secrets/${secretName}`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${githubToken}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + } + ); + + if (!deleteResponse.ok) { + // 404 is acceptable - secret might not exist + if (deleteResponse.status === 404) { + console.log( + ` ✓ Secret ${secretName} does not exist (already deleted)` + ); + return; + } + const errorText = await deleteResponse.text(); + throw new Error( + `Failed to delete secret: ${deleteResponse.statusText} - ${errorText}` + ); + } + + console.log(`✓ GitHub secret deleted successfully!`); + console.log(` Repository: ${repo}`); + console.log(` Secret: ${secretName}`); + } catch (error: any) { + console.error("❌ Failed to delete GitHub secret:", error.message); + } + } + + isCleaningUp: boolean = false; + private async cleanup(): Promise { + if (this.isCleaningUp) return; + this.isCleaningUp = true; + console.log("\n🧹 Cleaning up manager resources..."); + + // Stop periodic cleanup interval + if (this.resources.cleanupInterval) { + console.log(" Stopping periodic log cleanup..."); + clearInterval(this.resources.cleanupInterval); + this.resources.cleanupInterval = undefined; + } + + // Delete GitHub secret + if (!process.env.SKIP_SET_GH_SECRET) { + console.log(" Deleting GitHub Actions secret..."); + try { + await this.deleteGitHubSecret(); + } catch (error) { + console.error(" Failed to delete GitHub secret:", error); + } + } + + // Stop HTTP server + if (this.server) { + console.log(" Stopping HTTP server..."); + try { + this.server.stop(); + } catch (error) { + console.error(" Failed to stop HTTP server:", error); + } + } + + // Kill cloudflared process + if (this.resources.cloudflaredProcess) { + console.log(" Stopping cloudflared..."); + try { + this.resources.cloudflaredProcess.kill("SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + this.resources.cloudflaredProcess.kill("SIGKILL"); + } catch (error) { + // Process might already be dead + } + } + + for (const [commitHash, backendInfo] of this.resources.ephemeralBackends) { + const hash = commitHash.substring(0, 8); + console.log( + ` Cleaning up ephemeral backend ${hash} on port ${backendInfo.backend.getServerPort()}...` + ); + try { + clearTimeout(backendInfo.timeoutId); // Clear the timeout before cleanup + await backendInfo.backend.cleanup(); + } catch (error) { + console.error( + ` Failed to clean up backend ${hash} on port ${backendInfo.backend.getServerPort()}:`, + error + ); + } + } + + console.log("✅ Cleanup complete"); + process.exit(0); + } + + private findFreeDbPorts(): number { + const minPort = 5433; + for (let port = minPort; port < minPort + 100; port++) { + if ( + ![...this.resources.ephemeralBackends.values()].some( + (backendInfo) => port === backendInfo.backend.getDbPort() + ) + ) { + return port; + } + } + throw new Error("No free DB ports available"); + } + + private findFreeServerPorts(): number { + const minPort = 8002; + for (let port = minPort; port < minPort + 100; port++) { + if ( + ![...this.resources.ephemeralBackends.values()].some( + (backendInfo) => port === backendInfo.backend.getServerPort() + ) + ) { + return port; + } + } + throw new Error("No free server ports available"); + } +} + +// Main execution +async function main() { + const manager = new EphemeralBackendManager(); + await manager.start(); +} + +main().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); diff --git a/ephemeral-backends/package-lock.json b/ephemeral-backends/package-lock.json new file mode 100644 index 0000000000..f87fe39e23 --- /dev/null +++ b/ephemeral-backends/package-lock.json @@ -0,0 +1,88 @@ +{ + "name": "windmill-ephemeral-backends", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "windmill-ephemeral-backends", + "version": "1.0.0", + "dependencies": { + "@types/libsodium-wrappers-sumo": "^0.7.8", + "libsodium-wrappers-sumo": "^0.8.1" + }, + "devDependencies": { + "@types/bun": "^1.3.6", + "@types/node": "^20.0.0", + "bun-types": "^1.0.0" + } + }, + "node_modules/@types/bun": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.3.6.tgz", + "integrity": "sha512-uWCv6FO/8LcpREhenN1d1b6fcspAB+cefwD7uti8C8VffIv0Um08TKMn98FynpTiU38+y2dUO55T11NgDt8VAA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.3.6" + } + }, + "node_modules/@types/libsodium-wrappers": { + "version": "0.7.14", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers/-/libsodium-wrappers-0.7.14.tgz", + "integrity": "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ==", + "license": "MIT" + }, + "node_modules/@types/libsodium-wrappers-sumo": { + "version": "0.7.8", + "resolved": "https://registry.npmjs.org/@types/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.7.8.tgz", + "integrity": "sha512-N2+df4MB/A+W0RAcTw7A5oxKgzD+Vh6Ye7lfjWIi5SdTzVLfHPzxUjhwPqHLO5Ev9fv/+VHl+sUaUuTg4fUPqw==", + "license": "MIT", + "dependencies": { + "@types/libsodium-wrappers": "*" + } + }, + "node_modules/@types/node": { + "version": "20.19.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.30.tgz", + "integrity": "sha512-WJtwWJu7UdlvzEAUm484QNg5eAoq5QR08KDNx7g45Usrs2NtOPiX8ugDqmKdXkyL03rBqU5dYNYVQetEpBHq2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/bun-types": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.3.6.tgz", + "integrity": "sha512-OlFwHcnNV99r//9v5IIOgQ9Uk37gZqrNMCcqEaExdkVq3Avwqok1bJFmvGMCkCE0FqzdY8VMOZpfpR3lwI+CsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/libsodium-sumo": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/libsodium-sumo/-/libsodium-sumo-0.8.1.tgz", + "integrity": "sha512-q8EhpXKjbzWsLZEY3qOym5nA7emtTe/izt6ziSUbUfE8hk2eDP+7Hy0G0bum65JHDChkdtrImNZTR7N5WuiIXg==", + "license": "ISC" + }, + "node_modules/libsodium-wrappers-sumo": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/libsodium-wrappers-sumo/-/libsodium-wrappers-sumo-0.8.1.tgz", + "integrity": "sha512-xcX1+mQbdwko50AsZWn5r4crZO5H5bHh9ceMHsjBxPyrMn0zFHycms2jcHvXzk71nh4V0rQGCiOjB2ckshd/ew==", + "license": "ISC", + "dependencies": { + "libsodium-sumo": "^0.8.0" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/ephemeral-backends/package.json b/ephemeral-backends/package.json new file mode 100644 index 0000000000..74d662eefe --- /dev/null +++ b/ephemeral-backends/package.json @@ -0,0 +1,19 @@ +{ + "name": "windmill-ephemeral-backends", + "version": "1.0.0", + "description": "Ephemeral backend spawner for Windmill", + "type": "module", + "scripts": { + "start": "bun run spawn.ts", + "manager": "bun run manager.ts" + }, + "dependencies": { + "@types/libsodium-wrappers-sumo": "^0.7.8", + "libsodium-wrappers-sumo": "^0.8.1" + }, + "devDependencies": { + "@types/bun": "^1.3.6", + "@types/node": "^20.0.0", + "bun-types": "^1.0.0" + } +} diff --git a/ephemeral-backends/spawn.ts b/ephemeral-backends/spawn.ts new file mode 100644 index 0000000000..b334908159 --- /dev/null +++ b/ephemeral-backends/spawn.ts @@ -0,0 +1,670 @@ +#!/usr/bin/env node + +import { spawn, exec } from "child_process"; +import { promisify } from "util"; +import * as readline from "readline"; +import { WorktreePool, WorktreeInfo } from "./worktree-pool"; +import { Logger } from "./logger"; + +const execAsync = promisify(exec); + +export interface Config { + dbPort: number; + serverPort: number; + skipBuild: boolean; + commitHash: string; + worktreePool: WorktreePool; + adminPwd: string; + onCloudflaredUrl?: (url: string) => void; + onCleanup?: () => void; +} + +interface SpawnedResources { + dbContainerId: string; + dbProcess?: any; + backendProcess?: any; + cloudflaredProcess?: any; + tunnelUrl?: string; + worktree?: WorktreeInfo; + eeWorktreePath?: string; + logger?: Logger; +} + +// No default config needed since commitHash is always required + +export class EphemeralBackend { + private config: Config; + private resources: SpawnedResources = { dbContainerId: "" }; + + getDbPort(): number { + return this.config.dbPort; + } + getServerPort(): number { + return this.config.serverPort; + } + getLogFilePath(): string | undefined { + return this.resources.logger?.getLogFilePath(); + } + + constructor(config: Config) { + this.config = config; + } + + async spawn(): Promise<{ + tunnelUrl: string; + }> { + try { + // Initialize logger for this ephemeral backend + this.resources.logger = new Logger(this.config.commitHash, "backend"); + + this.resources.logger.log("🚀 Starting ephemeral backend..."); + this.resources.logger.log(`📊 Database port: ${this.config.dbPort}`); + this.resources.logger.log(`🌐 Server port: ${this.config.serverPort}`); + this.resources.logger.log(`📌 Commit hash: ${this.config.commitHash}`); + + await this.startCloudflared(); + if (!this.resources.tunnelUrl) + throw new Error("Cloudflare tunnel URL not available"); + await this.acquireWorktree(); + await this.setupEECode(); + await this.spawnPostgres(); + await this.waitForPostgres(); + if (!this.config.skipBuild) { + await this.buildBackend(); + } else { + this.resources.logger?.log( + "\n⏭️ Skipping backend build (using existing binary)" + ); + } + await this.startBackend(); + + // Release the worktree back to the pool now that the backend is running + // The binary is already compiled and running, so other spawns can reuse this worktree + if (this.resources.worktree) { + this.resources.logger?.log( + "\n♻️ Releasing worktree back to pool (backend is now running)..." + ); + await this.config.worktreePool.release(this.resources.worktree.id); + // Keep the reference for cleanup but mark it as released + this.resources.worktree = undefined; + } + + this.resources.logger?.log("\n✅ Ephemeral backend is ready!"); + this.resources.logger?.log(`📍 Tunnel URL: ${this.resources.tunnelUrl}`); + this.resources.logger?.log( + `📄 Log file: ${this.resources.logger.getLogFilePath()}` + ); + + return { + tunnelUrl: this.resources.tunnelUrl, + }; + } catch (error) { + this.resources.logger?.error( + `❌ Error spawning ephemeral backend: ${error}` + ); + throw error; + } + } + + private async acquireWorktree(): Promise { + this.resources.logger?.log("\n📂 Acquiring worktree from pool..."); + + // Acquire a worktree from the pool + this.resources.worktree = await this.config.worktreePool.acquire( + this.config.commitHash + ); + + this.resources.logger?.log( + `✓ Worktree acquired: ${this.resources.worktree.path}` + ); + } + + private async setupEECode(): Promise { + this.resources.logger?.log("\n🔐 Setting up Enterprise Edition code..."); + + if (!this.resources.worktree) { + throw new Error("Worktree not acquired"); + } + + const worktreePath = this.resources.worktree.path; + const eeRefPath = `${worktreePath}/backend/ee-repo-ref.txt`; + const eeWorktreePath = this.config.worktreePool.getEEWorktreePath( + this.resources.worktree + ); + this.resources.eeWorktreePath = eeWorktreePath; + + // Read the EE commit hash from ee-repo-ref.txt + this.resources.logger?.log( + ` Reading EE commit reference from ${eeRefPath}` + ); + let eeCommitHash: string; + try { + const { stdout } = await execAsync(`cat ${eeRefPath}`); + eeCommitHash = stdout.trim(); + if (!eeCommitHash) { + throw new Error("ee-repo-ref.txt is empty"); + } + this.resources.logger?.log(` ✓ EE commit hash: ${eeCommitHash}`); + } catch (error) { + throw new Error( + `Failed to read ee-repo-ref.txt: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + // Remove existing EE private folder if it exists (from previous runs) + this.resources.logger?.log(` Cleaning up any existing EE repository...`); + try { + await execAsync(`rm -rf ${eeWorktreePath}`); + } catch (error) { + // Ignore errors if directory doesn't exist + } + + // Clone the windmill-ee-private repo at the specific commit + this.resources.logger?.log( + ` Cloning windmill-ee-private at commit ${eeCommitHash}` + ); + try { + await execAsync( + `git clone git@github.com:windmill-labs/windmill-ee-private.git ${eeWorktreePath}`, + { + env: { + ...process.env, + GIT_SSH_COMMAND: `ssh -i ${process.env.GIT_EE_DEPLOY_KEY_FILE} -o StrictHostKeyChecking=accept-new`, + }, + } + ); + this.resources.logger?.log(` ✓ Repository cloned to ${eeWorktreePath}`); + } catch (error) { + throw new Error( + `Failed to clone windmill-ee-private: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + // Checkout the specific commit + this.resources.logger?.log(` Checking out commit ${eeCommitHash}`); + try { + await execAsync(`git checkout ${eeCommitHash}`, { + cwd: eeWorktreePath, + }); + this.resources.logger?.log(` ✓ Checked out commit ${eeCommitHash}`); + } catch (error) { + throw new Error( + `Failed to checkout EE commit: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + // Run the substitute_ee_code.sh script to copy EE files + this.resources.logger?.log( + ` Running substitute_ee_code.sh to copy EE files` + ); + try { + await execAsync(`./substitute_ee_code.sh --copy -d ${eeWorktreePath}`, { + cwd: `${worktreePath}/backend`, + }); + this.resources.logger?.log(` ✓ EE code substituted successfully`); + } catch (error) { + throw new Error( + `Failed to substitute EE code: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + + this.resources.logger?.log("✓ Enterprise Edition code setup complete"); + } + + private async spawnPostgres(): Promise { + this.resources.logger?.log("\n🐘 Spawning PostgreSQL container..."); + + this.resources.dbProcess = spawn("docker", [ + "run", + "--rm", + "-p", + `${this.config.dbPort}:5432`, + "-e", + "POSTGRES_PASSWORD=changeme", + "-e", + "POSTGRES_DB=windmill", + "postgres:16", + ]); + + // Capture and log postgres stdout + this.resources.dbProcess.stdout.on("data", (data: Buffer) => { + const output = data.toString().trim(); + if (output) { + this.resources.logger?.log(`[postgres] ${output}`); + } + }); + + // Capture and log postgres stderr + this.resources.dbProcess.stderr.on("data", (data: Buffer) => { + const output = data.toString().trim(); + if (output) { + this.resources.logger?.error(`[postgres] ${output}`); + } + }); + + this.resources.dbProcess.on("close", (code: number) => { + this.resources.logger?.log(`PostgreSQL process exited with code ${code}`); + }); + } + + private async waitForPostgres(): Promise { + this.resources.logger?.log("⏳ Waiting for PostgreSQL to be ready..."); + + const maxAttempts = 30; + const delayMs = 1000; + + // Determine the host to connect to + // On Linux, we need to use the host's IP or localhost + // On macOS/Windows, host.docker.internal works + const isLinux = process.platform === "linux"; + const dbHost = isLinux ? "172.17.0.1" : "host.docker.internal"; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + await execAsync( + `docker run --rm postgres:16 pg_isready -h ${dbHost} -p ${this.config.dbPort} -U postgres` + ); + this.resources.logger?.log("✓ PostgreSQL is ready"); + return; + } catch (error) { + if (attempt === maxAttempts) { + throw new Error("PostgreSQL failed to start in time"); + } + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + } + + private async buildBackend(): Promise { + this.resources.logger?.log( + "\n🔨 Building backend (this may take a while)..." + ); + + if (!this.resources.worktree) { + throw new Error("Worktree not acquired"); + } + + // Detect OS to use correct deno_core feature + const isMacOS = process.platform === "darwin"; + + const env = { ...process.env, SQLX_OFFLINE: "true" }; + + const features = [ + "enterprise", + "enterprise_saml", + "stripe", + "embedding", + "parquet", + "prometheus", + "openidconnect", + "cloud", + "jemalloc", + "agent_worker_server", + "tantivy", + "license", + "http_trigger", + "zip", + "oauth2", + "kafka", + "sqs_trigger", + "nats", + "otel", + "dind", + "postgres_trigger", + "mqtt_trigger", + "gcp_trigger", + "websocket", + "smtp", + "all_languages", + "private", + isMacOS ? "deno_core_mac" : "deno_core", + "mcp", + ].join(","); + + return new Promise((resolve, reject) => { + const backendDir = `${this.resources.worktree?.path}/backend`; + const buildProcess = spawn( + "cargo", + ["build", "--features", features, "--release"], + { cwd: backendDir, env } + ); + + buildProcess.stdout.on("data", (data) => { + const output = data.toString().trim(); + if (output) { + this.resources.logger?.log(`[build] ${output}`); + } + }); + + buildProcess.stderr.on("data", (data) => { + const output = data.toString().trim(); + if (output) { + this.resources.logger?.error(`[build] ${output}`); + } + }); + + buildProcess.on("close", (code) => { + if (code === 0) { + this.resources.logger?.log("✓ Backend built successfully"); + resolve(); + } else { + this.resources.logger?.error(`Build failed with code ${code}`); + reject(new Error(`Build failed with code ${code}`)); + } + }); + }); + } + + private async startBackend(): Promise { + this.resources.logger?.log("\n🚀 Starting Windmill backend..."); + + if (!this.resources.worktree) { + throw new Error("Worktree not acquired"); + } + + const env = { + // DO NOT PASS the entire process.env to avoid leaking sensitive information + ...(process.env.LICENSE_KEY && { LICENSE_KEY: process.env.LICENSE_KEY }), + DATABASE_URL: `postgres://postgres:changeme@localhost:${this.config.dbPort}/windmill?sslmode=disable`, + PORT: this.config.serverPort.toString(), + }; + + const releaseDir = `${this.resources.worktree.path}/backend/target/release`; + + // Get sandbox user UID/GID for proper sandboxing + const sandboxUid = await this.getSandboxUid(); + const sandboxGid = await this.getSandboxGid(); + + // Use bwrap for sandboxing the windmill process + this.resources.backendProcess = spawn( + "bwrap", + [ + ...["--ro-bind", "/", "/"], + ...["--bind", "/home/sandbox", "/home/sandbox"], + ...["--bind", "/tmp", "/tmp"], + ...["--dev", "/dev"], + ...["--proc", "/proc"], + "--unshare-user", + ...["--uid", sandboxUid], + ...["--gid", sandboxGid], + `${releaseDir}/windmill`, + ], + { + env, + } + ); + + // Wait for backend to be ready by watching for "Windmill Enterprise Edition" in output + const backendReady = new Promise((resolve, reject) => { + // Capture and log backend stdout, watching for ready signal + this.resources.backendProcess.stdout.on("data", (data: any) => { + const output = data.toString().trim(); + const timeout = setTimeout(() => { + reject(new Error("Backend failed to start in time")); + }, 30000); // 30 seconds timeout + if (output) { + this.resources.logger?.log(`[backend] ${output}`); + if (output.includes("Windmill Enterprise Edition")) { + clearTimeout(timeout); + resolve(); + } + } + }); + }); + + // Capture and log backend stderr + this.resources.backendProcess.stderr.on("data", (data: Buffer) => { + const output = data.toString().trim(); + if (output) { + this.resources.logger?.error(`[backend] ${output}`); + } + }); + + this.resources.backendProcess.on("close", (code: number) => { + this.resources.logger?.log(`Backend process exited with code ${code}`); + }); + + // Wait for backend to be ready + await backendReady; + this.resources.logger?.log("✓ Backend is running"); + + // Wait 2 additional seconds to be sure + await new Promise((resolve) => setTimeout(resolve, 2000)); + + // Setup admin password + await this.setupAdminPassword(); + } + + private async startCloudflared(): Promise { + if (process.env.SKIP_CLOUDFLARED) { + this.config.onCloudflaredUrl?.("SKIP_CLOUDFLARED"); + return; + } + this.resources.logger?.log("\n🌐 Starting Cloudflare tunnel..."); + + return new Promise((resolve, reject) => { + this.resources.cloudflaredProcess = spawn("cloudflared", [ + "tunnel", + "--url", + `http://localhost:${this.config.serverPort}`, + "--config", + "/dev/null", + ]); + + const rl = readline.createInterface({ + input: this.resources.cloudflaredProcess.stdout, + }); + + rl.on("line", (line: string) => { + if (line.trim()) { + this.resources.logger?.log(`[cloudflared] ${line}`); + } + }); + + this.resources.cloudflaredProcess.stderr.on("data", (data: Buffer) => { + const line = data.toString(); + if (line.trim()) { + this.resources.logger?.error(`[cloudflared] ${line.trim()}`); + } + const match = line.match(/https:\/\/([a-z0-9-]+\.trycloudflare\.com)/); + if (match) { + this.resources.tunnelUrl = match[1]; + this.resources.logger?.log( + `✓ Tunnel URL: ${this.resources.tunnelUrl}` + ); + this.config.onCloudflaredUrl?.(this.resources.tunnelUrl); + resolve(); + } + }); + + this.resources.cloudflaredProcess.on("close", (code: number) => { + this.resources.logger?.log( + `Cloudflared process exited with code ${code}` + ); + }); + + // Timeout if we can't find the URL in 30 seconds + setTimeout(() => { + if (!this.resources.tunnelUrl) { + this.resources.logger?.error( + "Failed to extract Cloudflare tunnel URL (timeout)" + ); + reject(new Error("Failed to extract Cloudflare tunnel URL")); + } + }, 30000); + }); + } + + private async getSandboxUid(): Promise { + try { + const { stdout } = await execAsync("id -u sandbox"); + return stdout.trim(); + } catch (error) { + this.resources.logger?.error( + `Failed to get sandbox UID: ${ + error instanceof Error ? error.message : String(error) + }` + ); + throw new Error("Failed to get sandbox user UID"); + } + } + + private async getSandboxGid(): Promise { + try { + const { stdout } = await execAsync("id -g sandbox"); + return stdout.trim(); + } catch (error) { + this.resources.logger?.error( + `Failed to get sandbox GID: ${ + error instanceof Error ? error.message : String(error) + }` + ); + throw new Error("Failed to get sandbox user GID"); + } + } + + private async setupAdminPassword(): Promise { + this.resources.logger?.log("\n🔑 Setting up admin password..."); + + const baseUrl = `http://localhost:${this.config.serverPort}`; + + try { + // Login with default credentials + this.resources.logger?.log(" Logging in with default credentials..."); + const loginResponse = await fetch(`${baseUrl}/api/auth/login`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + email: "admin@windmill.dev", + password: "changeme", + }), + }); + + if (!loginResponse.ok) { + throw new Error( + `Login failed with status ${ + loginResponse.status + }: ${await loginResponse.text()}` + ); + } + + const token = await loginResponse.text(); + this.resources.logger?.log(" ✓ Logged in successfully"); + + // Set new admin password + this.resources.logger?.log(" Setting new admin password..."); + const setPasswordResponse = await fetch( + `${baseUrl}/api/users/setpassword`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ + password: this.config.adminPwd, + }), + } + ); + + if (!setPasswordResponse.ok) { + throw new Error( + `Set password failed with status ${ + setPasswordResponse.status + }: ${await setPasswordResponse.text()}` + ); + } + + this.resources.logger?.log("✓ Admin password configured successfully"); + } catch (error) { + this.resources.logger?.error( + `Failed to setup admin password: ${ + error instanceof Error ? error.message : String(error) + }` + ); + throw error; + } + } + + async cleanup(): Promise { + this.resources.logger?.log("\n🧹 Cleaning up resources..."); + + // Kill backend process + if (this.resources.backendProcess) { + this.resources.logger?.log(" Stopping backend..."); + try { + this.resources.backendProcess.kill("SIGTERM"); + // Give it a moment to gracefully shutdown + await new Promise((resolve) => setTimeout(resolve, 1000)); + // Force kill if still running + this.resources.backendProcess.kill("SIGKILL"); + } catch (error) { + // Process might already be dead + } + } + + // Kill cloudflared process + if (this.resources.cloudflaredProcess) { + this.resources.logger?.log(" Stopping cloudflared..."); + try { + this.resources.cloudflaredProcess.kill("SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + this.resources.cloudflaredProcess.kill("SIGKILL"); + } catch (error) { + // Process might already be dead + } + } + + // Kill PostgreSQL process + if (this.resources.dbProcess) { + this.resources.logger?.log(" Stopping PostgreSQL container..."); + try { + this.resources.dbProcess.kill("SIGTERM"); + await new Promise((resolve) => setTimeout(resolve, 1000)); + this.resources.dbProcess.kill("SIGKILL"); + this.resources.logger?.log(" ✓ PostgreSQL container stopped"); + } catch (error) { + console.error(" Failed to stop PostgreSQL container:", error); + } + } + + // Remove EE private repository clone + if (this.resources.eeWorktreePath) { + this.resources.logger?.log(" Removing EE private repository clone..."); + try { + await execAsync(`rm -rf ${this.resources.eeWorktreePath}`); + this.resources.logger?.log(" ✓ EE private repository clone removed"); + } catch (error) { + console.error(" Failed to remove EE private repository clone:", error); + } + } + + // Release git worktree back to pool (do not delete it) + // Note: worktree might already be released if backend started successfully + if (this.resources.worktree) { + this.resources.logger?.log(" Releasing worktree back to pool..."); + try { + await this.config.worktreePool.release(this.resources.worktree.id); + this.resources.logger?.log(" ✓ Worktree released for reuse"); + } catch (error) { + console.error(" Failed to release worktree:", error); + } + } else { + this.resources.logger?.log(" ✓ Worktree already released"); + } + + this.config.onCleanup?.(); + + this.resources.logger?.log("✅ Cleanup complete"); + } +} diff --git a/ephemeral-backends/tsconfig.json b/ephemeral-backends/tsconfig.json new file mode 100644 index 0000000000..a374fadee8 --- /dev/null +++ b/ephemeral-backends/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "node", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "outDir": "./dist", + "rootDir": "." + }, + "include": ["*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/ephemeral-backends/windmill-ephemeral-manager.service b/ephemeral-backends/windmill-ephemeral-manager.service new file mode 100644 index 0000000000..21ed4540de --- /dev/null +++ b/ephemeral-backends/windmill-ephemeral-manager.service @@ -0,0 +1,34 @@ +[Unit] +Description=Windmill Ephemeral Backend Manager +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=sandbox +Group=sandbox +WorkingDirectory=/home/sandbox/ephemeral-backend/windmill +EnvironmentFile=/etc/windmill-ephemeral-manager/.env + +# Start the manager +ExecStart=bun run ephemeral-backends/manager + +# Restart policy +Restart=on-failure +RestartSec=10s + +# Resource limits +LimitNOFILE=65536 +LimitNPROC=4096 + +# Security settings +NoNewPrivileges=true +PrivateTmp=true + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=windmill-ephemeral-manager + +[Install] +WantedBy=multi-user.target diff --git a/ephemeral-backends/worktree-pool.ts b/ephemeral-backends/worktree-pool.ts new file mode 100644 index 0000000000..f0973dcc5d --- /dev/null +++ b/ephemeral-backends/worktree-pool.ts @@ -0,0 +1,231 @@ +import { promisify } from "util"; +import { exec } from "child_process"; +import path from "path"; + +const execAsync = promisify(exec); + +export interface WorktreeInfo { + id: number; + path: string; + inUse: boolean; + currentCommit?: string; +} + +export class WorktreePool { + private worktrees: Map = new Map(); + private baseWorktreePath: string; + private nextId: number = 0; + + constructor(baseWorktreePath: string = "../windmill-ephemeral-backends") { + this.baseWorktreePath = path.resolve(baseWorktreePath); + } + + /** + * Initialize the pool by discovering existing worktrees + */ + async initialize(): Promise { + console.log("🔍 Initializing worktree pool..."); + + // Ensure base directory exists + try { + await execAsync(`mkdir -p ${this.baseWorktreePath}`); + } catch (error) { + // Directory might already exist + } + + // Discover existing worktrees + await this.discoverExistingWorktrees(); + + console.log( + `✓ Worktree pool initialized with ${this.worktrees.size} existing worktree(s)` + ); + } + + /** + * Discover existing worktrees from git worktree list + */ + private async discoverExistingWorktrees(): Promise { + try { + const { stdout } = await execAsync("git worktree list --porcelain"); + const lines = stdout.split("\n"); + + let currentWorktreePath: string | null = null; + let isMainWorktree = false; + + for (const line of lines) { + if (line.startsWith("worktree ")) { + currentWorktreePath = line.substring("worktree ".length); + isMainWorktree = false; + } else if (line.startsWith("branch ")) { + // Main worktree has a branch entry, so this is NOT a detached HEAD worktree + // We want to skip the main worktree + isMainWorktree = true; + } else if (line === "" && currentWorktreePath) { + // End of worktree entry + // Check if this worktree is in our base path and not the main worktree + if ( + currentWorktreePath.startsWith(this.baseWorktreePath) && + !isMainWorktree + ) { + // Extract the ID from the path (e.g., .../worktree-0 -> 0) + const match = currentWorktreePath.match(/worktree-(\d+)$/); + if (match) { + const id = parseInt(match[1], 10); + this.worktrees.set(id, { + id, + path: currentWorktreePath, + inUse: false, + currentCommit: undefined, + }); + if (id >= this.nextId) { + this.nextId = id + 1; + } + console.log(` Found existing worktree: ${currentWorktreePath}`); + } + } + currentWorktreePath = null; + isMainWorktree = false; + } + } + } catch (error) { + // If git worktree list fails, we'll start with an empty pool + console.log(" No existing worktrees found"); + } + } + + /** + * Acquire a worktree for the given commit hash + * Returns a free worktree or creates a new one if needed + */ + async acquire(commitHash: string): Promise { + console.log(`\n📂 Acquiring worktree for commit ${commitHash}...`); + + // Try to find a free worktree + for (const worktree of this.worktrees.values()) { + if (!worktree.inUse) { + console.log(` Using existing worktree: ${worktree.path}`); + await this.prepareWorktree(worktree, commitHash); + worktree.inUse = true; + worktree.currentCommit = commitHash; + return worktree; + } + } + + // No free worktrees, create a new one + console.log(" No free worktrees available, creating new one..."); + const newWorktree = await this.createWorktree(commitHash); + newWorktree.inUse = true; + newWorktree.currentCommit = commitHash; + this.worktrees.set(newWorktree.id, newWorktree); + return newWorktree; + } + + /** + * Prepare a worktree for use by discarding changes and checking out the commit + */ + private async prepareWorktree( + worktree: WorktreeInfo, + commitHash: string + ): Promise { + console.log(` Preparing worktree at ${worktree.path}...`); + + try { + // Discard all local changes + await execAsync("git reset --hard", { cwd: worktree.path }); + await execAsync("git clean -fd", { cwd: worktree.path }); + + // Fetch to ensure we have the latest commits + await execAsync("git fetch origin", { cwd: worktree.path }); + + // Checkout the target commit + await execAsync(`git checkout ${commitHash}`, { cwd: worktree.path }); + + console.log(` ✓ Worktree prepared and checked out to ${commitHash}`); + } catch (error) { + throw new Error( + `Failed to prepare worktree at ${worktree.path}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + /** + * Create a new worktree for the given commit hash + */ + private async createWorktree(commitHash: string): Promise { + const id = this.nextId++; + const worktreePath = path.join(this.baseWorktreePath, `worktree-${id}`); + + console.log(` Creating new worktree at ${worktreePath}...`); + + try { + // First, fetch to ensure we have the commit + await execAsync(`git fetch origin ${commitHash}`); + + // Create the worktree at the specific commit + // Note: This command must be run from the main repository directory + await execAsync(`git worktree add ${worktreePath} ${commitHash}`); + console.log(` ✓ Worktree created at ${worktreePath}`); + + return { + id, + path: worktreePath, + inUse: false, + currentCommit: undefined, + }; + } catch (error) { + throw new Error( + `Failed to create worktree at ${worktreePath}: ${ + error instanceof Error ? error.message : String(error) + }` + ); + } + } + + /** + * Release a worktree back to the pool + * The worktree is marked as available but NOT deleted + */ + async release(worktreeId: number): Promise { + const worktree = this.worktrees.get(worktreeId); + if (!worktree) { + console.warn(`⚠️ Worktree ${worktreeId} not found in pool`); + return; + } + + console.log(`\n📂 Releasing worktree ${worktreeId} back to pool...`); + + // Mark as available + worktree.inUse = false; + worktree.currentCommit = undefined; + + console.log(`✓ Worktree ${worktreeId} is now available for reuse`); + } + + /** + * Get the EE worktree path for a given worktree + */ + getEEWorktreePath(worktree: WorktreeInfo): string { + return `${worktree.path}_private`; + } + + /** + * Get pool statistics + */ + getStats(): { + total: number; + inUse: number; + available: number; + } { + const total = this.worktrees.size; + const inUse = Array.from(this.worktrees.values()).filter( + (w) => w.inUse + ).length; + return { + total, + inUse, + available: total - inUse, + }; + } +} diff --git a/flake.nix b/flake.nix index 75bf82de8c..211c281e86 100644 --- a/flake.nix +++ b/flake.nix @@ -55,8 +55,11 @@ nodejs postgresql pkg-config - clang + llvmPackages_18.clang + mold cmake + cyrus_sasl + krb5 ]; coursier = pkgs.fetchFromGitHub { owner = "coursier"; @@ -66,7 +69,7 @@ }; PKG_CONFIG_PATH = pkgs.lib.makeSearchPath "lib/pkgconfig" - (with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev ]); + (with pkgs; [ openssl.dev libxml2.dev xmlsec.dev libxslt.dev cyrus_sasl.dev krb5.dev ]); RUSTY_V8_ARCHIVE = let # NOTE: needs to be same as in Cargo.toml version = "130.0.7"; @@ -303,6 +306,14 @@ # RUST_LOG = "debug"; # RUST_LOG = "kube=debug"; + # Override cargo linker to use clang 18 (stdenv brings clang 21 which causes SIGSEGV with mold) + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.llvmPackages_18.clang}/bin/clang"; + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER = "${pkgs.llvmPackages_18.clang}/bin/clang"; + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${pkgs.lib.makeLibraryPath [ pkgs.openssl pkgs.libffi pkgs.cyrus_sasl pkgs.krb5 pkgs.libxml2 pkgs.xmlsec pkgs.libxslt stdenv.cc.cc.lib ]}"; + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS = "-C link-arg=-fuse-ld=mold -C link-arg=-Wl,-rpath,${pkgs.lib.makeLibraryPath [ pkgs.openssl pkgs.libffi pkgs.cyrus_sasl pkgs.krb5 pkgs.libxml2 pkgs.xmlsec pkgs.libxslt stdenv.cc.cc.lib ]}"; + # rpath for build scripts and proc macros (host compilation) + CARGO_HOST_RUSTFLAGS = "-C link-arg=-Wl,-rpath,${pkgs.lib.makeLibraryPath [ pkgs.openssl pkgs.libffi pkgs.cyrus_sasl pkgs.krb5 pkgs.libxml2 pkgs.xmlsec pkgs.libxslt stdenv.cc.cc.lib ]}"; + # See this issue: https://github.com/NixOS/nixpkgs/issues/370494 # Allows to build jemalloc on nixos CFLAGS = "-Wno-error=int-conversion"; @@ -310,6 +321,7 @@ # Need to tell bindgen where to find libclang LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + # LD_LIBRARY_PATH set in shellHook with a wrapper to avoid leaking into git/ssh # LD_LIBRARY_PATH = "${pkgs.gcc.lib}/lib"; # Set C flags for Rust's bindgen program. Unlike ordinary C diff --git a/frontend/minifiedOpenflowJson.sh b/frontend/minifiedOpenflowJson.sh index ea58442c1e..8ae19cdc97 100755 --- a/frontend/minifiedOpenflowJson.sh +++ b/frontend/minifiedOpenflowJson.sh @@ -94,7 +94,18 @@ zodCode = zodCode.replace(/z\\.literal\\(\"__CIRCULAR_REF_FLOWMODULE__\"\\)/g, ' // FlowModuleValue is used for AI agent tool values zodCode = zodCode.replace(/z\\.literal\\(\"__CIRCULAR_REF_FLOWMODULEVALUE__\"\\)/g, 'z.lazy(() => flowModuleValueSchema)'); -zodCode = zodCode.replace('from \"zod\"', 'from \"zod/v3\"'); +// Fix z.record() calls for Zod 4 compatibility +// In Zod 4, z.record(valueSchema) means keySchema, not valueSchema +// We need z.record(z.string(), valueSchema) for string keys with typed values +zodCode = zodCode.replace(/z\\.record\\(z\\./g, 'z.record(z.string(), z.'); +valueSchemaExport = valueSchemaExport.replace(/z\\.record\\(z\\./g, 'z.record(z.string(), z.'); + +// Fix refinement API for Zod 4 compatibility +// Zod 4 uses 'errors' instead of 'unionErrors' and doesn't have ctx.path +zodCode = zodCode.replace(/unionErrors: errors/g, 'errors: errors.map(e => e.issues)'); +zodCode = zodCode.replace(/path: ctx\\.path,\\n/g, ''); +valueSchemaExport = valueSchemaExport.replace(/unionErrors: errors/g, 'errors: errors.map(e => e.issues)'); +valueSchemaExport = valueSchemaExport.replace(/path: ctx\\.path,\\n/g, ''); // Insert the valueSchemaExport before flowModuleSchema definition zodCode = zodCode.replace( diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b39bc81ea6..a788d21b23 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.625.0", + "version": "1.629.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.625.0", + "version": "1.629.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -79,14 +79,14 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.623.1", - "windmill-parser-wasm-regex": "1.623.1", + "windmill-parser-wasm-py": "^1.628.3", + "windmill-parser-wasm-regex": "1.625.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.623.1", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.2", + "windmill-utils-internal": "^1.3.4", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -2955,115 +2955,6 @@ "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", "license": "ISC" }, - "node_modules/@vitest/browser": { - "version": "4.0.15", - "resolved": "https://registry.npmjs.org/@vitest/browser/-/browser-4.0.15.tgz", - "integrity": "sha512-zedtczX688KehaIaAv7m25CeDLb0gBtAOa2Oi1G1cqvSO5aLSVfH6lpZMJLW8BKYuWMxLQc9/5GYoM+jgvGIrw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@vitest/mocker": "4.0.15", - "@vitest/utils": "4.0.15", - "magic-string": "^0.30.21", - "pixelmatch": "7.1.0", - "pngjs": "^7.0.0", - "sirv": "^3.0.2", - "tinyrainbow": "^3.0.3", - "ws": "^8.18.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "vitest": "4.0.15" - } - }, - "node_modules/@vitest/browser-playwright": { - "version": "4.0.15", - "resolved": "https://registry.npmjs.org/@vitest/browser-playwright/-/browser-playwright-4.0.15.tgz", - "integrity": "sha512-94yVpDbb+ykiT7mK6ToonGnq2GIHEQGBTZTAzGxBGQXcVNCh54YKC2/WkfaDzxy0m6Kgw05kq3FYHKHu+wRdIA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@vitest/browser": "4.0.15", - "@vitest/mocker": "4.0.15", - "tinyrainbow": "^3.0.3" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "playwright": "*", - "vitest": "4.0.15" - }, - "peerDependenciesMeta": { - "playwright": { - "optional": false - } - } - }, - "node_modules/@vitest/browser-playwright/node_modules/@vitest/mocker": { - "version": "4.0.15", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.15.tgz", - "integrity": "sha512-CZ28GLfOEIFkvCFngN8Sfx5h+Se0zN+h4B7yOsPVCcgtiO7t5jt9xQh2E1UkFep+eb9fjyMfuC5gBypwb07fvQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@vitest/spy": "4.0.15", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0-0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, - "node_modules/@vitest/browser/node_modules/@vitest/mocker": { - "version": "4.0.15", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.0.15.tgz", - "integrity": "sha512-CZ28GLfOEIFkvCFngN8Sfx5h+Se0zN+h4B7yOsPVCcgtiO7t5jt9xQh2E1UkFep+eb9fjyMfuC5gBypwb07fvQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@vitest/spy": "4.0.15", - "estree-walker": "^3.0.3", - "magic-string": "^0.30.21" - }, - "funding": { - "url": "https://opencollective.com/vitest" - }, - "peerDependencies": { - "msw": "^2.4.9", - "vite": "^6.0.0 || ^7.0.0-0" - }, - "peerDependenciesMeta": { - "msw": { - "optional": true - }, - "vite": { - "optional": true - } - } - }, "node_modules/@vitest/expect": { "version": "4.0.15", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.15.tgz", @@ -9950,21 +9841,6 @@ "node": ">= 6" } }, - "node_modules/pixelmatch": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-7.1.0.tgz", - "integrity": "sha512-1wrVzJ2STrpmONHKBy228LM1b84msXDUoAzVEl0R8Mz4Ce6EPr+IVtxm8+yvrqLYMHswREkjYFaMxnyGnaY3Ng==", - "dev": true, - "license": "ISC", - "optional": true, - "peer": true, - "dependencies": { - "pngjs": "^7.0.0" - }, - "bin": { - "pixelmatch": "bin/pixelmatch" - } - }, "node_modules/pkg-types": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", @@ -10040,18 +9916,6 @@ "node": ">=4" } }, - "node_modules/pngjs": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", - "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=14.19.0" - } - }, "node_modules/postcss": { "version": "8.5.6", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", @@ -14603,14 +14467,14 @@ "integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.623.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.623.1.tgz", - "integrity": "sha512-lFBlZg6hvhHzsU5oPJq0478UyMTZ9UKVd8Hc8ggmxPIHZaJBeJ+56NR75hmGwg0VJcRff8ed+zEm1PgmjVhD+w==" + "version": "1.628.3", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.628.3.tgz", + "integrity": "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.623.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.623.1.tgz", - "integrity": "sha512-rW3pl4ysIXmVAmwxSTKhR2sUfScXKfTqxrUebDg0ZNcUzY97S8mNRdQGZfaQ1NWhWPRQNXfL/z/yaUOQh5ConQ==" + "version": "1.625.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.625.0.tgz", + "integrity": "sha512-xXzr0O2+U4IaQW8Y+SlbU2wEN5lyZhb+ZjO0LouWVlbQkj/RVeQkQ1M7tZr5LlmvGwXbFmBjct/3IjVR8c3Qwg==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -14638,9 +14502,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.2.tgz", - "integrity": "sha512-sLE7sfmN3vBwPReqKorBrusnn83Qnm8vVFqh/7+kz8mf98ojMVDMgBginjlh9KBtVfWu9NhyjGS+WXRcnWalzA==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.4.tgz", + "integrity": "sha512-XVypDKIZ6P4fwIjZwvuvq1m+j0rtAA7BDp1rI2F7hQ+VBKZUHsLskP+jgstXs+kN1LqGGsQJj4ecMYDImpIZ6A==", "license": "Apache 2.0" }, "node_modules/word-wrap": { @@ -14785,29 +14649,6 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/ws": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", - "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/xml-utils": { "version": "1.10.2", "resolved": "https://registry.npmjs.org/xml-utils/-/xml-utils-1.10.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index c7d8e1ac36..d839470ec8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.625.0", + "version": "1.629.1", "scripts": { "dev": "vite dev", "build": "vite build", @@ -149,14 +149,14 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.623.1", - "windmill-parser-wasm-regex": "1.623.1", + "windmill-parser-wasm-py": "^1.628.3", + "windmill-parser-wasm-regex": "1.625.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.623.1", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.2", + "windmill-utils-internal": "^1.3.4", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/components/AIProviderPicker.svelte b/frontend/src/lib/components/AIProviderPicker.svelte index 821302887d..92742c7fee 100644 --- a/frontend/src/lib/components/AIProviderPicker.svelte +++ b/frontend/src/lib/components/AIProviderPicker.svelte @@ -2,7 +2,7 @@ import { type Snippet } from 'svelte' import Select from './select/Select.svelte' import { fetchAvailableModels, AI_PROVIDERS } from './copilot/lib' - import type { AIProvider } from '$lib/gen' + import type { AIProvider, ProviderConfig } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { get } from 'svelte/store' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' @@ -10,15 +10,10 @@ import ResourcePicker from './ResourcePicker.svelte' import ToggleButtonMore from './common/toggleButton-v2/ToggleButtonMore.svelte' import Toggle from './Toggle.svelte' - - interface ProviderValue { - kind?: AIProvider - resource?: string - model?: string - } + import { saveConfig, removeConfig, isSameAsStoredConfig } from './aiProviderStorage' interface Props { - value: ProviderValue | undefined + value: ProviderConfig | undefined disabled?: boolean actions?: Snippet } @@ -28,31 +23,19 @@ let loading = $state(false) let availableModels = $state([]) let filterText = $state('') - let useAsDefault = $state(false) let modelsCache = new Map() - const STORAGE_KEY = 'windmill_ai_provider_config' - - // Initialize value if undefined if (!value) { - const storedConfig = loadStoredConfig() - if (storedConfig) { - value = storedConfig - useAsDefault = true - } else { - const providers = Object.keys(AI_PROVIDERS) - value = { - kind: providers.length > 0 ? (providers[0] as AIProvider) : undefined, - resource: undefined, - model: undefined - } - useAsDefault = false + value = { + kind: 'openai', + resource: '', + model: '' } - } else { - useAsDefault = isSameAsStoredConfig(value) } + let useAsDefault = $derived(isSameAsStoredConfig(value)) + // Reactive items for the Select component let items = $derived.by(() => { const r = availableModels.map((model) => ({ @@ -74,61 +57,6 @@ label: details.label })) - // Check if the current config is the same as the stored config - function isSameAsStoredConfig(config: ProviderValue): boolean { - const storedConfig = loadStoredConfig() - return ( - storedConfig !== undefined && - storedConfig?.kind === config.kind && - storedConfig?.resource === config.resource && - storedConfig?.model === config.model - ) - } - - // Load stored configuration from localStorage - function loadStoredConfig(): ProviderValue | undefined { - if (typeof localStorage === 'undefined') { - return undefined - } - try { - const stored = localStorage.getItem(STORAGE_KEY) - if (stored) { - const parsed = JSON.parse(stored) - // Validate that the stored provider is still available - if (parsed.kind && AI_PROVIDERS[parsed.kind]) { - return parsed - } - } - } catch (e) { - console.error('Failed to load AI provider config from localStorage:', e) - } - return undefined - } - - // Save configuration to localStorage - function saveConfig(config: ProviderValue) { - if (typeof localStorage === 'undefined') { - return - } - try { - localStorage.setItem(STORAGE_KEY, JSON.stringify(config)) - } catch (e) { - console.error('Failed to save AI provider config to localStorage:', e) - } - } - - // Remove configuration from localStorage - function removeConfig() { - if (typeof localStorage === 'undefined') { - return - } - try { - localStorage.removeItem(STORAGE_KEY) - } catch (e) { - console.error('Failed to remove AI provider config from localStorage:', e) - } - } - async function loadModels(signal?: AbortSignal) { const provider = value?.kind const resourceValue = value?.resource @@ -171,8 +99,8 @@ function onProviderChange(selectedProvider: AIProvider) { if (value) { value.kind = selectedProvider - value.resource = undefined - value.model = undefined + value.resource = '' + value.model = '' } } @@ -261,7 +189,7 @@ () => resourceValueToPath(value?.resource), (v) => { if (value) { - value.resource = pathToResourceValue(v) + value.resource = pathToResourceValue(v) ?? '' } } } diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 2537e249fe..f4f1936e43 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -258,7 +258,11 @@ nvalue = structuredClone($state.snapshot(defaultValue)) if (defaultValue === undefined || defaultValue === null) { if (inputCat === 'string') { - nvalue = nullable ? null : format === 'uuid' && extra?.['x-auto-generate'] ? randomUUID() : '' + nvalue = nullable + ? null + : format === 'uuid' && extra?.['x-auto-generate'] + ? randomUUID() + : '' } else if (inputCat == 'enum' && required) { let firstV = enum_?.[0] if (typeof firstV === 'string') { @@ -1177,7 +1181,6 @@ titleClass="text-2xs" /> {/if} - {:else if disabled} {:else} diff --git a/frontend/src/lib/components/BedrockCredentialsCheck.svelte b/frontend/src/lib/components/BedrockCredentialsCheck.svelte index 989367ef23..877ca9ff72 100644 --- a/frontend/src/lib/components/BedrockCredentialsCheck.svelte +++ b/frontend/src/lib/components/BedrockCredentialsCheck.svelte @@ -1,5 +1,5 @@

Workspace to link to

- {#if deployableWorkspaces?.length == 0} {/if} @@ -113,8 +163,12 @@ {/each}
-

Deployable items

-
+

Deployable items

+
+ You can filter which items can be deployed to the production workspace. By default everything is + deployable. +
+
{#if Array.isArray(deployUiSettings?.include_path)}

- {#each deployUiSettings.include_path ?? [] as regexpPath, idx} -
- - + {#each deployUiSettings.include_path ?? [] as _, idx} +
+
+ + +
+ {#if pathValidationErrors[idx]} +
{pathValidationErrors[idx]}
+ {/if}
{/each} {/if} @@ -197,14 +267,18 @@
-{#if $enterpriseLicense} -
- -
+{#if hasValidationErrors} + + Please fix the validation errors in the path filters before saving. + +{/if} +{#if $enterpriseLicense} + {/if} diff --git a/frontend/src/lib/components/DeployWorkspace.svelte b/frontend/src/lib/components/DeployWorkspace.svelte index c1d1871604..60165f8c54 100644 --- a/frontend/src/lib/components/DeployWorkspace.svelte +++ b/frontend/src/lib/components/DeployWorkspace.svelte @@ -6,7 +6,6 @@ AppService, FlowService, FolderService, - RawAppService, ResourceService, ScheduleService, ScriptService, @@ -232,11 +231,6 @@ workspace: workspaceToDeployTo!, path: path }) - } else if (kind == 'raw_app') { - exists = await RawAppService.existsRawApp({ - workspace: workspaceToDeployTo!, - path: path - }) } else if (kind == 'variable') { exists = await VariableService.existsVariable({ workspace: workspaceToDeployTo!, @@ -498,21 +492,6 @@ } }) } - } else if (kind == 'raw_app') { - throw new Error('Raw app deploy not implemented yet') - // const app = await RawAppService.getRawAppData({ - // workspace: $workspaceStore!, - // path: path - // }) - // if (alreadyExists) { - // } - // await RawAppService.updateRawApp({ - // workspace: $workspaceStore!, - // path: path, - // requestBody: { - // path: path - // } - // }) } else if (kind == 'folder') { await FolderService.createFolder({ workspace: workspaceToDeployTo!, @@ -616,21 +595,6 @@ path: path }) return resource.schema - } else if (kind == 'raw_app') { - throw new Error('Raw app deploy not implemented yet') - // const app = await RawAppService.getRawAppData({ - // workspace: workspace, - // path: path - // }) - // if (alreadyExists) { - // } - // await RawAppService.updateRawApp({ - // workspace: workspace, - // path: path, - // requestBody: { - // path: path - // } - // }) } else if (kind == 'folder') { const folder = await FolderService.getFolder({ workspace: workspace, diff --git a/frontend/src/lib/components/EEOnly.svelte b/frontend/src/lib/components/EEOnly.svelte index 267795f9b2..c92377031c 100644 --- a/frontend/src/lib/components/EEOnly.svelte +++ b/frontend/src/lib/components/EEOnly.svelte @@ -1,27 +1,16 @@ - + EE only {#snippet text()} {#if children} {@render children()} diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index 8ed2a7a192..a3ddc1db6a 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -33,7 +33,7 @@ import { base } from '$lib/base' import { enterpriseLicense, workspaceStore } from '$lib/stores' import MsTeamsIcon from '$lib/components/icons/MSTeamsIcon.svelte' - import { emptySchema, emptyString, sendUserToast, tryEvery } from '$lib/utils' + import { classNames, emptySchema, emptyString, sendUserToast, tryEvery } from '$lib/utils' import MultiSelect from '$lib/components/select/MultiSelect.svelte' import { FlowService, @@ -80,6 +80,7 @@ customScriptTemplate: string customHandlerKind?: 'flow' | 'script' customTabTooltip?: import('svelte').Snippet + noMargin?: boolean } let { @@ -92,7 +93,8 @@ handlerExtraArgs = $bindable(), customScriptTemplate, customHandlerKind = $bindable('script'), - customTabTooltip + customTabTooltip, + noMargin = false }: Props = $props() let customHandlerSchema: Schema | undefined = $state() @@ -297,21 +299,26 @@ teams: undefined as string | undefined, email: undefined as string[] | undefined }) + let handlerPathCache: Partial> = $state({}) $effect(() => { if (lastHandlerSelected !== handlerSelected && lastHandlerSelected !== undefined) { if (lastHandlerSelected != 'custom') { const key = lastHandlerSelected === 'email' ? EMAIL_RECIPIENTS_KEY : CHANNEL_KEY handlerCache[lastHandlerSelected] = handlerExtraArgs[key] } + handlerPathCache[lastHandlerSelected] = handlerPath if (handlerSelected === 'custom') { - handlerExtraArgs[CHANNEL_KEY] = '' - handlerExtraArgs[EMAIL_RECIPIENTS_KEY] = [] - handlerPath = undefined + delete handlerExtraArgs[CHANNEL_KEY] + delete handlerExtraArgs[EMAIL_RECIPIENTS_KEY] + handlerPath = handlerPathCache['custom'] } else if (handlerSelected === 'email') { handlerExtraArgs[EMAIL_RECIPIENTS_KEY] = handlerCache[handlerSelected] ?? [] + delete handlerExtraArgs[CHANNEL_KEY] } else { handlerExtraArgs[CHANNEL_KEY] = handlerCache[handlerSelected] ?? '' + delete handlerExtraArgs[EMAIL_RECIPIENTS_KEY] + handlerPath = handlerPathCache[handlerSelected] } } @@ -343,7 +350,7 @@ }) -
+
{#snippet children({ item })} @@ -361,56 +368,60 @@
{#if handlerSelected === 'custom'} -
- +
+
+ - {#if !handlerPath} - + {#if !handlerPath} + + {/if} +
+ {#if showScriptHelpText} +
+ Example of error handler scripts can be found on + Windmill Hub +
{/if}
- {#if showScriptHelpText} -
- Example of error handler scripts can be found on - Windmill Hub -
- {/if} {#if handlerPath} -

Extra arguments

- {#await import('$lib/components/SchemaForm.svelte')} - - {:then Module} - - {/await} - {#if customHandlerSchema && customHandlerSchema.properties && Object.keys(customHandlerSchema.properties).length === 0} -
This error handler takes no extra arguments
- {/if} +
+

Extra arguments

+ {#await import('$lib/components/SchemaForm.svelte')} + + {:then Module} + + {/await} + {#if customHandlerSchema && customHandlerSchema.properties && Object.keys(customHandlerSchema.properties).length === 0} +
This error handler takes no extra arguments
+ {/if} +
{/if} {:else if handlerSelected === 'slack'} diff --git a/frontend/src/lib/components/ExploreAssetButton.svelte b/frontend/src/lib/components/ExploreAssetButton.svelte index d2f886fa6d..ba623b4e4e 100644 --- a/frontend/src/lib/components/ExploreAssetButton.svelte +++ b/frontend/src/lib/components/ExploreAssetButton.svelte @@ -58,8 +58,7 @@ {btnClasses} on:click={async () => { if (asset.kind === 'resource' && isDbType(_resourceMetadata?.resource_type)) { - let resourcePath = asset.path.split('/').slice(0, 3).join('/') - let specificTable = asset.path.split('/')[3] as string | undefined + let [resourcePath, specificTable] = asset.path.split('?table=') dbManagerDrawer?.openDrawer({ type: 'database', resourceType: _resourceMetadata.resource_type, diff --git a/frontend/src/lib/components/FlowRestartButton.svelte b/frontend/src/lib/components/FlowRestartButton.svelte index 962138798e..8af77f2e6b 100644 --- a/frontend/src/lib/components/FlowRestartButton.svelte +++ b/frontend/src/lib/components/FlowRestartButton.svelte @@ -12,6 +12,8 @@ selectedJobStepType: 'single' | 'forloop' | 'branchall' restartBranchNames?: [number, string][] flowPath?: string + /** The flow version ID used in this run (from job.script_hash) */ + flowVersionId?: number disabled?: boolean enterpriseOnly?: boolean variant?: 'default' | 'accent' @@ -28,6 +30,7 @@ selectedJobStepType, restartBranchNames = [], flowPath = undefined, + flowVersionId = undefined, disabled = false, enterpriseOnly = false, variant = 'default', @@ -36,11 +39,15 @@ onRestartComplete }: Props = $props() + // Sentinel value meaning "use the same version as the original run" (backend receives undefined) + const RUN_VERSION_SENTINEL = -1 + let branchOrIterationN = $state(0) - let selectedVersionMode: 'run' | 'custom' = $state('run') - let customFlowVersion: number | undefined = $state(undefined) + let selectedFlowVersion: number = $state(RUN_VERSION_SENTINEL) let flowVersions: Array = $state([]) let loadingVersions: boolean = $state(false) + let versionsLoaded: boolean = $state(false) + let runVersionInList: boolean = $state(false) async function restartFlow(stepId: string, branchOrIterationN: number, flowVersion?: number) { let run = await JobService.restartFlowAtStep({ @@ -56,16 +63,20 @@ } async function loadFlowVersions() { - if (!flowPath || loadingVersions) return + if (!flowPath || loadingVersions || versionsLoaded) return loadingVersions = true try { flowVersions = await FlowService.getFlowHistory({ workspace: $workspaceStore!, path: flowPath }) - if (flowVersions.length > 0 && customFlowVersion === undefined) { - customFlowVersion = flowVersions[0].id + if (flowVersions.length > 0) { + const match = flowVersionId ? flowVersions.find((v) => v.id === flowVersionId) : undefined + runVersionInList = match !== undefined + selectedFlowVersion = + match?.id ?? (flowVersionId ? RUN_VERSION_SENTINEL : flowVersions[0].id) } + versionsLoaded = true } catch (e) { sendUserToast('Failed to load flow versions', true) } finally { @@ -74,12 +85,12 @@ } function getFlowVersionForRestart(): number | undefined { - if (selectedVersionMode === 'run') { - return undefined // use run version - } else if (selectedVersionMode === 'custom') { - return customFlowVersion - } - return undefined + return selectedFlowVersion === RUN_VERSION_SENTINEL ? undefined : selectedFlowVersion + } + + function formatVersionLabel(version: FlowVersion): string { + const name = emptyString(version.deployment_msg) ? `v${version.id}` : version.deployment_msg! + return `${name} - ${new Date(version.created_at).toLocaleString()}` } function handleRestart() { @@ -95,37 +106,28 @@ {#snippet flowVersionSelector()}
` - Specific table in public schema +- `/:
` - Table in specific schema + +### Querying in TypeScript (Bun/Deno) + +```typescript +import * as wmill from 'windmill-client'; + +export async function main(user_id: string) { + const sql = wmill.datatable(); // Or: wmill.datatable('other_datatable') + + // Parameterized queries (safe from SQL injection) + const user = await sql`SELECT * FROM users WHERE id = ${user_id}`.fetchOne(); + const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch(); + + // Insert/Update + await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`; + await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`; + + return user; +} +``` + +### Querying in Python + +```python +import wmill + +def main(user_id: str): + db = wmill.datatable() # Or: wmill.datatable('other_datatable') + + # Use $1, $2, etc. for parameters + user = db.query('SELECT * FROM users WHERE id = $1', user_id).fetch_one() + users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() + + # Insert/Update + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + + return user +``` + +## SQL Migrations (sql_to_apply/) + +The `sql_to_apply/` folder is for creating/modifying database tables during development. + +### Workflow + +1. Create `.sql` files in `sql_to_apply/` +2. Run `wmill app dev` - the dev server watches this folder +3. When SQL files change, a modal appears in the browser to confirm execution +4. After creating tables, **add them to `data.tables`** in `raw_app.yaml` + +### Example Migration + +**sql_to_apply/001_create_users.sql:** +```sql +CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + name TEXT, + created_at TIMESTAMP DEFAULT NOW() +); +``` + +After applying, add to `raw_app.yaml`: +```yaml +data: + tables: + - main/users +``` + +### Migration Best Practices + +- **Use idempotent SQL**: `CREATE TABLE IF NOT EXISTS`, etc. +- **Number files**: `001_`, `002_` for ordering +- **Always whitelist tables** after creation +- This folder is NOT synced - it's for local development only + +## CLI Commands + +| Command | Description | +|---------|-------------| +| `wmill app new` | Create a new raw app interactively | +| `wmill app dev` | Start dev server with live reload | +| `wmill app generate-agents` | Refresh AGENTS.md and DATATABLES.md | +| `wmill app generate-locks` | Generate lock files for backend runnables | +| `wmill sync push` | Deploy app to Windmill | +| `wmill sync pull` | Pull latest from Windmill | + +## Best Practices + +1. **Check DATATABLES.md** for existing tables before creating new ones +2. **Use parameterized queries** - never concatenate user input into SQL +3. **Keep runnables focused** - one function per file +4. **Use descriptive IDs** - `get_user.ts` not `a.ts` +5. **Always whitelist tables** - add to `data.tables` before querying +6. **Generate locks** - run `wmill app generate-locks` after adding/modifying backend runnables diff --git a/system_prompts/auto-generated/skills/resources/SKILL.md b/system_prompts/auto-generated/skills/resources/SKILL.md new file mode 100644 index 0000000000..649cb39cbb --- /dev/null +++ b/system_prompts/auto-generated/skills/resources/SKILL.md @@ -0,0 +1,247 @@ +--- +name: resources +description: MUST use when managing resources. +--- + +# Windmill Resources + +Resources store credentials and configuration for external services. + +## File Format + +Resource files use the pattern: `{path}.resource.json` + +Example: `f/databases/postgres_prod.resource.json` + +## Resource Structure + +```json +{ + "value": { + "host": "db.example.com", + "port": 5432, + "user": "admin", + "password": "$var:g/all/db_password", + "dbname": "production" + }, + "description": "Production PostgreSQL database", + "resource_type": "postgresql" +} +``` + +## Required Fields + +- `value` - Object containing the resource configuration +- `resource_type` - Name of the resource type (e.g., "postgresql", "slack") + +## Variable References + +Reference variables in resource values: + +```json +{ + "value": { + "api_key": "$var:g/all/api_key", + "secret": "$var:u/admin/secret" + } +} +``` + +**Reference formats:** +- `$var:g/all/name` - Global variable +- `$var:u/username/name` - User variable +- `$var:f/folder/name` - Folder variable + +## Resource References + +Reference other resources: + +```json +{ + "value": { + "database": "$res:f/databases/postgres" + } +} +``` + +## Common Resource Types + +### PostgreSQL +```json +{ + "resource_type": "postgresql", + "value": { + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "$var:g/all/pg_password", + "dbname": "windmill", + "sslmode": "prefer" + } +} +``` + +### MySQL +```json +{ + "resource_type": "mysql", + "value": { + "host": "localhost", + "port": 3306, + "user": "root", + "password": "$var:g/all/mysql_password", + "database": "myapp" + } +} +``` + +### Slack +```json +{ + "resource_type": "slack", + "value": { + "token": "$var:g/all/slack_token" + } +} +``` + +### AWS S3 +```json +{ + "resource_type": "s3", + "value": { + "bucket": "my-bucket", + "region": "us-east-1", + "accessKeyId": "$var:g/all/aws_access_key", + "secretAccessKey": "$var:g/all/aws_secret_key" + } +} +``` + +### HTTP/API +```json +{ + "resource_type": "http", + "value": { + "baseUrl": "https://api.example.com", + "headers": { + "Authorization": "Bearer $var:g/all/api_token" + } + } +} +``` + +### Kafka +```json +{ + "resource_type": "kafka", + "value": { + "brokers": "broker1:9092,broker2:9092", + "sasl_mechanism": "PLAIN", + "security_protocol": "SASL_SSL", + "username": "$var:g/all/kafka_user", + "password": "$var:g/all/kafka_password" + } +} +``` + +### NATS +```json +{ + "resource_type": "nats", + "value": { + "servers": ["nats://localhost:4222"], + "user": "$var:g/all/nats_user", + "password": "$var:g/all/nats_password" + } +} +``` + +### MQTT +```json +{ + "resource_type": "mqtt", + "value": { + "host": "mqtt.example.com", + "port": 8883, + "username": "$var:g/all/mqtt_user", + "password": "$var:g/all/mqtt_password", + "tls": true + } +} +``` + +## Custom Resource Types + +Create custom resource types with JSON Schema: + +```json +{ + "name": "custom_api", + "schema": { + "type": "object", + "properties": { + "base_url": {"type": "string", "format": "uri"}, + "api_key": {"type": "string"}, + "timeout": {"type": "integer", "default": 30} + }, + "required": ["base_url", "api_key"] + }, + "description": "Custom API connection" +} +``` + +Save as: `custom_api.resource-type.json` + +## OAuth Resources + +OAuth resources are managed through the Windmill UI and marked: + +```json +{ + "is_oauth": true, + "account": 123 +} +``` + +OAuth tokens are automatically refreshed by Windmill. + +## Using Resources in Scripts + +### TypeScript (Bun/Deno) +```typescript +export async function main(db: RT.Postgresql) { + // db contains the resource values + const { host, port, user, password, dbname } = db; +} +``` + +### Python +```python +class postgresql(TypedDict): + host: str + port: int + user: str + password: str + dbname: str + +def main(db: postgresql): + # db contains the resource values + pass +``` + +## CLI Commands + +```bash +# List resources +wmill resource list + +# List resource types with schemas +wmill resource-type list --schema + +# Get specific resource type schema +wmill resource-type get postgresql + +# Push resources +wmill sync push +``` diff --git a/system_prompts/auto-generated/skills/schedules/SKILL.md b/system_prompts/auto-generated/skills/schedules/SKILL.md new file mode 100644 index 0000000000..1cd64cb002 --- /dev/null +++ b/system_prompts/auto-generated/skills/schedules/SKILL.md @@ -0,0 +1,51 @@ +--- +name: schedules +description: MUST use when configuring schedules. +--- + +# Windmill Schedules + +Schedules run scripts and flows automatically on a cron schedule. + +## File Naming + +Schedule files use the pattern: `{path}.schedule.yaml` + +Example: `f/folder/daily_sync.schedule.yaml` + +Note: The path is derived from the filename, not stored in the file content. + +## Cron Expression Format + +Windmill uses 6-field cron expressions (includes seconds): + +``` + ┌───────────── second (0-59) + │ ┌───────────── minute (0-59) + │ │ ┌───────────── hour (0-23) + │ │ │ ┌───────────── day of month (1-31) + │ │ │ │ ┌───────────── month (1-12 or jan-dec) + │ │ │ │ │ ┌───────────── day of week (0-6, 0=Sunday, or sun-sat) + │ │ │ │ │ │ + * * * * * * +``` + +**Common Examples:** +- `0 0 0 * * *` - Daily at midnight +- `0 0 12 * * *` - Daily at noon +- `0 */5 * * * *` - Every 5 minutes +- `0 0 9 * * 1-5` - Weekdays at 9 AM +- `0 0 0 1 * *` - First day of each month + +## CLI Commands + +```bash +# Push schedules to Windmill +wmill sync push + +# Pull schedules from Windmill +wmill sync pull + +# List schedules +wmill schedule +``` diff --git a/system_prompts/auto-generated/skills/triggers/SKILL.md b/system_prompts/auto-generated/skills/triggers/SKILL.md new file mode 100644 index 0000000000..5f3a532051 --- /dev/null +++ b/system_prompts/auto-generated/skills/triggers/SKILL.md @@ -0,0 +1,27 @@ +--- +name: triggers +description: MUST use when configuring triggers. +--- + +# Windmill Triggers + +Triggers allow external events to invoke your scripts and flows. + +## File Naming + +Trigger configuration files use the pattern: `{path}.{trigger_type}_trigger.yaml` + +Examples: +- `u/user/webhook.http_trigger.yaml` +- `f/data/kafka_consumer.kafka_trigger.yaml` +- `f/sync/postgres_cdc.postgres_trigger.yaml` + +## CLI Commands + +```bash +# Push trigger configuration +wmill sync push + +# Pull triggers from Windmill +wmill sync pull +``` diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md new file mode 100644 index 0000000000..efa0429195 --- /dev/null +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -0,0 +1,126 @@ +--- +name: write-flow +description: MUST use when creating flows. +--- + +# Windmill Flow Building Guide + +## CLI Commands + +Create a folder ending with `.flow` and add a YAML file with the flow definition. +For rawscript modules, use `!inline path/to/script.ts` for the content key. +After writing: +- `wmill flow generate-locks --yes` - Generate lock files +- `wmill sync push` - Deploy to Windmill + +## OpenFlow Schema + +The OpenFlow schema (openflow.openapi.yaml) is the source of truth for flow structure. Refer to OPENFLOW_SCHEMA for the complete type definitions. + +## Reserved Module IDs + +- `failure` - Reserved for failure handler module +- `preprocessor` - Reserved for preprocessor module +- `Input` - Reserved for flow input reference + +## Module ID Rules + +- Must be unique across the entire flow +- Use underscores, not spaces (e.g., `fetch_data` not `fetch data`) +- Use descriptive names that reflect the step's purpose + +## Common Mistakes to Avoid + +- Missing `input_transforms` - Rawscript parameters won't receive values without them +- Referencing future steps - `results.step_id` only works for steps that execute before the current one +- Duplicate module IDs - Each module ID must be unique in the flow + +## Data Flow Between Steps + +- `flow_input.property` - Access flow input parameters +- `results.step_id` - Access output from a previous step +- `results.step_id.property` - Access specific property from previous step output +- `flow_input.iter.value` - Current item when inside a for-loop +- `flow_input.iter.index` - Current index when inside a for-loop + +## Input Transforms + +Every rawscript module needs `input_transforms` to map function parameters to values: + +Static transform (fixed value): +{"param_name": {"type": "static", "value": "fixed_string"}} + +JavaScript transform (dynamic expression): +{"param_name": {"type": "javascript", "expr": "results.previous_step.data"}} + +## Resource References + +- For flow inputs: Use type `"object"` with format `"resource-{type}"` (e.g., `"resource-postgresql"`) +- For step inputs: Use static value `"$res:path/to/resource"` + +## Failure Handler + +Executes when any step fails. Has access to error details: + +- `error.message` - Error message +- `error.step_id` - ID of failed step +- `error.name` - Error name +- `error.stack` - Stack trace + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +To accept an S3 object as flow input: + +```json +{ + "type": "object", + "properties": { + "file": { + "type": "object", + "format": "resource-s3_object", + "description": "File to process" + } + } +} +``` + +## Using Resources in Flows + +On Windmill, credentials and configuration are stored in resources. Resource types define the format of the resource. + +### As Flow Input + +In the flow schema, set the property type to `"object"` with format `"resource-{type}"`: + +```json +{ + "type": "object", + "properties": { + "database": { + "type": "object", + "format": "resource-postgresql", + "description": "Database connection" + } + } +} +``` + +### As Step Input (Static Reference) + +Reference a specific resource using `$res:` prefix: + +```json +{ + "database": { + "type": "static", + "value": "$res:f/folder/my_database" + } +} +``` + + +## OpenFlow Schema + +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"number","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"number","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"number","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps","additionalProperties":{"type":"string"}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","description":"Custom error message shown when stopping"}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable"]},"access_type":{"type":"string","description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_images":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of image references for vision-capable models.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'images/photo.jpg' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/skills/write-script-bash/SKILL.md b/system_prompts/auto-generated/skills/write-script-bash/SKILL.md new file mode 100644 index 0000000000..a914902deb --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-bash/SKILL.md @@ -0,0 +1,62 @@ +--- +name: write-script-bash +description: MUST use when writing Bash scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Bash + +## Structure + +Do not include `#!/bin/bash`. Arguments are obtained as positional parameters: + +```bash +# Get arguments +var1="$1" +var2="$2" + +echo "Processing $var1 and $var2" + +# Return JSON by echoing to stdout +echo "{\"result\": \"$var1\", \"count\": $var2}" +``` + +**Important:** +- Do not include shebang (`#!/bin/bash`) +- Arguments are always strings +- Access with `$1`, `$2`, etc. + +## Output + +The script output is captured as the result. For structured data, output valid JSON: + +```bash +name="$1" +count="$2" + +# Output JSON result +cat << EOF +{ + "name": "$name", + "count": $count, + "timestamp": "$(date -Iseconds)" +} +EOF +``` + +## Environment Variables + +Environment variables set in Windmill are available: + +```bash +# Access environment variable +echo "Workspace: $WM_WORKSPACE" +echo "Job ID: $WM_JOB_ID" +``` diff --git a/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md b/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md new file mode 100644 index 0000000000..8561d3170e --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md @@ -0,0 +1,24 @@ +--- +name: write-script-bigquery +description: MUST use when writing BigQuery queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# BigQuery + +Arguments use `@name` syntax. + +Name the parameters by adding comments before the statement: + +```sql +-- @name1 (string) +-- @name2 (int64) = 0 +SELECT * FROM users WHERE name = @name1 AND age > @name2; +``` diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md new file mode 100644 index 0000000000..3ec6946859 --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -0,0 +1,594 @@ +--- +name: write-script-bun +description: MUST use when writing Bun/TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# TypeScript (Bun) + +Bun runtime with full npm ecosystem and fastest execution. + +## Structure + +Export a single **async** function called `main`: + +```typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +``` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the `RT` namespace for resource types: + +```typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +``` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +```typescript +import Stripe from "stripe"; +import { someFunction } from "some-package"; +``` + +## Windmill Client + +Import the windmill client for platform interactions: + +```typescript +import * as wmill from "windmill-client"; +``` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter: + +```typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +``` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +```typescript +type S3Object = { + s3: string; // Path within the bucket +}; +``` + +## TypeScript Operations + +```typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +``` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format `$res:path` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * ``` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * ``` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * ``` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getSlackApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age}::int + * `.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age} + * `.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md new file mode 100644 index 0000000000..06be126f5e --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -0,0 +1,592 @@ +--- +name: write-script-bunnative +description: MUST use when writing Bun Native scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# TypeScript (Bun Native) + +Native TypeScript execution with fetch only - no external imports allowed. + +## Structure + +Export a single **async** function called `main`: + +```typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +``` + +Do not call the main function. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the `RT` namespace for resource types: + +```typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +``` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +**No imports allowed.** Use the globally available `fetch` function: + +```typescript +export async function main(url: string) { + const response = await fetch(url); + return await response.json(); +} +``` + +## Windmill Client + +The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter: + +```typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +``` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +```typescript +type S3Object = { + s3: string; // Path within the bucket +}; +``` + +## TypeScript Operations + +```typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +``` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format `$res:path` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * ``` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * ``` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * ``` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getSlackApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age}::int + * `.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age} + * `.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction diff --git a/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md b/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md new file mode 100644 index 0000000000..e0d268d55e --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md @@ -0,0 +1,54 @@ +--- +name: write-script-csharp +description: MUST use when writing C# scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# C# + +The script must contain a public static `Main` method inside a class: + +```csharp +public class Script +{ + public static object Main(string name, int count) + { + return new { Name = name, Count = count }; + } +} +``` + +**Important:** +- Class name is irrelevant +- Method must be `public static` +- Return type can be `object` or specific type + +## NuGet Packages + +Add packages using the `#r` directive at the top: + +```csharp +#r "nuget: Newtonsoft.Json, 13.0.3" +#r "nuget: RestSharp, 110.2.0" + +using Newtonsoft.Json; +using RestSharp; + +public class Script +{ + public static object Main(string url) + { + var client = new RestClient(url); + var request = new RestRequest(); + var response = client.Get(request); + return JsonConvert.DeserializeObject(response.Content); + } +} +``` diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md new file mode 100644 index 0000000000..e9cec1305a --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -0,0 +1,598 @@ +--- +name: write-script-deno +description: MUST use when writing Deno/TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# TypeScript (Deno) + +Deno runtime with npm support via `npm:` prefix and native Deno libraries. + +## Structure + +Export a single **async** function called `main`: + +```typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +``` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the `RT` namespace for resource types: + +```typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +``` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +```typescript +// npm packages use npm: prefix +import Stripe from "npm:stripe"; +import { someFunction } from "npm:some-package"; + +// Deno standard library +import { serve } from "https://deno.land/std/http/server.ts"; +``` + +## Windmill Client + +Import the windmill client for platform interactions: + +```typescript +import * as wmill from "windmill-client"; +``` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter: + +```typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id, + }; +} +``` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +### S3Object Type + +The S3Object type represents a file in S3 storage: + +```typescript +type S3Object = { + s3: string; // Path within the bucket +}; +``` + +## TypeScript Operations + +```typescript +import * as wmill from "windmill-client"; + +// Load file content from S3 +const content: Uint8Array = await wmill.loadS3File(s3object); + +// Load file as stream +const blob: Blob = await wmill.loadS3FileStream(s3object); + +// Write file to S3 +const result: S3Object = await wmill.writeS3File( + s3object, // Target path (or undefined to auto-generate) + fileContent, // string or Blob + s3ResourcePath // Optional: specific S3 resource to use +); +``` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format `$res:path` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * ``` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * ``` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * ``` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getSlackApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age}::int + * `.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age} + * `.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction diff --git a/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md b/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md new file mode 100644 index 0000000000..04f6a3fdec --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md @@ -0,0 +1,64 @@ +--- +name: write-script-duckdb +description: MUST use when writing DuckDB queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# DuckDB + +Arguments are defined with comments and used with `$name` syntax: + +```sql +-- $name (text) = default +-- $age (integer) +SELECT * FROM users WHERE name = $name AND age > $age; +``` + +## Ducklake Integration + +Attach Ducklake for data lake operations: + +```sql +-- Main ducklake +ATTACH 'ducklake' AS dl; + +-- Named ducklake +ATTACH 'ducklake://my_lake' AS dl; + +-- Then query +SELECT * FROM dl.schema.table; +``` + +## External Database Connections + +Connect to external databases using resources: + +```sql +ATTACH '$res:path/to/resource' AS db (TYPE postgres); +SELECT * FROM db.schema.table; +``` + +## S3 File Operations + +Read files from S3 storage: + +```sql +-- Default storage +SELECT * FROM read_csv('s3:///path/to/file.csv'); + +-- Named storage +SELECT * FROM read_csv('s3://storage_name/path/to/file.csv'); + +-- Parquet files +SELECT * FROM read_parquet('s3:///path/to/file.parquet'); + +-- JSON files +SELECT * FROM read_json('s3:///path/to/file.json'); +``` diff --git a/system_prompts/auto-generated/skills/write-script-go/SKILL.md b/system_prompts/auto-generated/skills/write-script-go/SKILL.md new file mode 100644 index 0000000000..ff6b1c490c --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-go/SKILL.md @@ -0,0 +1,71 @@ +--- +name: write-script-go +description: MUST use when writing Go scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Go + +## Structure + +The file package must be `inner` and export a function called `main`: + +```go +package inner + +func main(param1 string, param2 int) (map[string]interface{}, error) { + return map[string]interface{}{ + "result": param1, + "count": param2, + }, nil +} +``` + +**Important:** +- Package must be `inner` +- Return type must be `({return_type}, error)` +- Function name is `main` (lowercase) + +## Return Types + +The return type can be any Go type that can be serialized to JSON: + +```go +package inner + +type Result struct { + Name string `json:"name"` + Count int `json:"count"` +} + +func main(name string, count int) (Result, error) { + return Result{ + Name: name, + Count: count, + }, nil +} +``` + +## Error Handling + +Return errors as the second return value: + +```go +package inner + +import "errors" + +func main(value int) (string, error) { + if value < 0 { + return "", errors.New("value must be positive") + } + return "success", nil +} +``` diff --git a/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md b/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md new file mode 100644 index 0000000000..452a1d4734 --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md @@ -0,0 +1,58 @@ +--- +name: write-script-graphql +description: MUST use when writing GraphQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# GraphQL + +## Structure + +Write GraphQL queries or mutations. Arguments can be added as query parameters: + +```graphql +query GetUser($id: ID!) { + user(id: $id) { + id + name + email + } +} +``` + +## Variables + +Variables are passed as script arguments and automatically bound to the query: + +```graphql +query SearchProducts($query: String!, $limit: Int = 10) { + products(search: $query, first: $limit) { + edges { + node { + id + name + price + } + } + } +} +``` + +## Mutations + +```graphql +mutation CreateUser($input: CreateUserInput!) { + createUser(input: $input) { + id + name + createdAt + } +} +``` diff --git a/system_prompts/auto-generated/skills/write-script-java/SKILL.md b/system_prompts/auto-generated/skills/write-script-java/SKILL.md new file mode 100644 index 0000000000..facc50899e --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-java/SKILL.md @@ -0,0 +1,51 @@ +--- +name: write-script-java +description: MUST use when writing Java scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Java + +The script must contain a Main public class with a `public static main()` method: + +```java +public class Main { + public static Object main(String name, int count) { + java.util.Map result = new java.util.HashMap<>(); + result.put("name", name); + result.put("count", count); + return result; + } +} +``` + +**Important:** +- Class must be named `Main` +- Method must be `public static Object main(...)` +- Return type is `Object` or `void` + +## Maven Dependencies + +Add dependencies using comments at the top: + +```java +//requirements: +//com.google.code.gson:gson:2.10.1 +//org.apache.httpcomponents:httpclient:4.5.14 + +import com.google.gson.Gson; + +public class Main { + public static Object main(String input) { + Gson gson = new Gson(); + return gson.fromJson(input, Object.class); + } +} +``` diff --git a/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md b/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md new file mode 100644 index 0000000000..58ea4982a2 --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md @@ -0,0 +1,24 @@ +--- +name: write-script-mssql +description: MUST use when writing MS SQL Server queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Microsoft SQL Server (MSSQL) + +Arguments use `@P1`, `@P2`, etc. + +Name the parameters by adding comments before the statement: + +```sql +-- @P1 name1 (varchar) +-- @P2 name2 (int) = 0 +SELECT * FROM users WHERE name = @P1 AND age > @P2; +``` diff --git a/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md b/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md new file mode 100644 index 0000000000..8028fa6f1c --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md @@ -0,0 +1,24 @@ +--- +name: write-script-mysql +description: MUST use when writing MySQL queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# MySQL + +Arguments use `?` placeholders. + +Name the parameters by adding comments before the statement: + +```sql +-- ? name1 (text) +-- ? name2 (int) = 0 +SELECT * FROM users WHERE name = ? AND age > ?; +``` diff --git a/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md b/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md new file mode 100644 index 0000000000..67efd5ab4a --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md @@ -0,0 +1,559 @@ +--- +name: write-script-nativets +description: MUST use when writing Native TypeScript scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# TypeScript (Native) + +Native TypeScript execution with fetch only - no external imports allowed. + +## Structure + +Export a single **async** function called `main`: + +```typescript +export async function main(param1: string, param2: number) { + // Your code here + return { result: param1, count: param2 }; +} +``` + +Do not call the main function. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +Use the `RT` namespace for resource types: + +```typescript +export async function main(stripe: RT.Stripe) { + // stripe contains API key and config from the resource +} +``` + +Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + +## Imports + +**No imports allowed.** Use the globally available `fetch` function: + +```typescript +export async function main(url: string) { + const response = await fetch(url); + return await response.json(); +} +``` + +## Windmill Client + +The windmill client is not available in native TypeScript mode. Use fetch to call APIs directly. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter: + +```typescript +type Event = { + kind: + | "webhook" + | "http" + | "websocket" + | "kafka" + | "email" + | "nats" + | "postgres" + | "sqs" + | "mqtt" + | "gcp"; + body: any; + headers: Record; + query: Record; +}; + +export async function preprocessor(event: Event) { + return { + param1: event.body.field1, + param2: event.query.id + }; +} +``` + + +# TypeScript SDK (windmill-client) + +Import: import * as wmill from 'windmill-client' + +/** + * Initialize the Windmill client with authentication token and base URL + * @param token - Authentication token (defaults to WM_TOKEN env variable) + * @param baseUrl - API base URL (defaults to BASE_INTERNAL_URL or BASE_URL env variable) + */ +setClient(token?: string, baseUrl?: string): void + +/** + * Create a client configuration from env variables + * @returns client configuration + */ +getWorkspace(): string + +/** + * Get a resource value by path + * @param path path of the resource, default to internal state path + * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error + * @returns resource value + */ +async getResource(path?: string, undefinedIfEmpty?: boolean): Promise + +/** + * Get the true root job id + * @param jobId job id to get the root job id from (default to current job) + * @returns root job id + */ +async getRootJobId(jobId?: string): Promise + +/** + * @deprecated Use runScriptByPath or runScriptByHash instead + */ +async runScript(path: string | null = null, hash_: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its path and wait for the result + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByPath(path: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Run a script synchronously by its hash and wait for the result + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param verbose - Enable verbose logging + * @returns Script execution result + */ +async runScriptByHash(hash_: string, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Append a text to the result stream + * @param text text to append to the result stream + */ +appendToResultStream(text: string): void + +/** + * Stream to the result stream + * @param stream stream to stream to the result stream + */ +async streamResult(stream: AsyncIterable): Promise + +/** + * Run a flow synchronously by its path and wait for the result + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param verbose - Enable verbose logging + * @returns Flow execution result + */ +async runFlow(path: string | null = null, args: Record | null = null, verbose: boolean = false): Promise + +/** + * Wait for a job to complete and return its result + * @param jobId - ID of the job to wait for + * @param verbose - Enable verbose logging + * @returns Job result when completed + */ +async waitJob(jobId: string, verbose: boolean = false): Promise + +/** + * Get the result of a completed job + * @param jobId - ID of the completed job + * @returns Job result + */ +async getResult(jobId: string): Promise + +/** + * Get the result of a job if completed, or its current status + * @param jobId - ID of the job + * @returns Object with started, completed, success, and result properties + */ +async getResultMaybe(jobId: string): Promise + +/** + * Wrap a function to execute as a Windmill task within a flow context + * @param f - Function to wrap as a task + * @returns Async wrapper function that executes as a Windmill job + */ +task(f: (_: P) => T): (_: P) => Promise + +/** + * @deprecated Use runScriptByPathAsync or runScriptByHashAsync instead + */ +async runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its path + * @param path - Script path in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a script asynchronously by its hash + * @param hash_ - Script hash in Windmill + * @param args - Arguments to pass to the script + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @returns Job ID of the created job + */ +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null): Promise + +/** + * Run a flow asynchronously by its path + * @param path - Flow path in Windmill + * @param args - Arguments to pass to the flow + * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param doNotTrackInParent - If false, tracks state in parent job (only use when fully awaiting the job) + * @returns Job ID of the created job + */ +async runFlowAsync(path: string | null, args: Record | null, scheduledInSeconds: number | null = null, // can only be set to false if this the job will be fully await and not concurrent with any other job // as otherwise the child flow and its own child will store their state in the parent job which will // lead to incorrectness and failures doNotTrackInParent: boolean = true): Promise + +/** + * Resolve a resource value in case the default value was picked because the input payload was undefined + * @param obj resource value or path of the resource under the format `$res:path` + * @returns resource value + */ +async resolveDefaultResource(obj: any): Promise + +/** + * Get the state file path from environment variables + * @returns State path string + */ +getStatePath(): string + +/** + * Set a resource value by path + * @param path path of the resource to set, default to state path + * @param value new value of the resource to set + * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type + */ +async setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise + +/** + * Set the state + * @param state state to set + * @deprecated use setState instead + */ +async setInternalState(state: any): Promise + +/** + * Set the state + * @param state state to set + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async setState(state: any, path?: string): Promise + +/** + * Set the progress + * Progress cannot go back and limited to 0% to 99% range + * @param percent Progress to set in % + * @param jobId? Job to set progress for + */ +async setProgress(percent: number, jobId?: any): Promise + +/** + * Get the progress + * @param jobId? Job to get progress from + * @returns Optional clamped between 0 and 100 progress value + */ +async getProgress(jobId?: any): Promise + +/** + * Set a flow user state + * @param key key of the state + * @param value value of the state + */ +async setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise + +/** + * Get a flow user state + * @param path path of the variable + */ +async getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise + +/** + * Get the internal state + * @deprecated use getState instead + */ +async getInternalState(): Promise + +/** + * Get the state shared across executions + * @param path Optional state resource path override. Defaults to `getStatePath()`. + */ +async getState(path?: string): Promise + +/** + * Get a variable by path + * @param path path of the variable + * @returns variable value + */ +async getVariable(path: string): Promise + +/** + * Set a variable by path, create if not exist + * @param path path of the variable + * @param value value of the variable + * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) + * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") + */ +async setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise + +/** + * Build a PostgreSQL connection URL from a database resource + * @param path - Path to the database resource + * @returns PostgreSQL connection URL string + */ +async databaseUrlFromResource(path: string): Promise + +async polarsConnectionSettings(s3_resource_path: string | undefined): Promise + +async duckdbConnectionSettings(s3_resource_path: string | undefined): Promise + +/** + * Get S3 client settings from a resource or workspace default + * @param s3_resource_path - Path to S3 resource (uses workspace default if undefined) + * @returns S3 client configuration settings + */ +async denoS3LightClientSettings(s3_resource_path: string | undefined): Promise + +/** + * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContent = await wmill.loadS3FileContent(inputFile) + * // if the file is a raw text file, it can be decoded and printed directly: + * const text = new TextDecoder().decode(fileContentStream) + * console.log(text); + * ``` + */ +async loadS3File(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * let fileContentBlob = await wmill.loadS3FileStream(inputFile) + * // if the content is plain text, the blob can be read directly: + * console.log(await fileContentBlob.text()); + * ``` + */ +async loadS3FileStream(s3object: S3Object, s3ResourcePath: string | undefined = undefined): Promise + +/** + * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. + * + * ```typescript + * const s3object = await writeS3File(s3Object, "Hello Windmill!") + * const fileContentAsUtf8Str = (await s3object.toArray()).toString('utf-8') + * console.log(fileContentAsUtf8Str) + * ``` + */ +async writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath: string | undefined = undefined, contentType: string | undefined = undefined, contentDisposition: string | undefined = undefined): Promise + +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +async signS3Objects(s3objects: S3Object[]): Promise + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +async signS3Object(s3object: S3Object): Promise + +/** + * Generate a presigned public URL for an array of S3 objects. + * If an S3 object is not signed yet, it will be signed first. + * @param s3Objects s3 objects to sign + * @returns list of signed public URLs + */ +async getPresignedS3PublicUrls(s3Objects: S3Object[], { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Generate a presigned public URL for an S3 object. If the S3 object is not signed yet, it will be signed first. + * @param s3Object s3 object to sign + * @returns signed public URL + */ +async getPresignedS3PublicUrl(s3Objects: S3Object, { baseUrl }: { baseUrl?: string } = {}): Promise + +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @param flowLevel if true, generate resume URLs for the parent flow instead of the specific step. + * This allows pre-approvals that can be consumed by any later suspend step in the same flow. + * @returns approval page UI URL, resume and cancel API URLs for resuming the flow + */ +async getResumeUrls(approver?: string, flowLevel?: boolean): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * @deprecated use getResumeUrls instead + */ +getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> + +/** + * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) + * @param audience audience of the token + * @param expiresIn Optional number of seconds until the token expires + * @returns jwt token + */ +async getIdToken(audience: string, expiresIn?: number): Promise + +/** + * Convert a base64-encoded string to Uint8Array + * @param data - Base64-encoded string + * @returns Decoded Uint8Array + */ +base64ToUint8Array(data: string): Uint8Array + +/** + * Convert a Uint8Array to base64-encoded string + * @param arrayBuffer - Uint8Array to encode + * @returns Base64-encoded string + */ +uint8ArrayToBase64(arrayBuffer: Uint8Array): string + +/** + * Get email from workspace username + * This method is particularly useful for apps that require the email address of the viewer. + * Indeed, in the viewer context, WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. + * @param username + * @returns email address + */ +async usernameToEmail(username: string): Promise + +/** + * Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Slack approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Slack approval request. + * @param {string} options.slackResourcePath - The path to the Slack resource in Windmill. + * @param {string} options.channelId - The Slack channel ID where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Slack approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Slack approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getSlackApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveSlackApproval({ + * slackResourcePath: "/u/alex/my_slack_resource", + * channelId: "admins-slack-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveSlackApproval({ slackResourcePath, channelId, message, approver, defaultArgsJson, dynamicEnumsJson, }: SlackApprovalOptions): Promise + +/** + * Sends an interactive approval request via Teams, allowing optional customization of the message, approver, and form fields. + * + * **[Enterprise Edition Only]** To include form fields in the Teams approval request, go to **Advanced -> Suspend -> Form** + * and define a form. Learn more at [Windmill Documentation](https://www.windmill.dev/docs/flows/flow_approval#form). + * + * @param {Object} options - The configuration options for the Teams approval request. + * @param {string} options.teamName - The Teams team name where the approval request will be sent. + * @param {string} options.channelName - The Teams channel name where the approval request will be sent. + * @param {string} [options.message] - Optional custom message to include in the Teams approval request. + * @param {string} [options.approver] - Optional user ID or name of the approver for the request. + * @param {DefaultArgs} [options.defaultArgsJson] - Optional object defining or overriding the default arguments to a form field. + * @param {Enums} [options.dynamicEnumsJson] - Optional object overriding the enum default values of an enum form field. + * + * @returns {Promise} Resolves when the Teams approval request is successfully sent. + * + * @throws {Error} If the function is not called within a flow or flow preview. + * @throws {Error} If the `JobService.getTeamsApprovalPayload` call fails. + * + * **Usage Example:** + * ```typescript + * await requestInteractiveTeamsApproval({ + * teamName: "admins-teams", + * channelName: "admins-teams-channel", + * message: "Please approve this request", + * approver: "approver123", + * defaultArgsJson: { key1: "value1", key2: 42 }, + * dynamicEnumsJson: { foo: ["choice1", "choice2"], bar: ["optionA", "optionB"] }, + * }); + * ``` + * + * **Note:** This function requires execution within a Windmill flow or flow preview. + */ +async requestInteractiveTeamsApproval({ teamName, channelName, message, approver, defaultArgsJson, dynamicEnumsJson, }: TeamsApprovalOptions): Promise + +/** + * Parse an S3 object from URI string or record format + * @param s3Object - S3 object as URI string (s3://storage/key) or record + * @returns S3 object record with storage and s3 key + */ +parseS3Object(s3Object: S3Object): S3ObjectRecord + +/** + * Create a SQL template function for PostgreSQL/datatable queries + * @param name - Database/datatable name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age}::int + * `.fetch() + */ +datatable(name: string = "main"): DatatableSqlTemplateFunction + +/** + * Create a SQL template function for DuckDB/ducklake queries + * @param name - DuckDB database name (default: "main") + * @returns SQL template function for building parameterized queries + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age} + * `.fetch() + */ +ducklake(name: string = "main"): SqlTemplateFunction diff --git a/system_prompts/auto-generated/skills/write-script-php/SKILL.md b/system_prompts/auto-generated/skills/write-script-php/SKILL.md new file mode 100644 index 0000000000..c2d02ff3bf --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-php/SKILL.md @@ -0,0 +1,70 @@ +--- +name: write-script-php +description: MUST use when writing PHP scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# PHP + +## Structure + +The script must start with ` $param1, "count" => $param2]; +} +``` + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +You need to **redefine** the type of the resources that are needed before the main function. Always check if the class already exists using `class_exists`: + +```php + $2::INT; +``` diff --git a/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md b/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md new file mode 100644 index 0000000000..e54f3e647b --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md @@ -0,0 +1,68 @@ +--- +name: write-script-powershell +description: MUST use when writing PowerShell scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# PowerShell + +## Structure + +Arguments are obtained by calling the `param` function on the first line: + +```powershell +param($Name, $Count = 0, [int]$Age) + +# Your code here +Write-Output "Processing $Name, count: $Count, age: $Age" + +# Return object +@{ + name = $Name + count = $Count + age = $Age +} +``` + +## Parameter Types + +You can specify types for parameters: + +```powershell +param( + [string]$Name, + [int]$Count = 0, + [bool]$Enabled = $true, + [array]$Items +) + +@{ + name = $Name + count = $Count + enabled = $Enabled + items = $Items +} +``` + +## Return Values + +Return values by outputting them at the end of the script: + +```powershell +param($Input) + +$result = @{ + processed = $true + data = $Input + timestamp = Get-Date -Format "o" +} + +$result +``` diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md new file mode 100644 index 0000000000..1e15bc2f64 --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -0,0 +1,716 @@ +--- +name: write-script-python3 +description: MUST use when writing Python scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Python + +## Structure + +The script must contain at least one function called `main`: + +```python +def main(param1: str, param2: int): + # Your code here + return {"result": param1, "count": param2} +``` + +Do not call the main function. Libraries are installed automatically. + +## Resource Types + +On Windmill, credentials and configuration are stored in resources and passed as parameters to main. + +You need to **redefine** the type of the resources that are needed before the main function as TypedDict: + +```python +from typing import TypedDict + +class postgresql(TypedDict): + host: str + port: int + user: str + password: str + dbname: str + +def main(db: postgresql): + # db contains the database connection details + pass +``` + +**Important rules:** + +- The resource type name must be **IN LOWERCASE** +- Only include resource types if they are actually needed +- If an import conflicts with a resource type name, **rename the imported object, not the type name** +- Make sure to import TypedDict from typing **if you're using it** + +## Imports + +Libraries are installed automatically. Do not show installation instructions. + +```python +import requests +import pandas as pd +from datetime import datetime +``` + +If an import name conflicts with a resource type: + +```python +# Wrong - don't rename the type +import stripe as stripe_lib +class stripe_type(TypedDict): ... + +# Correct - rename the import +import stripe as stripe_sdk +class stripe(TypedDict): + api_key: str +``` + +## Windmill Client + +Import the windmill client for platform interactions: + +```python +import wmill +``` + +See the SDK documentation for available methods. + +## Preprocessor Scripts + +For preprocessor scripts, the function should be named `preprocessor` and receives an `event` parameter: + +```python +from typing import TypedDict, Literal, Any + +class Event(TypedDict): + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + body: Any + headers: dict[str, str] + query: dict[str, str] + +def preprocessor(event: Event): + # Transform the event into flow input parameters + return { + "param1": event["body"]["field1"], + "param2": event["query"]["id"] + } +``` + +## S3 Object Operations + +Windmill provides built-in support for S3-compatible storage operations. + +```python +import wmill + +# Load file content from S3 +content: bytes = wmill.load_s3_file(s3object) + +# Load file as stream reader +reader: BufferedReader = wmill.load_s3_file_reader(s3object) + +# Write file to S3 +result: S3Object = wmill.write_s3_file( + s3object, # Target path (or None to auto-generate) + file_content, # bytes or BufferedReader + s3_resource_path, # Optional: specific S3 resource + content_type, # Optional: MIME type + content_disposition # Optional: Content-Disposition header +) +``` + + +# Python SDK (wmill) + +Import: import wmill + +def get_mocked_api() -> Optional[dict] + +# Get the HTTP client instance. +# +# Returns: +# Configured httpx.Client for API requests +def get_client() -> httpx.Client + +# Make an HTTP GET request to the Windmill API. +# +# Args: +# endpoint: API endpoint path +# raise_for_status: Whether to raise an exception on HTTP errors +# **kwargs: Additional arguments passed to httpx.get +# +# Returns: +# HTTP response object +def get(endpoint, raise_for_status = True, **kwargs) -> httpx.Response + +# Make an HTTP POST request to the Windmill API. +# +# Args: +# endpoint: API endpoint path +# raise_for_status: Whether to raise an exception on HTTP errors +# **kwargs: Additional arguments passed to httpx.post +# +# Returns: +# HTTP response object +def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response + +# Create a new authentication token. +# +# Args: +# duration: Token validity duration (default: 1 day) +# +# Returns: +# New authentication token string +def create_token(duration = dt.timedelta(days=1)) -> str + +# Create a script job and return its job id. +# +# .. deprecated:: Use run_script_by_path_async or run_script_by_hash_async instead. +def run_script_async(path: str = None, hash_: str = None, args: dict = None, scheduled_in_secs: int = None) -> str + +# Create a script job by path and return its job id. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None) -> str + +# Create a script job by hash and return its job id. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None) -> str + +# Create a flow job and return its job id. +def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True) -> str + +# Run script synchronously and return its result. +# +# .. deprecated:: Use run_script_by_path or run_script_by_hash instead. +def run_script(path: str = None, hash_: str = None, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any + +# Run script by path synchronously and return its result. +def run_script_by_path(path: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any + +# Run script by hash synchronously and return its result. +def run_script_by_hash(hash_: str, args: dict = None, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) -> Any + +# Run a script on the current worker without creating a job +def run_inline_script_preview(content: str, language: str, args: dict = None) -> Any + +# Wait for a job to complete and return its result. +# +# Args: +# job_id: ID of the job to wait for +# timeout: Maximum time to wait (seconds or timedelta) +# verbose: Enable verbose logging +# cleanup: Register cleanup handler to cancel job on exit +# assert_result_is_not_none: Raise exception if result is None +# +# Returns: +# Job result when completed +# +# Raises: +# TimeoutError: If timeout is reached +# Exception: If job fails +def wait_job(job_id, timeout: dt.timedelta | int | float | None = None, verbose: bool = False, cleanup: bool = True, assert_result_is_not_none: bool = False) + +# Cancel a specific job by ID. +# +# Args: +# job_id: UUID of the job to cancel +# reason: Optional reason for cancellation +# +# Returns: +# Response message from the cancel endpoint +def cancel_job(job_id: str, reason: str = None) -> str + +# Cancel currently running executions of the same script. +def cancel_running() -> dict + +# Get job details by ID. +# +# Args: +# job_id: UUID of the job +# +# Returns: +# Job details dictionary +def get_job(job_id: str) -> dict + +# Get the root job ID for a flow hierarchy. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# Root job ID +def get_root_job_id(job_id: str | None = None) -> dict + +# Get an OIDC JWT token for authentication to external services. +# +# Args: +# audience: Token audience (e.g., "vault", "aws") +# expires_in: Optional expiration time in seconds +# +# Returns: +# JWT token string +def get_id_token(audience: str, expires_in: int | None = None) -> str + +# Get the status of a job. +# +# Args: +# job_id: UUID of the job +# +# Returns: +# Job status: "RUNNING", "WAITING", or "COMPLETED" +def get_job_status(job_id: str) -> JobStatus + +# Get the result of a completed job. +# +# Args: +# job_id: UUID of the completed job +# assert_result_is_not_none: Raise exception if result is None +# +# Returns: +# Job result +def get_result(job_id: str, assert_result_is_not_none: bool = True) -> Any + +# Get a variable value by path. +# +# Args: +# path: Variable path in Windmill +# +# Returns: +# Variable value as string +def get_variable(path: str) -> str + +# Set a variable value by path, creating it if it doesn't exist. +# +# Args: +# path: Variable path in Windmill +# value: Variable value to set +# is_secret: Whether the variable should be secret (default: False) +def set_variable(path: str, value: str, is_secret: bool = False) -> None + +# Get a resource value by path. +# +# Args: +# path: Resource path in Windmill +# none_if_undefined: Return None instead of raising if not found +# interpolated: if variables and resources are fully unrolled +# +# Returns: +# Resource value dictionary or None +def get_resource(path: str, none_if_undefined: bool = False, interpolated: bool = True) -> dict | None + +# Set a resource value by path, creating it if it doesn't exist. +# +# Args: +# value: Resource value to set +# path: Resource path in Windmill +# resource_type: Resource type for creation +def set_resource(value: Any, path: str, resource_type: str) + +# List resources from Windmill workspace. +# +# Args: +# resource_type: Optional resource type to filter by (e.g., "postgresql", "mysql", "s3") +# page: Optional page number for pagination +# per_page: Optional number of results per page +# +# Returns: +# List of resource dictionaries +def list_resources(resource_type: str = None, page: int = None, per_page: int = None) -> list[dict] + +# Set the workflow state. +# +# Args: +# value: State value to set +# path: Optional state resource path override. +def set_state(value: Any, path: str | None = None) -> None + +# Get the workflow state. +# +# Args: +# path: Optional state resource path override. +# +# Returns: +# State value or None if not set +def get_state(path: str | None = None) -> Any + +# Set job progress percentage (0-99). +# +# Args: +# value: Progress percentage +# job_id: Job ID (defaults to current WM_JOB_ID) +def set_progress(value: int, job_id: Optional[str] = None) + +# Get job progress percentage. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# Progress value (0-100) or None if not set +def get_progress(job_id: Optional[str] = None) -> Any + +# Set the user state of a flow at a given key +def set_flow_user_state(key: str, value: Any) -> None + +# Get the user state of a flow at a given key +def get_flow_user_state(key: str) -> Any + +# Get the Windmill server version. +# +# Returns: +# Version string +def version() + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from DuckDB +def get_duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings | None + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from Polars +def get_polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection using boto3 +def get_boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings + +# Load a file from the workspace s3 bucket and returns its content as bytes. +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# my_obj_content = client.load_s3_file(s3_obj) +# file_content = my_obj_content.decode("utf-8") +# ''' +def load_s3_file(s3object: S3Object | str, s3_resource_path: str | None) -> bytes + +# Load a file from the workspace s3 bucket and returns the bytes stream. +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# with wmill.load_s3_file_reader(s3object, s3_resource_path) as file_reader: +# print(file_reader.read()) +# ''' +def load_s3_file_reader(s3object: S3Object | str, s3_resource_path: str | None) -> BufferedReader + +# Write a file to the workspace S3 bucket +# +# '''python +# from wmill import S3Object +# +# s3_obj = S3Object(s3="/path/to/my_file.txt") +# +# # for an in memory bytes array: +# file_content = b'Hello Windmill!' +# client.write_s3_file(s3_obj, file_content) +# +# # for a file: +# with open("my_file.txt", "rb") as my_file: +# client.write_s3_file(s3_obj, my_file) +# ''' +def write_s3_file(s3object: S3Object | str | None, file_content: BufferedReader | bytes, s3_resource_path: str | None, content_type: str | None = None, content_disposition: str | None = None) -> S3Object + +# Sign S3 objects for use by anonymous users in public apps. +# +# Args: +# s3_objects: List of S3 objects to sign +# +# Returns: +# List of signed S3 objects +def sign_s3_objects(s3_objects: list[S3Object | str]) -> list[S3Object] + +# Sign a single S3 object for use by anonymous users in public apps. +# +# Args: +# s3_object: S3 object to sign +# +# Returns: +# Signed S3 object +def sign_s3_object(s3_object: S3Object | str) -> S3Object + +# Generate presigned public URLs for an array of S3 objects. +# If an S3 object is not signed yet, it will be signed first. +# +# Args: +# s3_objects: List of S3 objects to sign +# base_url: Optional base URL for the presigned URLs (defaults to WM_BASE_URL) +# +# Returns: +# List of signed public URLs +# +# Example: +# >>> s3_objs = [S3Object(s3="/path/to/file1.txt"), S3Object(s3="/path/to/file2.txt")] +# >>> urls = client.get_presigned_s3_public_urls(s3_objs) +def get_presigned_s3_public_urls(s3_objects: list[S3Object | str], base_url: str | None = None) -> list[str] + +# Generate a presigned public URL for an S3 object. +# If the S3 object is not signed yet, it will be signed first. +# +# Args: +# s3_object: S3 object to sign +# base_url: Optional base URL for the presigned URL (defaults to WM_BASE_URL) +# +# Returns: +# Signed public URL +# +# Example: +# >>> s3_obj = S3Object(s3="/path/to/file.txt") +# >>> url = client.get_presigned_s3_public_url(s3_obj) +def get_presigned_s3_public_url(s3_object: S3Object | str, base_url: str | None = None) -> str + +# Get the current user information. +# +# Returns: +# User details dictionary +def whoami() -> dict + +# Get the current user information (alias for whoami). +# +# Returns: +# User details dictionary +def user() -> dict + +# Get the state resource path from environment. +# +# Returns: +# State path string +def state_path() -> str + +# Get the workflow state. +# +# Returns: +# State value or None if not set +def state() -> Any + +# Set the state in the shared folder using pickle +def set_shared_state_pickle(value: Any, path: str = 'state.pickle') -> None + +# Get the state in the shared folder using pickle +def get_shared_state_pickle(path: str = 'state.pickle') -> Any + +# Set the state in the shared folder using pickle +def set_shared_state(value: Any, path: str = 'state.json') -> None + +# Get the state in the shared folder using pickle +def get_shared_state(path: str = 'state.json') -> None + +# Get URLs needed for resuming a flow after suspension. +# +# Args: +# approver: Optional approver name +# flow_level: If True, generate resume URLs for the parent flow instead of the +# specific step. This allows pre-approvals that can be consumed by any later +# suspend step in the same flow. +# +# Returns: +# Dictionary with approvalPage, resume, and cancel URLs +def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict + +# Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. +# +# **[Enterprise Edition Only]** To include form fields in the Slack approval request, use the "Advanced -> Suspend -> Form" functionality. +# Learn more at: https://www.windmill.dev/docs/flows/flow_approval#form +# +# :param slack_resource_path: The path to the Slack resource in Windmill. +# :type slack_resource_path: str +# :param channel_id: The Slack channel ID where the approval request will be sent. +# :type channel_id: str +# :param message: Optional custom message to include in the Slack approval request. +# :type message: str, optional +# :param approver: Optional user ID or name of the approver for the request. +# :type approver: str, optional +# :param default_args_json: Optional dictionary defining or overriding the default arguments for form fields. +# :type default_args_json: dict, optional +# :param dynamic_enums_json: Optional dictionary overriding the enum default values of enum form fields. +# :type dynamic_enums_json: dict, optional +# +# :raises Exception: If the function is not called within a flow or flow preview. +# :raises Exception: If the required flow job or flow step environment variables are not set. +# +# :return: None +# +# **Usage Example:** +# >>> client.request_interactive_slack_approval( +# ... slack_resource_path="/u/alex/my_slack_resource", +# ... channel_id="admins-slack-channel", +# ... message="Please approve this request", +# ... approver="approver123", +# ... default_args_json={"key1": "value1", "key2": 42}, +# ... dynamic_enums_json={"foo": ["choice1", "choice2"], "bar": ["optionA", "optionB"]}, +# ... ) +# +# **Notes:** +# - This function must be executed within a Windmill flow or flow preview. +# - The function checks for required environment variables (`WM_FLOW_JOB_ID`, `WM_FLOW_STEP_ID`) to ensure it is run in the appropriate context. +def request_interactive_slack_approval(slack_resource_path: str, channel_id: str, message: str = None, approver: str = None, default_args_json: dict = None, dynamic_enums_json: dict = None) -> None + +# Get email from workspace username +# This method is particularly useful for apps that require the email address of the viewer. +# Indeed, in the viewer context WM_USERNAME is set to the username of the viewer but WM_EMAIL is set to the email of the creator of the app. +def username_to_email(username: str) -> str + +# Send a message to a Microsoft Teams conversation with conversation_id, where success is used to style the message +def send_teams_message(conversation_id: str, text: str, success: bool = True, card_block: dict = None) + +# Get a DataTable client for SQL queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DataTableClient instance +def datatable(name: str = 'main') + +# Get a DuckLake client for DuckDB queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DucklakeClient instance +def ducklake(name: str = 'main') + +def init_global_client(f) + +def deprecate(in_favor_of: str) + +# Get the current workspace ID. +# +# Returns: +# Workspace ID string +def get_workspace() -> str + +def get_version() -> str + +# Run a script synchronously by hash and return its result. +# +# Args: +# hash: Script hash +# args: Script arguments +# verbose: Enable verbose logging +# assert_result_is_not_none: Raise exception if result is None +# cleanup: Register cleanup handler to cancel job on exit +# timeout: Maximum time to wait +# +# Returns: +# Script result +def run_script_sync(hash: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None) -> Any + +# Run a script synchronously by path and return its result. +# +# Args: +# path: Script path +# args: Script arguments +# verbose: Enable verbose logging +# assert_result_is_not_none: Raise exception if result is None +# cleanup: Register cleanup handler to cancel job on exit +# timeout: Maximum time to wait +# +# Returns: +# Script result +def run_script_by_path_sync(path: str, args: Dict[str, Any] = None, verbose: bool = False, assert_result_is_not_none: bool = True, cleanup: bool = True, timeout: dt.timedelta = None) -> Any + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from DuckDB +def duckdb_connection_settings(s3_resource_path: str = '') -> DuckDbConnectionSettings + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection from Polars +def polars_connection_settings(s3_resource_path: str = '') -> PolarsConnectionSettings + +# Convenient helpers that takes an S3 resource as input and returns the settings necessary to +# initiate an S3 connection using boto3 +def boto3_connection_settings(s3_resource_path: str = '') -> Boto3ConnectionSettings + +# Get the state resource path from environment. +# +# Returns: +# State path string +def get_state_path() -> str + +# Decorator to mark a function as a workflow task. +# +# When executed inside a Windmill job, the decorated function runs as a +# separate workflow step. Outside Windmill, it executes normally. +# +# Args: +# tag: Optional worker tag for execution +# +# Returns: +# Decorated function +def task(*args, **kwargs) + +# Parse resource syntax from string. +def parse_resource_syntax(s: str) -> Optional[str] + +# Parse S3 object from string or S3Object format. +def parse_s3_object(s3_object: S3Object | str) -> S3Object + +# Parse variable syntax from string. +def parse_variable_syntax(s: str) -> Optional[str] + +# Append a text to the result stream. +# +# Args: +# text: text to append to the result stream +def append_to_result_stream(text: str) -> None + +# Stream to the result stream. +# +# Args: +# stream: stream to stream to the result stream +def stream_result(stream) -> None + +# Execute a SQL query against the DataTable. +# +# Args: +# sql: SQL query string with $1, $2, etc. placeholders +# *args: Positional arguments to bind to query placeholders +# +# Returns: +# SqlQuery instance for fetching results +def query(sql: str, *args) -> SqlQuery + +# Execute query and fetch results. +# +# Args: +# result_collection: Optional result collection mode +# +# Returns: +# Query results +def fetch(result_collection: str | None = None) + +# Execute query and fetch first row of results. +# +# Returns: +# First row of query results +def fetch_one() + +# Execute query and fetch first row of results. Return result as a scalar value. +# +# Returns: +# First row of query result as a scalar value +def fetch_one_scalar() + +# Execute query and don't return any results. +# +def execute() + +# DuckDB executor requires explicit argument types at declaration +# These types exist in both DuckDB and Postgres +# Check that the types exist if you plan to extend this function for other SQL engines. +def infer_sql_type(value) -> str + +def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] + diff --git a/system_prompts/auto-generated/skills/write-script-rust/SKILL.md b/system_prompts/auto-generated/skills/write-script-rust/SKILL.md new file mode 100644 index 0000000000..3dad884ebd --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-rust/SKILL.md @@ -0,0 +1,88 @@ +--- +name: write-script-rust +description: MUST use when writing Rust scripts. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Rust + +## Structure + +The script must contain a function called `main` with proper return type: + +```rust +use anyhow::anyhow; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct ReturnType { + result: String, + count: i32, +} + +fn main(param1: String, param2: i32) -> anyhow::Result { + Ok(ReturnType { + result: param1, + count: param2, + }) +} +``` + +**Important:** +- Arguments should be owned types +- Return type must be serializable (`#[derive(Serialize)]`) +- Return type is `anyhow::Result` + +## Dependencies + +Packages must be specified with a partial cargo.toml at the beginning of the script: + +```rust +//! ```cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! reqwest = { version = "0.11", features = ["json"] } +//! tokio = { version = "1", features = ["full"] } +//! ``` + +use anyhow::anyhow; +// ... rest of the code +``` + +**Note:** Serde is already included, no need to add it again. + +## Async Functions + +If you need to handle async functions (e.g., using tokio), keep the main function sync and create the runtime inside: + +```rust +//! ```cargo +//! [dependencies] +//! anyhow = "1.0.86" +//! tokio = { version = "1", features = ["full"] } +//! reqwest = { version = "0.11", features = ["json"] } +//! ``` + +use anyhow::anyhow; +use serde::Serialize; + +#[derive(Serialize, Debug)] +struct Response { + data: String, +} + +fn main(url: String) -> anyhow::Result { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { + let resp = reqwest::get(&url).await?.text().await?; + Ok(Response { data: resp }) + }) +} +``` diff --git a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md new file mode 100644 index 0000000000..54667c0432 --- /dev/null +++ b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md @@ -0,0 +1,24 @@ +--- +name: write-script-snowflake +description: MUST use when writing Snowflake queries. +--- + +## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types. + +# Snowflake + +Arguments use `?` placeholders. + +Name the parameters by adding comments before the statement: + +```sql +-- ? name1 (text) +-- ? name2 (number) = 0 +SELECT * FROM users WHERE name = ? AND age > ?; +``` diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index 4d74775c7c..bfd837002d 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -1,5 +1,15 @@ # Windmill Flow Building Guide +## CLI Commands + +Create a folder ending with `.flow` and add a YAML file with the flow definition. +For rawscript modules, use `!inline path/to/script.ts` for the content key. +After writing: +- `wmill flow generate-locks --yes` - Generate lock files +- `wmill sync push` - Deploy to Windmill + +## OpenFlow Schema + The OpenFlow schema (openflow.openapi.yaml) is the source of truth for flow structure. Refer to OPENFLOW_SCHEMA for the complete type definitions. ## Reserved Module IDs diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md new file mode 100644 index 0000000000..0dbcac9462 --- /dev/null +++ b/system_prompts/base/raw-app.md @@ -0,0 +1,251 @@ +# Windmill Raw Apps + +Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. + +## Creating a Raw App + +```bash +wmill app new +``` + +This interactive command creates a complete app structure with your choice of frontend framework (React, Svelte, or Vue). + +## App Structure + +``` +my_app.raw_app/ +├── AGENTS.md # AI agent instructions (auto-generated) +├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) +├── raw_app.yaml # App configuration (summary, path, data settings) +├── index.tsx # Frontend entry point +├── App.tsx # Main React/Svelte/Vue component +├── index.css # Styles +├── package.json # Frontend dependencies +├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) +├── backend/ # Backend runnables (server-side scripts) +│ ├── . # Code file (e.g., get_user.ts) +│ ├── .yaml # Optional: config for fields, or to reference existing scripts +│ └── .lock # Lock file (run 'wmill app generate-locks' to create) +└── sql_to_apply/ # SQL migrations (dev only, not synced) + └── *.sql # SQL files to apply via dev server +``` + +## Backend Runnables + +Backend runnables are server-side scripts that your frontend can call. They live in the `backend/` folder. + +### Creating a Backend Runnable + +Add a code file to the `backend/` folder: + +``` +backend/. +``` + +The runnable ID is the filename without extension. For example, `get_user.ts` creates a runnable with ID `get_user`. + +### Supported Languages + +| Language | Extension | Example | +|------------------|--------------|------------------| +| TypeScript | `.ts` | `myFunc.ts` | +| TypeScript (Bun) | `.bun.ts` | `myFunc.bun.ts` | +| TypeScript (Deno)| `.deno.ts` | `myFunc.deno.ts` | +| Python | `.py` | `myFunc.py` | +| Go | `.go` | `myFunc.go` | +| Bash | `.sh` | `myFunc.sh` | +| PowerShell | `.ps1` | `myFunc.ps1` | +| PostgreSQL | `.pg.sql` | `myFunc.pg.sql` | +| MySQL | `.my.sql` | `myFunc.my.sql` | +| BigQuery | `.bq.sql` | `myFunc.bq.sql` | +| Snowflake | `.sf.sql` | `myFunc.sf.sql` | +| MS SQL | `.ms.sql` | `myFunc.ms.sql` | +| GraphQL | `.gql` | `myFunc.gql` | +| PHP | `.php` | `myFunc.php` | +| Rust | `.rs` | `myFunc.rs` | +| C# | `.cs` | `myFunc.cs` | +| Java | `.java` | `myFunc.java` | + +### Example Backend Runnable + +**backend/get_user.ts:** +```typescript +import * as wmill from 'windmill-client'; + +export async function main(user_id: string) { + const sql = wmill.datatable(); + const user = await sql`SELECT * FROM users WHERE id = ${user_id}`.fetchOne(); + return user; +} +``` + +After creating, generate lock files: +```bash +wmill app generate-locks +``` + +### Optional YAML Configuration + +Add a `.yaml` file to configure fields or static values: + +**backend/get_user.yaml:** +```yaml +type: inline +fields: + user_id: + type: static + value: "default_user" +``` + +### Referencing Existing Scripts + +To use an existing Windmill script instead of inline code: + +**backend/existing_script.yaml:** +```yaml +type: script +path: f/my_folder/existing_script +``` + +For flows: +```yaml +type: flow +path: f/my_folder/my_flow +``` + +### Calling Backend from Frontend + +Import from the auto-generated `wmill.ts`: + +```typescript +import { backend } from './wmill'; + +// Call a backend runnable +const user = await backend.get_user({ user_id: '123' }); +``` + +The `wmill.ts` file provides type-safe access to all backend runnables. + +## Data Tables + +Raw apps can query Windmill datatables (PostgreSQL databases managed by Windmill). + +### Critical Rules + +1. **ONLY USE WHITELISTED TABLES**: You can ONLY query tables listed in `raw_app.yaml` → `data.tables`. Tables not in this list are NOT accessible. + +2. **ADD TABLES BEFORE USING**: To use a new table, first add it to `data.tables` in `raw_app.yaml`. + +3. **USE CONFIGURED DATATABLE/SCHEMA**: Check the app's `raw_app.yaml` for the default datatable and schema. + +### Configuration in raw_app.yaml + +```yaml +data: + datatable: main # Default datatable + schema: app_schema # Default schema (optional) + tables: + - main/users # Table in public schema + - main/app_schema:items # Table in specific schema +``` + +**Table reference formats:** +- `` - All tables in the datatable +- `/
` - Specific table in public schema +- `/:
` - Table in specific schema + +### Querying in TypeScript (Bun/Deno) + +```typescript +import * as wmill from 'windmill-client'; + +export async function main(user_id: string) { + const sql = wmill.datatable(); // Or: wmill.datatable('other_datatable') + + // Parameterized queries (safe from SQL injection) + const user = await sql`SELECT * FROM users WHERE id = ${user_id}`.fetchOne(); + const users = await sql`SELECT * FROM users WHERE active = ${true}`.fetch(); + + // Insert/Update + await sql`INSERT INTO users (name, email) VALUES (${name}, ${email})`; + await sql`UPDATE users SET name = ${newName} WHERE id = ${user_id}`; + + return user; +} +``` + +### Querying in Python + +```python +import wmill + +def main(user_id: str): + db = wmill.datatable() # Or: wmill.datatable('other_datatable') + + # Use $1, $2, etc. for parameters + user = db.query('SELECT * FROM users WHERE id = $1', user_id).fetch_one() + users = db.query('SELECT * FROM users WHERE active = $1', True).fetch() + + # Insert/Update + db.query('INSERT INTO users (name, email) VALUES ($1, $2)', name, email) + db.query('UPDATE users SET name = $1 WHERE id = $2', new_name, user_id) + + return user +``` + +## SQL Migrations (sql_to_apply/) + +The `sql_to_apply/` folder is for creating/modifying database tables during development. + +### Workflow + +1. Create `.sql` files in `sql_to_apply/` +2. Run `wmill app dev` - the dev server watches this folder +3. When SQL files change, a modal appears in the browser to confirm execution +4. After creating tables, **add them to `data.tables`** in `raw_app.yaml` + +### Example Migration + +**sql_to_apply/001_create_users.sql:** +```sql +CREATE TABLE IF NOT EXISTS users ( + id SERIAL PRIMARY KEY, + email TEXT NOT NULL UNIQUE, + name TEXT, + created_at TIMESTAMP DEFAULT NOW() +); +``` + +After applying, add to `raw_app.yaml`: +```yaml +data: + tables: + - main/users +``` + +### Migration Best Practices + +- **Use idempotent SQL**: `CREATE TABLE IF NOT EXISTS`, etc. +- **Number files**: `001_`, `002_` for ordering +- **Always whitelist tables** after creation +- This folder is NOT synced - it's for local development only + +## CLI Commands + +| Command | Description | +|---------|-------------| +| `wmill app new` | Create a new raw app interactively | +| `wmill app dev` | Start dev server with live reload | +| `wmill app generate-agents` | Refresh AGENTS.md and DATATABLES.md | +| `wmill app generate-locks` | Generate lock files for backend runnables | +| `wmill sync push` | Deploy app to Windmill | +| `wmill sync pull` | Pull latest from Windmill | + +## Best Practices + +1. **Check DATATABLES.md** for existing tables before creating new ones +2. **Use parameterized queries** - never concatenate user input into SQL +3. **Keep runnables focused** - one function per file +4. **Use descriptive IDs** - `get_user.ts` not `a.ts` +5. **Always whitelist tables** - add to `data.tables` before querying +6. **Generate locks** - run `wmill app generate-locks` after adding/modifying backend runnables diff --git a/system_prompts/base/resources.md b/system_prompts/base/resources.md new file mode 100644 index 0000000000..290d6f617b --- /dev/null +++ b/system_prompts/base/resources.md @@ -0,0 +1,242 @@ +# Windmill Resources + +Resources store credentials and configuration for external services. + +## File Format + +Resource files use the pattern: `{path}.resource.json` + +Example: `f/databases/postgres_prod.resource.json` + +## Resource Structure + +```json +{ + "value": { + "host": "db.example.com", + "port": 5432, + "user": "admin", + "password": "$var:g/all/db_password", + "dbname": "production" + }, + "description": "Production PostgreSQL database", + "resource_type": "postgresql" +} +``` + +## Required Fields + +- `value` - Object containing the resource configuration +- `resource_type` - Name of the resource type (e.g., "postgresql", "slack") + +## Variable References + +Reference variables in resource values: + +```json +{ + "value": { + "api_key": "$var:g/all/api_key", + "secret": "$var:u/admin/secret" + } +} +``` + +**Reference formats:** +- `$var:g/all/name` - Global variable +- `$var:u/username/name` - User variable +- `$var:f/folder/name` - Folder variable + +## Resource References + +Reference other resources: + +```json +{ + "value": { + "database": "$res:f/databases/postgres" + } +} +``` + +## Common Resource Types + +### PostgreSQL +```json +{ + "resource_type": "postgresql", + "value": { + "host": "localhost", + "port": 5432, + "user": "postgres", + "password": "$var:g/all/pg_password", + "dbname": "windmill", + "sslmode": "prefer" + } +} +``` + +### MySQL +```json +{ + "resource_type": "mysql", + "value": { + "host": "localhost", + "port": 3306, + "user": "root", + "password": "$var:g/all/mysql_password", + "database": "myapp" + } +} +``` + +### Slack +```json +{ + "resource_type": "slack", + "value": { + "token": "$var:g/all/slack_token" + } +} +``` + +### AWS S3 +```json +{ + "resource_type": "s3", + "value": { + "bucket": "my-bucket", + "region": "us-east-1", + "accessKeyId": "$var:g/all/aws_access_key", + "secretAccessKey": "$var:g/all/aws_secret_key" + } +} +``` + +### HTTP/API +```json +{ + "resource_type": "http", + "value": { + "baseUrl": "https://api.example.com", + "headers": { + "Authorization": "Bearer $var:g/all/api_token" + } + } +} +``` + +### Kafka +```json +{ + "resource_type": "kafka", + "value": { + "brokers": "broker1:9092,broker2:9092", + "sasl_mechanism": "PLAIN", + "security_protocol": "SASL_SSL", + "username": "$var:g/all/kafka_user", + "password": "$var:g/all/kafka_password" + } +} +``` + +### NATS +```json +{ + "resource_type": "nats", + "value": { + "servers": ["nats://localhost:4222"], + "user": "$var:g/all/nats_user", + "password": "$var:g/all/nats_password" + } +} +``` + +### MQTT +```json +{ + "resource_type": "mqtt", + "value": { + "host": "mqtt.example.com", + "port": 8883, + "username": "$var:g/all/mqtt_user", + "password": "$var:g/all/mqtt_password", + "tls": true + } +} +``` + +## Custom Resource Types + +Create custom resource types with JSON Schema: + +```json +{ + "name": "custom_api", + "schema": { + "type": "object", + "properties": { + "base_url": {"type": "string", "format": "uri"}, + "api_key": {"type": "string"}, + "timeout": {"type": "integer", "default": 30} + }, + "required": ["base_url", "api_key"] + }, + "description": "Custom API connection" +} +``` + +Save as: `custom_api.resource-type.json` + +## OAuth Resources + +OAuth resources are managed through the Windmill UI and marked: + +```json +{ + "is_oauth": true, + "account": 123 +} +``` + +OAuth tokens are automatically refreshed by Windmill. + +## Using Resources in Scripts + +### TypeScript (Bun/Deno) +```typescript +export async function main(db: RT.Postgresql) { + // db contains the resource values + const { host, port, user, password, dbname } = db; +} +``` + +### Python +```python +class postgresql(TypedDict): + host: str + port: int + user: str + password: str + dbname: str + +def main(db: postgresql): + # db contains the resource values + pass +``` + +## CLI Commands + +```bash +# List resources +wmill resource list + +# List resource types with schemas +wmill resource-type list --schema + +# Get specific resource type schema +wmill resource-type get postgresql + +# Push resources +wmill sync push +``` diff --git a/system_prompts/base/schedules.md b/system_prompts/base/schedules.md new file mode 100644 index 0000000000..bf14d24cbd --- /dev/null +++ b/system_prompts/base/schedules.md @@ -0,0 +1,46 @@ +# Windmill Schedules + +Schedules run scripts and flows automatically on a cron schedule. + +## File Naming + +Schedule files use the pattern: `{path}.schedule.yaml` + +Example: `f/folder/daily_sync.schedule.yaml` + +Note: The path is derived from the filename, not stored in the file content. + +## Cron Expression Format + +Windmill uses 6-field cron expressions (includes seconds): + +``` + ┌───────────── second (0-59) + │ ┌───────────── minute (0-59) + │ │ ┌───────────── hour (0-23) + │ │ │ ┌───────────── day of month (1-31) + │ │ │ │ ┌───────────── month (1-12 or jan-dec) + │ │ │ │ │ ┌───────────── day of week (0-6, 0=Sunday, or sun-sat) + │ │ │ │ │ │ + * * * * * * +``` + +**Common Examples:** +- `0 0 0 * * *` - Daily at midnight +- `0 0 12 * * *` - Daily at noon +- `0 */5 * * * *` - Every 5 minutes +- `0 0 9 * * 1-5` - Weekdays at 9 AM +- `0 0 0 1 * *` - First day of each month + +## CLI Commands + +```bash +# Push schedules to Windmill +wmill sync push + +# Pull schedules from Windmill +wmill sync pull + +# List schedules +wmill schedule +``` diff --git a/system_prompts/base/triggers.md b/system_prompts/base/triggers.md new file mode 100644 index 0000000000..4998b85342 --- /dev/null +++ b/system_prompts/base/triggers.md @@ -0,0 +1,22 @@ +# Windmill Triggers + +Triggers allow external events to invoke your scripts and flows. + +## File Naming + +Trigger configuration files use the pattern: `{path}.{trigger_type}_trigger.yaml` + +Examples: +- `u/user/webhook.http_trigger.yaml` +- `f/data/kafka_consumer.kafka_trigger.yaml` +- `f/sync/postgres_cdc.postgres_trigger.yaml` + +## CLI Commands + +```bash +# Push trigger configuration +wmill sync push + +# Pull triggers from Windmill +wmill sync pull +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 03299c532d..0af7fe0eb6 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -20,103 +20,47 @@ from pathlib import Path import yaml -# Paths relative to this script -SCRIPT_DIR = Path(__file__).parent -ROOT_DIR = SCRIPT_DIR.parent - -TS_SDK_DIR = ROOT_DIR / "typescript-client" -PY_SDK_PATH = ROOT_DIR / "python-client" / "wmill" / "wmill" / "client.py" -OPENFLOW_SCHEMA_PATH = ROOT_DIR / "openflow.openapi.yaml" - -OUTPUT_SDKS_DIR = SCRIPT_DIR / "auto-generated" / "sdks" -OUTPUT_GENERATED_DIR = SCRIPT_DIR / "auto-generated" -OUTPUT_CLI_DIR = SCRIPT_DIR / "auto-generated" / "cli" - -# CLI guidance directory (DNT can't import from outside cli/, so we copy files there) -CLI_GUIDANCE_DIR = ROOT_DIR / "cli" / "src" / "guidance" - -# CLI source paths for extracting command documentation -CLI_DIR = ROOT_DIR / "cli" -CLI_MAIN = CLI_DIR / "src" / "main.ts" -CLI_COMMANDS_DIR = CLI_DIR / "src" / "commands" +from utils import ( + # Path constants + SCRIPT_DIR, + TS_SDK_DIR, + PY_SDK_PATH, + OPENFLOW_SCHEMA_PATH, + BACKEND_OPENAPI_PATH, + OUTPUT_SDKS_DIR, + OUTPUT_GENERATED_DIR, + OUTPUT_CLI_DIR, + OUTPUT_SKILLS_DIR, + OUTPUT_SCHEMAS_DIR, + CLI_GUIDANCE_DIR, + CLI_MAIN, + CLI_COMMANDS_DIR, + # Language metadata + LANGUAGE_METADATA, + TS_SDK_LANGUAGES, + PY_SDK_LANGUAGES, + # Schema mappings + SCHEMA_MAPPINGS, + # String/file utilities + clean_jsdoc, + clean_params, + escape_for_ts, + read_markdown_file, + # Parsing utilities + extract_balanced, + extract_return_type, + parse_default_imports, + extract_options, + # Schema utilities + extract_cli_schema, + format_schema_for_markdown, + format_schema_as_json, +) -def clean_jsdoc(jsdoc: str) -> str: - """Clean up JSDoc comment, removing delimiters and leading asterisks.""" - # Remove /** and */ - jsdoc = re.sub(r'^/\*\*\s*', '', jsdoc) - jsdoc = re.sub(r'\s*\*/$', '', jsdoc) - # Remove leading * from each line - lines = jsdoc.split('\n') - cleaned = [] - for line in lines: - line = re.sub(r'^\s*\*\s?', '', line) - cleaned.append(line) - return '\n'.join(cleaned).strip() - - -def extract_balanced(content: str, start_pos: int, open_char: str, close_char: str) -> tuple[str, int]: - """ - Extract content between balanced brackets starting at start_pos. - Returns (extracted_content, end_position) or ('', -1) if not found. - """ - if start_pos >= len(content) or content[start_pos] != open_char: - return '', -1 - - depth = 0 - i = start_pos - while i < len(content): - if content[i] == open_char: - depth += 1 - elif content[i] == close_char: - depth -= 1 - if depth == 0: - return content[start_pos + 1:i], i - i += 1 - return '', -1 - - -def extract_return_type(content: str, start_pos: int) -> tuple[str, int]: - """ - Extract return type from position after ')', handling nested braces. - Returns (return_type, end_position of function body open brace). - """ - i = start_pos - # Skip whitespace - while i < len(content) and content[i] in ' \t\n': - i += 1 - - if i >= len(content) or content[i] != ':': - # No return type, find opening brace - while i < len(content) and content[i] != '{': - i += 1 - return '', i - - i += 1 # Skip ':' - - # Now extract the return type, handling nested braces and angle brackets - return_type_start = i - brace_depth = 0 - angle_depth = 0 - - while i < len(content): - char = content[i] - if char == '<': - angle_depth += 1 - elif char == '>': - angle_depth -= 1 - elif char == '{': - if angle_depth > 0: - # Inside a type like Promise<{...}> - brace_depth += 1 - else: - # This is the function body opening brace - return content[return_type_start:i].strip(), i - elif char == '}': - brace_depth -= 1 - i += 1 - - return '', -1 +# ============================================================================= +# TypeScript SDK Parsing +# ============================================================================= def extract_ts_functions(content: str) -> list[dict]: @@ -124,24 +68,16 @@ def extract_ts_functions(content: str) -> list[dict]: functions = [] seen_names = set() - # Pattern to find JSDoc followed by export function declaration - # Only captures up to the function name and optional generic - jsdoc_func_pattern = re.compile( - r'(/\*\*(?:[^*]|\*(?!/))*\*/)\s*' # JSDoc comment + # Pattern to find export function declarations (with or without JSDoc) + # Captures JSDoc if present, then the function declaration + pattern = re.compile( + r'(?:(/\*\*(?:[^*]|\*(?!/))*\*/)\s*)?' # Optional JSDoc comment r'export\s+(async\s+)?function\s+(\w+)\s*' # export [async] function name r'(<[^>]+>)?\s*', # optional generic re.MULTILINE ) - # Pattern to find export function without JSDoc - func_pattern = re.compile( - r'export\s+(async\s+)?function\s+(\w+)\s*' # export [async] function name - r'(<[^>]+>)?\s*', # optional generic - re.MULTILINE - ) - - # First, find all functions with JSDoc - for match in jsdoc_func_pattern.finditer(content): + for match in pattern.finditer(content): jsdoc_raw, is_async, name, generic = match.groups() if name in seen_names: @@ -166,7 +102,7 @@ def extract_ts_functions(content: str) -> list[dict]: if not return_type: return_type = 'Promise' if is_async else 'void' - docstring = clean_jsdoc(jsdoc_raw) + docstring = clean_jsdoc(jsdoc_raw) if jsdoc_raw else '' seen_names.add(name) functions.append({ 'name': name, @@ -177,54 +113,9 @@ def extract_ts_functions(content: str) -> list[dict]: 'docstring': docstring }) - # Then find functions without JSDoc (that weren't already captured) - for match in func_pattern.finditer(content): - is_async, name, generic = match.groups() - - if name in seen_names: - continue - - # Find the opening parenthesis for parameters - pos = match.end() - while pos < len(content) and content[pos] in ' \t\n': - pos += 1 - - if pos >= len(content) or content[pos] != '(': - continue - - # Extract balanced parameters - params, paren_end = extract_balanced(content, pos, '(', ')') - if paren_end == -1: - continue - - # Extract return type (handles multi-line types like Promise<{...}>) - return_type, _ = extract_return_type(content, paren_end + 1) - - if not return_type: - return_type = 'Promise' if is_async else 'void' - - seen_names.add(name) - functions.append({ - 'name': name, - 'generic': generic or '', - 'params': clean_params(params), - 'return_type': return_type, - 'async': bool(is_async), - 'docstring': '' - }) - return functions -def clean_params(params: str) -> str: - """Clean up parameter string.""" - if not params: - return '' - # Remove excessive whitespace and newlines - params = re.sub(r'\s+', ' ', params).strip() - return params - - def extract_ts_types(content: str) -> list[dict]: """Extract exported type definitions from TypeScript SDK.""" types = [] @@ -260,6 +151,11 @@ def extract_ts_types(content: str) -> list[dict]: return types +# ============================================================================= +# Python SDK Parsing +# ============================================================================= + + def extract_py_functions(content: str) -> list[dict]: """Extract function signatures from Python SDK using AST.""" functions = [] @@ -377,10 +273,27 @@ def extract_py_classes(content: str) -> list[dict]: return classes -def parse_command_block(content: str) -> dict: +# ============================================================================= +# CLI Command Parsing +# ============================================================================= + + +# Reusable option pattern for CLI parsing +OPTION_PATTERN = re.compile( + r'\.option\(\s*"([^"]+)"\s*,\s*"([^"]+)"' # double-quoted + r'|' + r"\.option\(\s*'([^']+)'\s*,\s*'([^']+)'", # single-quoted + re.MULTILINE | re.DOTALL +) + + +def parse_command_block(content: str, file_path: Path | None = None) -> dict: """ Parse a Cliffy Command() definition block and extract metadata. Returns a dict with: description, options, subcommands, arguments, alias + + If file_path is provided, imported subcommands will be resolved by parsing + the imported files. """ result = { 'description': '', @@ -390,8 +303,7 @@ def parse_command_block(content: str) -> dict: 'alias': '' } - # Find the command block - starts with "new Command()" or "const command = new Command()" - # and ends with "export default" + # Find the command block command_match = re.search( r'(?:const\s+command\s*=\s*)?new\s+Command\(\)([\s\S]*?)(?=export\s+default)', content @@ -401,17 +313,15 @@ def parse_command_block(content: str) -> dict: block = command_match.group(1) - # Extract main description (first one in the block, before any subcommand) + # Find where subcommands start first_subcommand_pos = block.find('.command(') if first_subcommand_pos == -1: first_subcommand_pos = len(block) top_section = block[:first_subcommand_pos] - # Handle multi-line descriptions with template literals or string concatenation - # Pattern: .description("text") or .description(\n "text",\n) + # Extract main description desc_match = re.search(r'\.description\(\s*["\']([^"\']+)["\']\s*,?\s*\)', top_section, re.DOTALL) if not desc_match: - # Try matching descriptions that use template literals desc_match = re.search(r'\.description\(\s*`([^`]+)`\s*,?\s*\)', top_section, re.DOTALL) if desc_match: result['description'] = desc_match.group(1).strip() @@ -421,73 +331,84 @@ def parse_command_block(content: str) -> dict: if alias_match: result['alias'] = alias_match.group(1) - # Extract options pattern - option_pattern = re.compile( - r'\.option\(\s*["\']([^"\']+)["\']\s*,\s*["\']([^"\']+)["\']\s*\)', - re.MULTILINE - ) - # Extract top-level options (before any .command() or .action()) top_section_until_action = re.split(r'\.action\(', top_section)[0] - for match in option_pattern.finditer(top_section_until_action): - flag, desc = match.groups() - result['options'].append({'flag': flag, 'description': desc}) + result['options'] = extract_options(top_section_until_action, OPTION_PATTERN) # Extract top-level arguments args_match = re.search(r'\.arguments\(\s*["\']([^"\']+)["\']\s*\)', top_section) if args_match: result['arguments'] = args_match.group(1) - # Extract subcommands with their arguments, options, and descriptions - # Split by .command( to find subcommand boundaries + # Parse imports if we have a file path (for resolving imported subcommands) + imports = parse_default_imports(content) if file_path else {} + + # Extract subcommands subcommand_sections = re.split(r'(?=\.command\()', block) for section in subcommand_sections: - # Check if this starts a new subcommand - cmd_match = re.match(r'\.command\(\s*["\']([^"\']+)["\']\s*(?:,\s*["\']([^"\']+)["\'])?\s*\)', section) - if cmd_match: - cmd_name = cmd_match.group(1) - cmd_desc = cmd_match.group(2) or '' + cmd_match = re.match(r'\.command\(\s*["\']([^"\']+)["\']\s*(?:,\s*([^)]+))?\s*\)', section) + if not cmd_match: + continue - # Check for description in chained .description() call - # Handle multi-line descriptions - desc_match = re.search(r'\.description\(\s*["\']([^"\']+)["\']\s*,?\s*\)', section, re.DOTALL) - if desc_match: - cmd_desc = desc_match.group(1).strip() + cmd_name = cmd_match.group(1) + second_arg = cmd_match.group(2).strip() if cmd_match.group(2) else '' - # Check for arguments - args_match = re.search(r'\.arguments\(\s*["\']([^"\']+)["\']\s*\)', section) - cmd_args = args_match.group(1) if args_match else '' + # Check if second arg is a string (description) or a variable (imported command) + is_string_desc = second_arg.startswith('"') or second_arg.startswith("'") - # Check for options specific to this subcommand - # Only get options that appear before .action() - cmd_options = [] - section_until_action = re.split(r'\.action\(', section)[0] - for opt_match in option_pattern.finditer(section_until_action): - flag, desc = opt_match.groups() - cmd_options.append({'flag': flag, 'description': desc}) + if is_string_desc: + cmd_desc = second_arg.strip('"\'') + elif second_arg and second_arg in imports and file_path: + # Imported command - resolve and parse the imported file + import_path = imports[second_arg] + if import_path.startswith('./') or import_path.startswith('../'): + imported_file = (file_path.parent / import_path).resolve() + if imported_file.exists(): + try: + imported_content = imported_file.read_text() + imported_cmd = parse_command_block(imported_content, imported_file) + result['subcommands'].append({ + 'name': cmd_name, + 'description': imported_cmd.get('description', ''), + 'arguments': imported_cmd.get('arguments', ''), + 'options': imported_cmd.get('options', []) + }) + continue + except Exception as e: + print(f" Warning: Could not parse imported command {second_arg}: {e}") + cmd_desc = '' + else: + cmd_desc = '' - result['subcommands'].append({ - 'name': cmd_name, - 'description': cmd_desc, - 'arguments': cmd_args, - 'options': cmd_options - }) + # Check for description in chained .description() call + desc_match = re.search(r'\.description\(\s*["\']([^"\']+)["\']\s*,?\s*\)', section, re.DOTALL) + if desc_match: + cmd_desc = desc_match.group(1).strip() + + # Check for arguments + args_match = re.search(r'\.arguments\(\s*["\']([^"\']+)["\']\s*\)', section) + cmd_args = args_match.group(1) if args_match else '' + + # Extract options specific to this subcommand (before .action()) + section_until_action = re.split(r'\.action\(', section)[0] + cmd_options = extract_options(section_until_action, OPTION_PATTERN) + + result['subcommands'].append({ + 'name': cmd_name, + 'description': cmd_desc, + 'arguments': cmd_args, + 'options': cmd_options + }) return result def find_command_file(cmd_name: str) -> Path | None: - """Find the command file for a given command name. - - Convention: directory name should match main command file name. - E.g., flow/flow.ts, app/app.ts, worker-groups/worker-groups.ts - """ - # Standard pattern: command-name/command-name.ts + """Find the command file for a given command name.""" standard_path = CLI_COMMANDS_DIR / cmd_name / f"{cmd_name}.ts" if standard_path.exists(): return standard_path - return None @@ -523,13 +444,11 @@ def extract_cli_commands() -> dict: result['global_options'].append({'flag': flag, 'description': desc}) # Extract command registrations from main.ts - # Pattern: .command("name", importedCommand) or .command("name with desc", ...) cmd_reg_pattern = re.compile( r'\.command\(\s*["\']([^"\']+)["\']\s*,\s*(\w+)\s*\)', re.MULTILINE ) - # Also handle inline commands like .command("version --version", "description") inline_cmd_pattern = re.compile( r'\.command\(\s*["\']([^"\']+)["\']\s*,\s*["\']([^"\']+)["\']\s*\)', re.MULTILINE @@ -538,7 +457,7 @@ def extract_cli_commands() -> dict: registered_commands = [] for match in cmd_reg_pattern.finditer(main_content): - cmd_name = match.group(1).split()[0] # Get just the name, not flags + cmd_name = match.group(1).split()[0] registered_commands.append(cmd_name) # Process each registered command @@ -547,14 +466,11 @@ def extract_cli_commands() -> dict: if cmd_file: try: cmd_content = cmd_file.read_text() - cmd_data = parse_command_block(cmd_content) + cmd_data = parse_command_block(cmd_content, cmd_file) cmd_data['name'] = cmd_name result['commands'].append(cmd_data) except Exception as e: print(f"Warning: Could not parse command file for {cmd_name}: {e}") - else: - # Some commands might be inline (like 'version', 'upgrade', 'completions') - pass # Handle special inline commands from main.ts for match in inline_cmd_pattern.finditer(main_content): @@ -573,6 +489,11 @@ def extract_cli_commands() -> dict: return result +# ============================================================================= +# Markdown Generation +# ============================================================================= + + def generate_cli_commands_markdown(cli_data: dict) -> str: """Generate markdown documentation from extracted CLI command data.""" md = "# Windmill CLI Commands\n\n" @@ -636,7 +557,7 @@ def generate_cli_commands_markdown(cli_data: dict) -> str: return md -def generate_ts_sdk_markdown(functions: list[dict], types: list[dict]) -> str: +def generate_ts_sdk_markdown(functions: list[dict], _types: list[dict]) -> str: """Generate compact documentation for TypeScript SDK.""" md = "# TypeScript SDK (windmill-client)\n\n" md += "Import: import * as wmill from 'windmill-client'\n\n" @@ -658,7 +579,7 @@ def generate_ts_sdk_markdown(functions: list[dict], types: list[dict]) -> str: return md -def generate_py_sdk_markdown(functions: list[dict], classes: list[dict]) -> str: +def generate_py_sdk_markdown(functions: list[dict], _classes: list[dict]) -> str: """Generate compact documentation for Python SDK.""" md = "# Python SDK (wmill)\n\n" md += "Import: import wmill\n\n" @@ -681,11 +602,6 @@ def generate_py_sdk_markdown(functions: list[dict], classes: list[dict]) -> str: return md -def escape_for_ts(content: str) -> str: - """Escape content for TypeScript template literal.""" - return content.replace('\\', '\\\\').replace('`', '\\`').replace('${', '\\${') - - def generate_ts_exports(prompts: dict[str, str]) -> str: """Generate TypeScript file that exports all prompts.""" ts = "// Auto-generated by generate.py - DO NOT EDIT\n\n" @@ -697,11 +613,300 @@ def generate_ts_exports(prompts: dict[str, str]) -> str: return ts -def read_markdown_file(path: Path) -> str: - """Read a markdown file and return its content.""" - if path.exists(): - return path.read_text() - return '' +# ============================================================================= +# Schema File Generation +# ============================================================================= + + +def generate_schema_files(cli_schemas: dict[str, dict]) -> dict[str, str]: + """ + Generate standalone YAML schema files for triggers and schedules. + + Returns a dict mapping schema keys (e.g., 'http_trigger') to YAML content. + """ + print("Generating standalone schema files...") + + # Ensure schemas directory exists + OUTPUT_SCHEMAS_DIR.mkdir(parents=True, exist_ok=True) + + schema_yaml_content = {} + + # Collect all schema types from SCHEMA_MAPPINGS + for skill_name, schema_types in SCHEMA_MAPPINGS.items(): + for schema_name, file_suffix in schema_types: + if schema_name not in cli_schemas: + print(f" Warning: Schema '{schema_name}' not found, skipping") + continue + + # Convert the schema to JSON Schema format + json_schema = format_schema_as_json(cli_schemas[schema_name]) + if not json_schema: + print(f" Warning: Empty schema for '{schema_name}', skipping") + continue + + # Convert to YAML + schema_yaml = yaml.dump(json_schema, default_flow_style=False, sort_keys=False, allow_unicode=True) + + # Write to file + schema_file = OUTPUT_SCHEMAS_DIR / f"{file_suffix}.schema.yaml" + schema_file.write_text(schema_yaml) + + # Store for return + schema_yaml_content[file_suffix] = schema_yaml + + print(f" Generated {len(schema_yaml_content)} schema files") + return schema_yaml_content + + +# ============================================================================= +# Skill Generation +# ============================================================================= + + +def generate_skill_content( + skill_name: str, + description: str, + intro: str, + content: str, + sdk_content: str = '' +) -> str: + """Generate a skill file with YAML frontmatter.""" + parts = [ + "---", + f"name: {skill_name}", + f"description: {description}", + "---", + "", + ] + if intro: + parts.extend([intro, ""]) + parts.append(content) + if sdk_content: + parts.extend(["", sdk_content]) + return '\n'.join(parts) + + +# Skill definitions for config-driven generation +SKILL_DEFINITIONS = [ + { + 'name': 'write-flow', + 'description': 'MUST use when creating flows.', + 'content_key': 'flow', + }, + { + 'name': 'raw-app', + 'description': 'MUST use when creating raw apps.', + 'content_key': 'raw_app', + }, + { + 'name': 'triggers', + 'description': 'MUST use when configuring triggers.', + 'content_key': 'triggers', + 'schema_types': [ + ('HttpTrigger', 'http_trigger'), + ('WebsocketTrigger', 'websocket_trigger'), + ('KafkaTrigger', 'kafka_trigger'), + ('NatsTrigger', 'nats_trigger'), + ('PostgresTrigger', 'postgres_trigger'), + ('MqttTrigger', 'mqtt_trigger'), + ('SqsTrigger', 'sqs_trigger'), + ('GcpTrigger', 'gcp_trigger'), + ], + }, + { + 'name': 'schedules', + 'description': 'MUST use when configuring schedules.', + 'content_key': 'schedules', + 'schema_types': [('Schedule', 'schedule')], + }, + { + 'name': 'resources', + 'description': 'MUST use when managing resources.', + 'content_key': 'resources', + }, + { + 'name': 'cli-commands', + 'description': 'MUST use when using the CLI.', + 'content_key': 'cli_commands', + }, +] + + +def generate_skills( + languages: dict[str, str], + ts_sdk_md: str, + py_sdk_md: str, + flow_base: str, + openflow_content: str, + cli_commands: str, + cli_schemas: dict[str, dict] | None = None +): + """Generate individual skill files for Claude Code.""" + print("Generating skill files...") + + cli_schemas = cli_schemas or {} + + # Ensure skills directory exists + OUTPUT_SKILLS_DIR.mkdir(parents=True, exist_ok=True) + + # Read base files for additional skills + base_dir = SCRIPT_DIR / "base" + base_content = { + 'flow': f"{flow_base}\n\n{openflow_content}", + 'raw_app': read_markdown_file(base_dir / "raw-app.md"), + 'triggers': read_markdown_file(base_dir / "triggers.md"), + 'schedules': read_markdown_file(base_dir / "schedules.md"), + 'resources': read_markdown_file(base_dir / "resources.md"), + 'cli_commands': cli_commands, + } + + # CLI intro for script skills + script_cli_intro = """## CLI Commands + +Place scripts in a folder. After writing, run: +- `wmill script generate-metadata` - Generate .script.yaml and .lock files +- `wmill sync push` - Deploy to Windmill + +Use `wmill resource-type list --schema` to discover available resource types.""" + + skills_generated = [] + + # Generate script skills for each language + for lang_key, lang_content in languages.items(): + if lang_key not in LANGUAGE_METADATA: + print(f" Warning: No metadata for language '{lang_key}', skipping") + continue + + metadata = LANGUAGE_METADATA[lang_key] + skill_name = f"write-script-{lang_key}" + skill_dir = OUTPUT_SKILLS_DIR / skill_name + skill_dir.mkdir(parents=True, exist_ok=True) + + # Determine which SDK to include + sdk_content = '' + if lang_key in TS_SDK_LANGUAGES: + sdk_content = ts_sdk_md + elif lang_key in PY_SDK_LANGUAGES: + sdk_content = py_sdk_md + + skill_content = generate_skill_content( + skill_name=skill_name, + description=metadata['description'], + intro=script_cli_intro, + content=lang_content, + sdk_content=sdk_content + ) + + (skill_dir / "SKILL.md").write_text(skill_content) + skills_generated.append(skill_name) + + # Generate other skills from definitions + # Note: Skills with schema_types (triggers, schedules) get base content only. + # Schemas are stored separately and combined at CLI init time. + for skill_def in SKILL_DEFINITIONS: + content = base_content.get(skill_def['content_key'], '') + if not content: + continue + + skill_name = skill_def['name'] + skill_dir = OUTPUT_SKILLS_DIR / skill_name + skill_dir.mkdir(parents=True, exist_ok=True) + + # Note: We no longer append schemas here. Skills with 'schema_types' + # will have schemas combined at CLI init time from SCHEMAS export. + + skill_content = generate_skill_content( + skill_name=skill_name, + description=skill_def['description'], + intro="", + content=content + ) + + (skill_dir / "SKILL.md").write_text(skill_content) + skills_generated.append(skill_name) + + print(f" Generated {len(skills_generated)} skills") + return skills_generated + + +def generate_skills_ts_export(skills: list[str], schema_yaml_content: dict[str, str] | None = None) -> str: + """Generate TypeScript file that exports skill metadata for the CLI. + + Args: + skills: List of skill names + schema_yaml_content: Dict mapping schema keys (e.g., 'http_trigger') to YAML content + """ + schema_yaml_content = schema_yaml_content or {} + + ts = "// Auto-generated by generate.py - DO NOT EDIT\n\n" + ts += "export interface SkillMetadata {\n" + ts += " name: string;\n" + ts += " description: string;\n" + ts += " languageKey?: string;\n" + ts += "}\n\n" + + ts += "export const SKILLS: SkillMetadata[] = [\n" + + skill_desc_map = {s['name']: s['description'] for s in SKILL_DEFINITIONS} + + for skill in skills: + if skill.startswith('write-script-'): + lang_key = skill.replace('write-script-', '') + if lang_key in LANGUAGE_METADATA: + metadata = LANGUAGE_METADATA[lang_key] + ts += f' {{ name: "{skill}", description: "{metadata["description"]}", languageKey: "{lang_key}" }},\n' + elif skill in skill_desc_map: + ts += f' {{ name: "{skill}", description: "{skill_desc_map[skill]}" }},\n' + + ts += "];\n\n" + + # Generate the skills content inline for bundling + ts += "// Skill content for each skill (loaded inline for bundling)\n" + ts += "export const SKILL_CONTENT: Record = {\n" + + # We'll read the generated files and embed them + for skill in skills: + skill_path = OUTPUT_SKILLS_DIR / skill / "SKILL.md" + if skill_path.exists(): + content = skill_path.read_text() + escaped = escape_for_ts(content) + ts += f' "{skill}": `{escaped}`,\n' + + ts += "};\n\n" + + # Generate SCHEMAS export (YAML content for each schema) + ts += "// YAML schema content for triggers and schedules\n" + ts += "export const SCHEMAS: Record = {\n" + + for schema_key, yaml_content in sorted(schema_yaml_content.items()): + escaped = escape_for_ts(yaml_content) + ts += f' "{schema_key}": `{escaped}`,\n' + + ts += "};\n\n" + + # Generate SCHEMA_MAPPINGS export (maps skill names to their schemas) + ts += "// Maps skill names to their schema types and file patterns\n" + ts += "export interface SchemaMapping {\n" + ts += " name: string;\n" + ts += " schemaKey: string;\n" + ts += " filePattern: string;\n" + ts += "}\n\n" + ts += "export const SCHEMA_MAPPINGS: Record = {\n" + + for skill_name, schema_types in SCHEMA_MAPPINGS.items(): + ts += f' "{skill_name}": [\n' + for schema_name, file_suffix in schema_types: + ts += f' {{ name: "{schema_name}", schemaKey: "{file_suffix}", filePattern: "*.{file_suffix}.yaml" }},\n' + ts += " ],\n" + + ts += "};\n" + + return ts + + +# ============================================================================= +# Main Entry Point +# ============================================================================= def main(): @@ -764,6 +969,37 @@ def main(): (OUTPUT_CLI_DIR / "cli-commands.md").write_text(cli_commands) print(f" Found {len(cli_data['commands'])} commands, {len(cli_data['global_options'])} global options") + # Extract schemas from backend OpenAPI for CLI format documentation + print("Extracting backend OpenAPI schemas...") + cli_schemas = {} + if BACKEND_OPENAPI_PATH.exists(): + backend_openapi_raw = BACKEND_OPENAPI_PATH.read_text() + backend_openapi = yaml.safe_load(backend_openapi_raw) + backend_schemas = backend_openapi.get('components', {}).get('schemas', {}) + + # Extract and transform schemas for CLI format (removing server-managed fields) + schema_names = [ + 'Schedule', 'NewSchedule', + 'HttpTrigger', 'NewHttpTrigger', + 'WebsocketTrigger', 'NewWebsocketTrigger', + 'KafkaTrigger', 'NewKafkaTrigger', + 'NatsTrigger', 'NewNatsTrigger', + 'PostgresTrigger', 'NewPostgresTrigger', + 'MqttTrigger', 'NewMqttTrigger', + 'SqsTrigger', 'NewSqsTrigger', + 'GcpTrigger', + ] + for schema_name in schema_names: + if schema_name in backend_schemas: + cli_schemas[schema_name] = extract_cli_schema(backend_schemas[schema_name], backend_schemas, openflow_schemas) + + print(f" Extracted {len(cli_schemas)} schemas for CLI format") + else: + print(f" Warning: Backend OpenAPI file not found at {BACKEND_OPENAPI_PATH}") + + # Generate standalone schema files for triggers and schedules + schema_yaml_content = generate_schema_files(cli_schemas) + # Assemble prompts for export prompts = { # Base prompts @@ -845,16 +1081,21 @@ export function getFlowPrompt(): string { """ (OUTPUT_GENERATED_DIR / "index.ts").write_text(index_content) - # Generate CLI-specific prompts.ts with only SCRIPT_PROMPT and FLOW_PROMPT - print("Generating CLI prompts...") + # Generate skill files for Claude Code CLI_GUIDANCE_DIR.mkdir(parents=True, exist_ok=True) - cli_prompts = { - 'SCRIPT_PROMPT': script_md, - 'FLOW_PROMPT': flow_md, - 'CLI_COMMANDS': cli_commands, - } - cli_prompts_ts = generate_ts_exports(cli_prompts) - (CLI_GUIDANCE_DIR / "prompts.ts").write_text(cli_prompts_ts) + skills = generate_skills( + languages=languages, + ts_sdk_md=ts_sdk_md, + py_sdk_md=py_sdk_md, + flow_base=flow_base, + cli_commands=cli_commands, + openflow_content=openflow_content, + cli_schemas=cli_schemas + ) + + # Generate skills TypeScript export for CLI + skills_ts = generate_skills_ts_export(skills, schema_yaml_content) + (CLI_GUIDANCE_DIR / "skills.ts").write_text(skills_ts) print(f"\nGenerated files:") print(f" - auto-generated/sdks/typescript.md") @@ -864,8 +1105,10 @@ export function getFlowPrompt(): string { print(f" - auto-generated/index.ts") print(f" - auto-generated/script.md") print(f" - auto-generated/flow.md") + print(f" - auto-generated/skills/ ({len(skills)} skills)") + print(f" - auto-generated/schemas/ ({len(schema_yaml_content)} schema files)") print(f"\nGenerated for CLI:") - print(f" - cli/src/guidance/prompts.ts") + print(f" - cli/src/guidance/skills.ts") print("\nDone!") diff --git a/system_prompts/utils.py b/system_prompts/utils.py new file mode 100644 index 0000000000..ee77e20c13 --- /dev/null +++ b/system_prompts/utils.py @@ -0,0 +1,496 @@ +""" +Utility functions and constants for system prompts generation. + +This module contains: +- Path constants for SDK files and output directories +- Field exclusion lists for CLI format +- Language metadata for script generation +- String/file utility functions +- Parsing utilities for TypeScript/Python code +- Schema transformation utilities +""" + +import json +import re +from pathlib import Path + +# ============================================================================= +# Path Constants +# ============================================================================= + +SCRIPT_DIR = Path(__file__).parent +ROOT_DIR = SCRIPT_DIR.parent + +TS_SDK_DIR = ROOT_DIR / "typescript-client" +PY_SDK_PATH = ROOT_DIR / "python-client" / "wmill" / "wmill" / "client.py" +OPENFLOW_SCHEMA_PATH = ROOT_DIR / "openflow.openapi.yaml" +BACKEND_OPENAPI_PATH = ROOT_DIR / "backend" / "windmill-api" / "openapi.yaml" + +OUTPUT_SDKS_DIR = SCRIPT_DIR / "auto-generated" / "sdks" +OUTPUT_GENERATED_DIR = SCRIPT_DIR / "auto-generated" +OUTPUT_CLI_DIR = SCRIPT_DIR / "auto-generated" / "cli" +OUTPUT_SKILLS_DIR = SCRIPT_DIR / "auto-generated" / "skills" +OUTPUT_SCHEMAS_DIR = SCRIPT_DIR / "auto-generated" / "schemas" + +# ============================================================================= +# Schema Mappings for Triggers and Schedules +# ============================================================================= + +# Maps skill names to their schema types and file patterns +# Used for generating standalone schema files and combining at CLI init time +SCHEMA_MAPPINGS = { + 'triggers': [ + ('HttpTrigger', 'http_trigger'), + ('WebsocketTrigger', 'websocket_trigger'), + ('KafkaTrigger', 'kafka_trigger'), + ('NatsTrigger', 'nats_trigger'), + ('PostgresTrigger', 'postgres_trigger'), + ('MqttTrigger', 'mqtt_trigger'), + ('SqsTrigger', 'sqs_trigger'), + ('GcpTrigger', 'gcp_trigger'), + ], + 'schedules': [ + ('Schedule', 'schedule'), + ], +} + +# CLI guidance directory (DNT can't import from outside cli/, so we copy files there) +CLI_GUIDANCE_DIR = ROOT_DIR / "cli" / "src" / "guidance" + +# CLI source paths for extracting command documentation +CLI_DIR = ROOT_DIR / "cli" +CLI_MAIN = CLI_DIR / "src" / "main.ts" +CLI_COMMANDS_DIR = CLI_DIR / "src" / "commands" + +# ============================================================================= +# Field Exclusion Lists +# ============================================================================= + +# Fields stripped by CLI/sync format (to_string_without_metadata equivalent) +# These are server-managed fields that don't appear in YAML/JSON files pulled via CLI +CLI_EXCLUDED_FIELDS = [ + 'workspace_id', 'path', 'name', 'versions', 'id', + 'created_at', 'updated_at', 'created_by', 'updated_by', + 'edited_at', 'edited_by', 'archived', 'has_draft', + 'error', 'last_server_ping', 'server_id', + 'extra_perms', 'email', 'mode' +] + +# ============================================================================= +# Language Metadata +# ============================================================================= + +# Mapping of language file names to friendly names and descriptions +LANGUAGE_METADATA = { + 'bun': { + 'name': 'TypeScript (Bun)', + 'description': 'MUST use when writing Bun/TypeScript scripts.', + 'use_cases': 'TypeScript automation, npm packages, data processing, API integrations' + }, + 'deno': { + 'name': 'TypeScript (Deno)', + 'description': 'MUST use when writing Deno/TypeScript scripts.', + 'use_cases': 'TypeScript with Deno stdlib, secure sandboxed execution' + }, + 'nativets': { + 'name': 'Native TypeScript', + 'description': 'MUST use when writing Native TypeScript scripts.', + 'use_cases': 'simple API calls, lightweight TypeScript, no dependencies' + }, + 'bunnative': { + 'name': 'Bun Native', + 'description': 'MUST use when writing Bun Native scripts.', + 'use_cases': 'simple Bun scripts, lightweight, no dependencies' + }, + 'python3': { + 'name': 'Python', + 'description': 'MUST use when writing Python scripts.', + 'use_cases': 'Python automation, data processing, machine learning, scripting' + }, + 'bash': { + 'name': 'Bash', + 'description': 'MUST use when writing Bash scripts.', + 'use_cases': 'shell scripts, system administration, CLI tools' + }, + 'go': { + 'name': 'Go', + 'description': 'MUST use when writing Go scripts.', + 'use_cases': 'Go automation, high performance, concurrent processing' + }, + 'rust': { + 'name': 'Rust', + 'description': 'MUST use when writing Rust scripts.', + 'use_cases': 'Rust automation, high performance, memory safety' + }, + 'postgresql': { + 'name': 'PostgreSQL', + 'description': 'MUST use when writing PostgreSQL queries.', + 'use_cases': 'PostgreSQL database queries, data analysis' + }, + 'mysql': { + 'name': 'MySQL', + 'description': 'MUST use when writing MySQL queries.', + 'use_cases': 'MySQL database queries, data operations' + }, + 'mssql': { + 'name': 'MS SQL Server', + 'description': 'MUST use when writing MS SQL Server queries.', + 'use_cases': 'SQL Server database queries, enterprise data' + }, + 'bigquery': { + 'name': 'BigQuery', + 'description': 'MUST use when writing BigQuery queries.', + 'use_cases': 'BigQuery analytics, large-scale data analysis' + }, + 'snowflake': { + 'name': 'Snowflake', + 'description': 'MUST use when writing Snowflake queries.', + 'use_cases': 'Snowflake data warehouse queries, analytics' + }, + 'duckdb': { + 'name': 'DuckDB', + 'description': 'MUST use when writing DuckDB queries.', + 'use_cases': 'DuckDB analytics, local data processing, Ducklake' + }, + 'graphql': { + 'name': 'GraphQL', + 'description': 'MUST use when writing GraphQL queries.', + 'use_cases': 'GraphQL API calls, federated queries' + }, + 'php': { + 'name': 'PHP', + 'description': 'MUST use when writing PHP scripts.', + 'use_cases': 'PHP automation, web integrations' + }, + 'powershell': { + 'name': 'PowerShell', + 'description': 'MUST use when writing PowerShell scripts.', + 'use_cases': 'Windows automation, system administration' + }, + 'csharp': { + 'name': 'C#', + 'description': 'MUST use when writing C# scripts.', + 'use_cases': 'C# automation, .NET integrations' + }, + 'java': { + 'name': 'Java', + 'description': 'MUST use when writing Java scripts.', + 'use_cases': 'Java automation, enterprise integrations' + }, +} + +# Languages that use TypeScript SDK +TS_SDK_LANGUAGES = ['bun', 'deno', 'nativets', 'bunnative'] + +# Languages that use Python SDK +PY_SDK_LANGUAGES = ['python3'] + +# ============================================================================= +# String/File Utilities +# ============================================================================= + + +def clean_jsdoc(jsdoc: str) -> str: + """Clean up JSDoc comment, removing delimiters and leading asterisks.""" + # Remove /** and */ + jsdoc = re.sub(r'^/\*\*\s*', '', jsdoc) + jsdoc = re.sub(r'\s*\*/$', '', jsdoc) + # Remove leading * from each line + lines = jsdoc.split('\n') + cleaned = [] + for line in lines: + line = re.sub(r'^\s*\*\s?', '', line) + cleaned.append(line) + return '\n'.join(cleaned).strip() + + +def clean_params(params: str) -> str: + """Clean up parameter string.""" + if not params: + return '' + # Remove excessive whitespace and newlines + params = re.sub(r'\s+', ' ', params).strip() + return params + + +def escape_for_ts(content: str) -> str: + """Escape content for TypeScript template literal.""" + return content.replace('\\', '\\\\').replace('`', '\\`').replace('${', '\\${') + + +def read_markdown_file(path: Path) -> str: + """Read a markdown file and return its content.""" + if path.exists(): + return path.read_text() + return '' + + +# ============================================================================= +# Parsing Utilities +# ============================================================================= + + +def extract_balanced(content: str, start_pos: int, open_char: str, close_char: str) -> tuple[str, int]: + """ + Extract content between balanced brackets starting at start_pos. + Returns (extracted_content, end_position) or ('', -1) if not found. + """ + if start_pos >= len(content) or content[start_pos] != open_char: + return '', -1 + + depth = 0 + i = start_pos + while i < len(content): + if content[i] == open_char: + depth += 1 + elif content[i] == close_char: + depth -= 1 + if depth == 0: + return content[start_pos + 1:i], i + i += 1 + return '', -1 + + +def extract_return_type(content: str, start_pos: int) -> tuple[str, int]: + """ + Extract return type from position after ')', handling nested braces. + Returns (return_type, end_position of function body open brace). + """ + i = start_pos + # Skip whitespace + while i < len(content) and content[i] in ' \t\n': + i += 1 + + if i >= len(content) or content[i] != ':': + # No return type, find opening brace + while i < len(content) and content[i] != '{': + i += 1 + return '', i + + i += 1 # Skip ':' + + # Now extract the return type, handling nested braces and angle brackets + return_type_start = i + brace_depth = 0 + angle_depth = 0 + + while i < len(content): + char = content[i] + if char == '<': + angle_depth += 1 + elif char == '>': + angle_depth -= 1 + elif char == '{': + if angle_depth > 0: + # Inside a type like Promise<{...}> + brace_depth += 1 + else: + # This is the function body opening brace + return content[return_type_start:i].strip(), i + elif char == '}': + brace_depth -= 1 + i += 1 + + return '', -1 + + +def parse_default_imports(content: str) -> dict[str, str]: + """ + Parse default imports from TypeScript content. + Returns a dict mapping variable names to relative file paths. + E.g., 'import devCommand from "./dev.ts"' -> {'devCommand': './dev.ts'} + """ + imports = {} + # Match: import varName from "./path.ts" or import varName from './path.ts' + import_pattern = re.compile( + r'import\s+(\w+)\s+from\s+["\']([^"\']+)["\']', + re.MULTILINE + ) + for match in import_pattern.finditer(content): + var_name, path = match.groups() + imports[var_name] = path + return imports + + +def extract_options(text: str, option_pattern: re.Pattern) -> list[dict]: + """ + Extract options from text using the given regex pattern. + + The pattern should have 4 groups: (dq_flag, dq_desc, sq_flag, sq_desc) + for double-quoted and single-quoted variants. + + Returns a list of dicts with 'flag' and 'description' keys. + """ + options = [] + for match in option_pattern.finditer(text): + groups = match.groups() + # Pattern has 4 groups: (dq_flag, dq_desc, sq_flag, sq_desc) + # Either double-quoted or single-quoted pair will be non-None + flag = groups[0] or groups[2] + desc = groups[1] or groups[3] + if flag and desc: + options.append({'flag': flag, 'description': desc}) + return options + + +# ============================================================================= +# Schema Utilities +# ============================================================================= + + +def extract_cli_schema(schema: dict, all_schemas: dict, openflow_schemas: dict | None = None) -> dict: + """ + Transform an OpenAPI schema to CLI format by removing server-managed fields. + Resolves $ref references and handles allOf compositions. + + Args: + schema: The schema to transform + all_schemas: All schemas from the backend OpenAPI + openflow_schemas: Schemas from the openflow.openapi.yaml file (for external refs) + """ + if not schema: + return {} + + openflow_schemas = openflow_schemas or {} + result = {'type': 'object', 'properties': {}, 'required': []} + + # Handle allOf (used for composition, e.g., TriggerExtraProperty) + if 'allOf' in schema: + for item in schema['allOf']: + if '$ref' in item: + ref_name = item['$ref'].split('/')[-1] + if ref_name in all_schemas: + ref_schema = extract_cli_schema(all_schemas[ref_name], all_schemas, openflow_schemas) + result['properties'].update(ref_schema.get('properties', {})) + result['required'].extend(ref_schema.get('required', [])) + + # Handle direct properties + if 'properties' in schema: + for key, value in schema['properties'].items(): + if key not in CLI_EXCLUDED_FIELDS: + # Resolve $ref in property values + if '$ref' in value: + ref_path = value['$ref'] + # Handle local references + if ref_path.startswith('#/components/schemas/'): + ref_name = ref_path.split('/')[-1] + if ref_name in all_schemas: + result['properties'][key] = all_schemas[ref_name] + else: + result['properties'][key] = {'type': 'string', 'description': f'See {ref_name}'} + elif 'openflow.openapi.yaml' in ref_path: + # External reference to openflow schema - resolve it + ref_name = ref_path.split('/')[-1] + if ref_name in openflow_schemas: + result['properties'][key] = openflow_schemas[ref_name] + else: + result['properties'][key] = {'type': 'object', 'description': value.get('description', f'See {ref_name}')} + else: + # Other external reference + result['properties'][key] = {'type': 'object', 'description': value.get('description', f'See {ref_path}')} + else: + result['properties'][key] = value + + # Handle required fields + if 'required' in schema: + result['required'].extend([ + r for r in schema['required'] + if r not in CLI_EXCLUDED_FIELDS + ]) + + # Remove duplicates from required + result['required'] = list(dict.fromkeys(result['required'])) + + # Filter out required fields that don't exist in properties + result['required'] = [r for r in result['required'] if r in result['properties']] + + return result + + +def format_schema_as_json(schema: dict) -> dict: + """Convert a CLI schema to a clean JSON Schema representation.""" + if not schema or not schema.get('properties'): + return {} + + result = { + 'type': 'object', + 'properties': {}, + } + + props = schema.get('properties', {}) + required = schema.get('required', []) + + for key, value in props.items(): + prop_def = {} + + # Get type + prop_type = value.get('type', 'string') + if prop_type == 'array': + items = value.get('items', {}) + prop_def['type'] = 'array' + item_type = items.get('type', 'object') + if item_type == 'object' and items.get('properties'): + prop_def['items'] = {'type': 'object', 'properties': items.get('properties', {})} + else: + prop_def['items'] = {'type': item_type} + elif '$ref' in value: + # For refs, just indicate the type + ref_name = value['$ref'].split('/')[-1] + prop_def['type'] = ref_name + elif prop_type == 'object' and value.get('properties'): + # Nested object with properties - include them + prop_def['type'] = 'object' + prop_def['properties'] = value.get('properties', {}) + else: + prop_def['type'] = prop_type + + # Add enum if present + if 'enum' in value: + prop_def['enum'] = value['enum'] + + # Add description if present + if value.get('description'): + prop_def['description'] = value['description'] + + result['properties'][key] = prop_def + + if required: + result['required'] = required + + return result + + +def format_schema_for_markdown(schema: dict, schema_name: str, as_json_schema: bool = False) -> str: + """Format a CLI schema as markdown documentation.""" + if not schema or not schema.get('properties'): + return '' + + if as_json_schema: + # Output as JSON Schema + json_schema = format_schema_as_json(schema) + schema_json = json.dumps(json_schema, indent=2) + return f"## {schema_name}\n\nMust be a YAML file that adheres to the following schema:\n\n```json\n{schema_json}\n```" + else: + # Output as field list (for schedules) + lines = [f"### {schema_name} Schema (CLI Format)\n"] + lines.append("Fields available in `.yaml`/`.json` files:\n") + + props = schema.get('properties', {}) + required = set(schema.get('required', [])) + + for key, value in sorted(props.items()): + prop_type = value.get('type', 'any') + if prop_type == 'array': + items = value.get('items', {}) + item_type = items.get('type', 'any') + prop_type = f"array[{item_type}]" + elif '$ref' in value: + prop_type = value['$ref'].split('/')[-1] + + req_marker = ' (required)' if key in required else '' + desc = value.get('description', '') + desc_str = f" - {desc}" if desc else '' + + lines.append(f"- `{key}`: {prop_type}{req_marker}{desc_str}") + + return '\n'.join(lines) diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index fdeb27bedc..a5141aaa0b 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.625.0", + "version": "1.629.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 521c19474f..751b6a16a8 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.625.0", + "version": "1.629.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index 1bb512367a..a301a9d961 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.625.0 +1.629.1 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 1aa6ebf542..f211310a12 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.0.1", + "version": "1.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.0.1", + "version": "1.0.4", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index 4b4d1c1fc1..2f16ef4a27 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.0.2", + "version": "1.0.4", "description": "YAML validator for Windmill", "main": "dist/index.js", "types": "dist/index.d.ts",