Compare commits

..

2 Commits

Author SHA1 Message Date
centdix
698b969f78 better 2025-05-14 16:46:26 +02:00
centdix
6fe3bf3eff add probe usage doc 2025-05-14 16:41:26 +02:00
91 changed files with 1321 additions and 2630 deletions

View File

@@ -1,3 +0,0 @@
/*
!/backend/
!/frontend/

View File

@@ -1,170 +0,0 @@
name: Aider Auto-fix PR Review Change Requests
on:
pull_request_review:
types: [submitted]
jobs:
auto-fix-review:
if: github.event.review.state == 'changes_requested' && contains(github.event.pull_request.title, '[Aider PR]')
runs-on: ubicloud-standard-8
permissions:
contents: write
pull-requests: write
env:
GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WINDMILL_TOKEN: ${{ secrets.WINDMILL_TOKEN }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@v2
with:
egress-policy: audit
- name: Check out code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Configure Git User
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
- name: Checkout PR Branch
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "PR review trigger: Checking out PR branch..."
PR_NUMBER=${{ github.event.pull_request.number }}
PR_HEAD_REF=$(gh pr view $PR_NUMBER --json headRefName -q .headRefName --repo $GITHUB_REPOSITORY)
if [[ -z "$PR_HEAD_REF" || "$PR_HEAD_REF" == "null" ]]; then
echo "::error::Could not determine PR head branch for PR #$PR_NUMBER via gh CLI."
exit 1
fi
echo "Checking out PR head branch: $PR_HEAD_REF for PR #$PR_NUMBER"
git fetch origin "refs/heads/${PR_HEAD_REF}:refs/remotes/origin/${PR_HEAD_REF}" --no-tags
git checkout "$PR_HEAD_REF"
echo "Successfully checked out branch $(git rev-parse --abbrev-ref HEAD)"
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install Aider and Dependencies
run: |
python -m pip install aider-install; aider-install
pip install -U google-generativeai
sudo apt-get update && sudo apt-get install -y jq
- name: Generate Prompt from Review
id: generate_prompt
shell: bash
run: |
mkdir -p .github/aider
PROMPT_FILE_PATH=".github/aider/review-prompt.txt"
# Get PR review body
REVIEW_BODY="${{ github.event.review.body }}"
PR_NUMBER="${{ github.event.pull_request.number }}"
# Get PR description for context NOT USED FOR NOW
# PR_DETAILS=$(gh pr view $PR_NUMBER --json title,body --repo $GITHUB_REPOSITORY)
# PR_TITLE=$(echo "$PR_DETAILS" | jq -r .title)
# PR_BODY=$(echo "$PR_DETAILS" | jq -r .body)
# Get all PR review comments
REVIEW_COMMENTS=$(gh pr view $PR_NUMBER --json reviews -q '.reviews[] | select(.state == "CHANGES_REQUESTED") | .body' --repo $GITHUB_REPOSITORY)
REVIEW_BODY_Q=$(printf '%q' "$REVIEW_BODY")
# Update query to get review comments from all review types, not just "CHANGES_REQUESTED"
ALL_REVIEW_COMMENTS=$(gh api \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/comments \
| jq '[.[] | {diff_hunk: .diff_hunk, path: .path, body: .body}]')
BASE_PROMPT="Fix the following issues in the PR based on the review feedback. The review body is prepended with REVIEW. The review comments are prepended with REVIEW_COMMENTS. The review body and comments are separated by a blank line."
printf "%s\nREVIEW:\n%s\nREVIEW_COMMENTS:\n%s" \
"$BASE_PROMPT" "$REVIEW_BODY_Q" "$ALL_REVIEW_COMMENTS" > "$PROMPT_FILE_PATH"
echo "PROMPT_FILE_PATH=$PROMPT_FILE_PATH" >> $GITHUB_OUTPUT
- name: Run Aider with review prompt
run: |
aider \
--read .cursor/rules/rust-best-practices.mdc \
--read .cursor/rules/svelte5-best-practices.mdc \
--model gemini/gemini-2.5-pro-preview-05-06 \
--message-file .github/aider/review-prompt.txt \
--yes \
--no-check-update \
--auto-commits \
--no-analytics \
--no-gitignore \
| tee .github/aider/aider-output.txt || true
echo "Aider command completed. Output saved to .github/aider/aider-output.txt"
# Check if there are any changes to commit
if [[ -z "$(git status --porcelain)" ]]; then
echo "No changes detected after running Aider."
exit 0
fi
- name: Clean up prompt file
if: always()
run: rm -f .github/aider/review-prompt.txt
- name: Commit and Push Changes
id: commit_and_push
if: ${{ success() }}
run: |
CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.pull_request.number }}"
# Pull latest changes to avoid rejection due to non-fast-forward
git pull origin $CURRENT_BRANCH_NAME
if git push origin $CURRENT_BRANCH_NAME; then
echo "Push to $CURRENT_BRANCH_NAME successful."
echo "CHANGES_APPLIED=true" >> $GITHUB_OUTPUT
else
echo "::warning::Push to PR branch $CURRENT_BRANCH_NAME failed."
echo "CHANGES_APPLIED=false" >> $GITHUB_OUTPUT
fi
- name: Comment on PR
if: success()
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUM: ${{ github.event.pull_request.number }}
run: |
# Create comment body in a temporary file to avoid command line length limits
if [[ "${{ steps.commit_and_push.outputs.CHANGES_APPLIED }}" == "true" ]]; then
cat > /tmp/pr-comment.md << EOL
🤖 I've automatically addressed the feedback based on the review.
## Aider Output
\`\`\`
$(cat .github/aider/aider-output.txt || echo 'No output available')
\`\`\`
Please review the changes and let me know if further adjustments are needed.
EOL
else
cat > /tmp/pr-comment.md << EOL
🤖 I attempted to address the review feedback, but no modifications were made.
## Aider Output
\`\`\`
$(cat .github/aider/aider-output.txt || echo 'No output available')
\`\`\`
Please review the output and provide additional guidance if needed.
EOL
fi
# Use the file for comment body
gh pr comment $PR_NUM --body-file /tmp/pr-comment.md

View File

@@ -6,7 +6,7 @@ on:
jobs:
auto-fix:
runs-on: ubicloud-standard-8
runs-on: ubuntu-latest
if: |
github.event_name == 'issue_comment' &&
contains(github.event.comment.body, '/aider') &&
@@ -237,7 +237,7 @@ jobs:
'{ "message": "I'\''m giving you a request that needs to be implemented. Your role is ONLY to give me the files that are relevant to the request and nothing else. The request is prepended with the word REQUEST.\\nREQUEST: \($prompt_escaped). Give me all the files relevant to this request. Your output MUST be a single json array that can be parsed with programatic json parsing, with the relevant files. Files can be rust or typescript or javascript files. DO NOT INCLUDE ANY OTHER TEXT IN YOUR OUTPUT. ONLY THE JSON ARRAY. Example of output: [\"file1.py\", \"file2.py\"]" }' | jq -r .message)
set -o pipefail
PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE") || {
PROBE_OUTPUT=$(npx --yes @buger/probe-chat@latest --max-iterations 50 --model-name gemini-2.5-pro-preview-05-06 --message "$MESSAGE_FOR_PROBE" 2>&1) || {
echo "::error::probe-chat command failed. Output:"
echo "$PROBE_OUTPUT"
exit 1
@@ -264,8 +264,6 @@ jobs:
run: |
echo "Files identified by probe-chat: ${{ env.FILES_TO_EDIT }}"
aider \
--read .cursor/rules/rust-best-practices.mdc \
--read .cursor/rules/svelte5-best-practices.mdc \
${{ env.FILES_TO_EDIT }} \
--model gemini/gemini-2.5-pro-preview-05-06 \
--message-file .github/aider/issue-prompt.txt \
@@ -273,8 +271,7 @@ jobs:
--no-check-update \
--auto-commits \
--no-analytics \
--no-gitignore \
| tee .github/aider/aider-output.txt || true
--no-stream > .github/aider/aider-output.txt 2>&1 || true
echo "Aider command completed. Output saved to .github/aider/aider-output.txt"
- name: Clean up prompt file
@@ -287,23 +284,12 @@ jobs:
run: |
if [[ -z "${{ github.event.issue.pull_request }}" ]]; then
BRANCH_NAME="aider-fix-issue-${{ github.event.issue.number }}"
# Check if branch exists remotely
if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then
echo "Branch $BRANCH_NAME already exists remotely, fetching it"
git fetch origin $BRANCH_NAME
git checkout $BRANCH_NAME
git pull origin $BRANCH_NAME
else
echo "Creating new branch $BRANCH_NAME"
git checkout -b $BRANCH_NAME
fi
echo "Created/checked out branch $BRANCH_NAME for issue #${{ github.event.issue.number }}"
git checkout -b $BRANCH_NAME
echo "Created branch $BRANCH_NAME for issue #${{ github.event.issue.number }}"
git push origin $BRANCH_NAME
echo "Pushed to branch $BRANCH_NAME"
echo "Pushed to new branch $BRANCH_NAME"
echo "PR_BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_OUTPUT
echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to branch $BRANCH_NAME." >> $GITHUB_OUTPUT
echo "CHANGES_APPLIED_MESSAGE=Aider changes pushed to new branch $BRANCH_NAME." >> $GITHUB_OUTPUT
else
CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
echo "Attempting to push changes to PR branch $CURRENT_BRANCH_NAME for PR #${{ github.event.issue.number }}"
@@ -324,19 +310,13 @@ jobs:
PR_BRANCH: ${{ steps.commit_and_push.outputs.PR_BRANCH_NAME }}
ISSUE_NUM: ${{ github.event.issue.number }}
run: |
# Create PR description in a temporary file to avoid command line length limits
cat > /tmp/pr-description.md << EOL
This PR was created automatically by Aider to fix issue #${ISSUE_NUM}.
## Aider Output
\`\`\`
$(cat .github/aider/aider-output.txt || echo "No output available")
\`\`\`
EOL
# Create PR using the file for the body content
gh pr create \
--title "[Aider PR] Add fixes for issue #${ISSUE_NUM}" \
--body-file /tmp/pr-description.md \
--body "This PR was created automatically by Aider to fix issue #${ISSUE_NUM}.
## Aider Output
\`\`\`
$(cat .github/aider/aider-output.txt || echo "No output available")
\`\`\`" \
--head "$PR_BRANCH" \
--base main

View File

@@ -1,15 +0,0 @@
on:
issue_comment:
types: [created]
jobs:
trigger-docs:
if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/docs') }}
uses: windmill-labs/windmilldocs/.github/workflows/create-docs.yml@main
with:
pr_number: ${{ github.event.issue.number }}
repo: ${{ github.event.repository.name }}
comment_text: ${{ github.event.comment.body }}
secrets:
DOCS_TOKEN: ${{ secrets.DOCS_TOKEN }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}

View File

@@ -1,32 +0,0 @@
name: Create discord thread when a PR is opened, react with green checkmark when PR is merged
on:
pull_request:
types:
- opened
- ready_for_review
- closed
jobs:
notify_discord_when_pr_opened:
if: (github.event.pull_request.draft == false) && (github.event.action == 'opened' || github.event.action == 'ready_for_review')
uses: ./.github/workflows/shareable-discord-notification.yml
with:
PR_TITLE: ${{ github.event.pull_request.title }}
PR_URL: ${{ github.event.pull_request.html_url }}
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
PR_STATUS: "opened"
PR_NUMBER: ${{ github.event.pull_request.number }}
secrets:
DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_PR_REVIEWS_WEBHOOK }}
merge_success_emoji:
if: github.event.pull_request.merged == true
uses: ./.github/workflows/shareable-discord-notification.yml
with:
PR_STATUS: "merged"
DISCORD_CHANNEL_ID: "1372204995868491786"
DISCORD_GUILD_ID: "930051556043276338"
PR_NUMBER: ${{ github.event.pull_request.number }}
secrets:
DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_PR_BOT_TOKEN }}

View File

@@ -1,66 +0,0 @@
name: Publish Helm Chart on Release
on:
release:
types: [published]
jobs:
bump-helm-version:
runs-on: ubicloud-standard-2
steps:
- name: Checkout on helm repository
uses: actions/checkout@v3
with:
repository: windmill-labs/windmill-helm-charts
token: ${{ secrets.DOCS_TOKEN }}
- name: Get version
id: get_version
run: |
echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
- name: Create new branch
run: |
# Check if branch already exists remotely
if git ls-remote --heads origin bump-helm-version-${{ env.VERSION }} | grep -q bump-helm-version-${{ env.VERSION }}; then
# Branch exists, check it out
git fetch origin bump-helm-version-${{ env.VERSION }}
git checkout bump-helm-version-${{ env.VERSION }}
else
# Create new branch
git checkout -b bump-helm-version-${{ env.VERSION }}
fi
git config --local user.email "action@github.com"
git config --local user.name "GitHub Action"
- name: Bump helm version
run: |
# Get current version and increment it by 1
CURRENT_VERSION=$(grep "version:" ./charts/windmill/Chart.yaml | awk '{print $2}' | head -n 1)
NEW_VERSION=$(echo "$CURRENT_VERSION" | awk -F. '{$NF = $NF + 1;} 1' | sed 's/ /./g')
sed -i "s/^version: .*/version: $NEW_VERSION/" ./charts/windmill/Chart.yaml
# Get the app version from the version
VERSION=${{ env.VERSION }}
APP_VERSION=${VERSION#refs/tag/}
APP_VERSION=${APP_VERSION#v}
APP_VERSION=${APP_VERSION%/}
sed -i "s/appVersion: .*/appVersion: $APP_VERSION/" ./charts/windmill/Chart.yaml
- name: Commit and push
run: |
git add .
git commit -m "Bump helm version to ${{ env.VERSION }}"
git push origin bump-helm-version-${{ env.VERSION }}
- name: Create PR
env:
GH_TOKEN: ${{ secrets.DOCS_TOKEN }}
run: |
gh pr create \
--title "helm: bump version to ${{ env.VERSION }}" \
--body "This PR was auto-generated to bring the helm chart up to date for [release ${{ env.VERSION }}](https://github.com/windmill-labs/windmill/releases/tag/v${{ env.VERSION }}) in the main repo." \
--head bump-helm-version-${{ env.VERSION }} \
--base main

34
.github/workflows/pr-opened.yml vendored Normal file
View File

@@ -0,0 +1,34 @@
name: "Notify Discord on New PR (with Thread)"
on:
pull_request:
types:
- opened
- ready_for_review
jobs:
discord_notification:
# still guard out any drafts (just in case)
if: github.event.pull_request.draft == false
runs-on: ubicloud-standard-2
steps:
- name: Send Discord notification and start a thread
env:
WEBHOOK_URL: ${{ secrets.DISCORD_PR_REVIEWS_WEBHOOK }}
PR_TITLE: ${{ github.event.pull_request.title }}
PR_URL: ${{ github.event.pull_request.html_url }}
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
run: |
payload=$(jq -n \
--arg content "${PR_URL}" \
--arg thread "$PR_TITLE by \`${PR_AUTHOR}\`" \
'{
content: $content,
thread_name: $thread,
auto_archive_duration: 10080
}'
)
curl -H "Content-Type: application/json" \
-X POST \
--data "$payload" \
"$WEBHOOK_URL"

View File

@@ -1,98 +0,0 @@
name: "Notify Discord when a PR is opened or merged"
on:
workflow_call:
inputs:
PR_TITLE:
description: "The title of the PR"
type: string
PR_URL:
description: "The URL of the PR"
type: string
PR_AUTHOR:
description: "The author of the PR"
type: string
PR_STATUS:
description: "The status of the PR"
type: string
DISCORD_CHANNEL_ID:
description: "The Discord channel ID"
type: string
PR_NUMBER:
description: "The number of the PR"
type: string
DISCORD_GUILD_ID:
description: "The Discord guild ID"
type: string
secrets:
DISCORD_WEBHOOK_URL:
description: "Discord Webhook URL"
DISCORD_BOT_TOKEN:
description: "Discord Bot Token"
jobs:
open_thread:
runs-on: ubicloud-standard-2
if: ${{ inputs.PR_STATUS == 'opened' }}
steps:
- name: Send Discord notification and start a thread
env:
WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK_URL }}
PR_TITLE: ${{ inputs.PR_TITLE }}
PR_NUMBER: ${{ inputs.PR_NUMBER }}
PR_URL: ${{ inputs.PR_URL }}
PR_AUTHOR: ${{ inputs.PR_AUTHOR }}
run: |
payload=$(jq -n \
--arg content "${PR_URL}" \
--arg thread "#${PR_NUMBER}: $PR_TITLE by \`${PR_AUTHOR}\`" \
'{
content: $content,
thread_name: $thread,
auto_archive_duration: 10080
}'
)
curl -H "Content-Type: application/json" \
-X POST \
-d "$payload" \
"$WEBHOOK_URL"
merge_success_emoji:
runs-on: ubuntu-latest
if: ${{ inputs.PR_STATUS == 'merged' }}
steps:
- name: React
env:
BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
CHANNEL_ID: ${{ inputs.DISCORD_CHANNEL_ID }}
GUILD_ID: ${{ inputs.DISCORD_GUILD_ID }}
PR_NUMBER: ${{ inputs.PR_NUMBER }}
run: |
# 1) get PR thread
threads=$(curl -H "Authorization: Bot $BOT_TOKEN" "https://discord.com/api/v10/guilds/${GUILD_ID}/threads/active")
thread_id=$(
echo "$threads" \
| jq -r --arg cid "$CHANNEL_ID" \
--arg pref "#${PR_NUMBER}:" \
'.threads[]
| select(.parent_id == $cid and (.name | startswith($pref)))
| .id'
)
if [ -z "$thread_id" ]; then
echo "Thread not found"
exit 1
fi
# 2) get the first message in that thread
messages=$(curl -H "Authorization: Bot $BOT_TOKEN" \
"https://discord.com/api/v10/channels/$thread_id/messages?limit=1")
message_id=$(echo "$messages" | jq -r '.[-1].id')
if [ -z "$message_id" ]; then
echo "Message not found"
exit 1
fi
# 3) add the ✅ reaction
curl -X PUT \
-H "Authorization: Bot $BOT_TOKEN" \
"https://discord.com/api/v10/channels/$thread_id/messages/$message_id/reactions/%E2%9C%85/@me"

2
.gitignore vendored
View File

@@ -10,5 +10,3 @@ CaddyfileRemoteMalo
.vscode
.dev-docker-wrapper*
backend/.minio-data
.aider*
!.aiderignore

View File

@@ -1,50 +1,5 @@
# Changelog
## [1.491.5](https://github.com/windmill-labs/windmill/compare/v1.491.4...v1.491.5) (2025-05-17)
### Bug Fixes
* improve handling of custom concurrency key/tag with preprocessors ([#5762](https://github.com/windmill-labs/windmill/issues/5762)) ([59afa49](https://github.com/windmill-labs/windmill/commit/59afa493fa20cc70b6825e6356713cef84d75312))
* S3 sql mode returns S3Object ([#5764](https://github.com/windmill-labs/windmill/issues/5764)) ([b29c6e7](https://github.com/windmill-labs/windmill/commit/b29c6e7636bb21c4d977bdaf89ac90e2a1a1086c))
## [1.491.4](https://github.com/windmill-labs/windmill/compare/v1.491.3...v1.491.4) (2025-05-15)
### Bug Fixes
* add v1 preprocessor support to workspace preprocessor script ([#5757](https://github.com/windmill-labs/windmill/issues/5757)) ([9b1c30e](https://github.com/windmill-labs/windmill/commit/9b1c30eeff35291ad50f3ddeb64831eac88e2f66))
## [1.491.3](https://github.com/windmill-labs/windmill/compare/v1.491.2...v1.491.3) (2025-05-15)
### Bug Fixes
* **frontend:** fix accordeon tabs initialization ([f488903](https://github.com/windmill-labs/windmill/commit/f488903635a1457f839ca641ed4f8d0891ef8212))
* http trigger routers cache version sequence ([#5755](https://github.com/windmill-labs/windmill/issues/5755)) ([d53bceb](https://github.com/windmill-labs/windmill/commit/d53bceb8004541b79d33220ae8de06d25521da91))
## [1.491.2](https://github.com/windmill-labs/windmill/compare/v1.491.1...v1.491.2) (2025-05-15)
### Bug Fixes
* **cli:** --version improvement ([f8f2015](https://github.com/windmill-labs/windmill/commit/f8f201564f7a323eb96f6dc684a525a0784d41f2))
* http trigger signature validation ([#5753](https://github.com/windmill-labs/windmill/issues/5753)) ([9e9514b](https://github.com/windmill-labs/windmill/commit/9e9514b9af2337e143a9e4cf1e915e1477032e80))
* Improve indexer performance by factoring required queries to the DB # ([#5749](https://github.com/windmill-labs/windmill/issues/5749)) ([b12feaf](https://github.com/windmill-labs/windmill/commit/b12feaf50ae0ef03816719ff39157fcf55159dbf))
* improve perf of job deletion ([0efba94](https://github.com/windmill-labs/windmill/commit/0efba945bac9b84a489c6ef552e834593f209fe1))
### Performance Improvements
* cache http trigger routers and auth ([#5748](https://github.com/windmill-labs/windmill/issues/5748)) ([ddd18d2](https://github.com/windmill-labs/windmill/commit/ddd18d22a615408a9f57f910d0a58f17e6d6e29d))
## [1.491.1](https://github.com/windmill-labs/windmill/compare/v1.491.0...v1.491.1) (2025-05-15)
### Bug Fixes
* avoid deadlocks in sending completed job to result processors ([#5742](https://github.com/windmill-labs/windmill/issues/5742)) ([e87d4f3](https://github.com/windmill-labs/windmill/commit/e87d4f3c1afb4ad356b326b7600c89e6c7803eff))
## [1.491.0](https://github.com/windmill-labs/windmill/compare/v1.490.0...v1.491.0) (2025-05-14)

150
PROBE_USAGE.md Normal file
View File

@@ -0,0 +1,150 @@
# Probe Usage
Probe is a powerful, AI-augmented code search and extraction tool. It combines fast, tree-sitterpowered queries with optional LLM integration to help you find, extract, and understand code across any codebase.
---
## 📦 Installation
The easiest way to install Probe is via npm:
```bash
npm install -g @buger/probe@latest
```
Or using curl (for macOS and Linux):
```bash
curl -fsSL https://raw.githubusercontent.com/buger/probe/main/install.sh | bash
```
## 🔍 Basic Search Example
Search for code containing specific phrases in the current directory:
```bash
probe search "parser" ./backend
```
This will search for the terms "parser" in your codebase and return the most relevant code blocks.
## 💾 Session-Based Caching
Use session IDs to avoid seeing the same code blocks multiple times in related searches:
```bash
# First search - generates a session ID
probe search "authentication" --session ""
# Session: a1b2 (example output)
# Subsequent searches - reuse the session ID
probe search "login" --session "a1b2"
# Will skip code blocks already shown in the previous search
```
## 🔎 Elastic Search Queries
Use advanced query syntax for more powerful searches:
```bash
# Use AND operator for terms that must appear together
probe search "error AND handling" ./
# Use OR operator for alternative terms
probe search "login OR authentication OR auth" ./src
# Group terms with parentheses for complex queries
probe search "(error OR exception) AND (handle OR process)" ./
# Use wildcards for partial matching
probe search "auth* connect*" ./
# Exclude terms with NOT operator
probe search "database NOT sqlite" ./
```
## 📋 Extract Code Blocks
Extract a specific function or code block containing a specific line:
```bash
probe extract src/main.rs:42
```
This uses tree-sitter to find the closest suitable parent node (function, struct, class, etc.) for that line.
You can even pipe failing test output:
```bash
go test | probe extract
```
Extract code with LLM prompt and instructions for AI integration:
```bash
# Extract with engineer prompt template
probe extract src/auth.rs#authenticate --prompt engineer --instructions "Explain this authentication function"
# Extract with architect prompt template
probe extract src/api.js --prompt architect --instructions "Analyze this API module"
```
## 🔍 Query Code Structures
Find specific code structures using tree-sitter patterns:
```bash
# Find JavaScript functions
probe query "function $NAME($$$PARAMS) $$$BODY" ./src --language javascript
# Find Python functions
probe query "def $NAME($$$PARAMS): $$$BODY" ./src --language python
# Find Go structs
probe query "type $NAME struct { $$$FIELDS }" ./src --language go
```
## 💬 Interactive AI Chat
Use the built-in AI assistant with web interface:
```bash
# Run directly with npx (no installation needed)
npx -y @buger/probe-chat@latest --web
npx -y @buger/probe-chat@latest
# Set your API key first
export ANTHROPIC_API_KEY=your_api_key
# Or for OpenAI
# export OPENAI_API_KEY=your_api_key
# Or for Gemini
# export GOOGLE_API_KEY=your_api_key
# Specify a directory to search (optional)
npx -y @buger/probe-chat@latest /path/to/your/project
```
Example questions you might ask:
- "How does the workers pick up jobs?"
- "How does the mcp functionality work?"
- "What are the main components of the windmill backend?"
## 🔌 MCP Server Integration
Integrate with any AI editor by adding this to your MCP configuration:
```json
{
"mcpServers": {
"memory": {
"command": "npx",
"args": ["-y", "@buger/probe-mcp"]
}
}
}
```
## 📖 Official Documentation
https://probeai.dev/features

View File

@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n value->'preprocessor_module' IS NOT NULL as has_preprocessor,\n value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,\n schema as \"schema: _\"\n FROM flow \n WHERE workspace_id = $1 \n AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "has_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "is_v1_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "schema: _",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null,
null,
true
]
},
"hash": "0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73"
}

View File

@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT nextval('http_trigger_version_seq')",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "nextval",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "16be720bf1c88ecfa2a4bf6adbb1924df4817b6236b3a949209465f3a2c42bb9"
}

View File

@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job_completed c\n USING v2_job j\n WHERE\n created_at <= now() - ($1::bigint::text || ' s')::interval\n AND completed_at + ($1::bigint::text || ' s')::interval <= now()\n AND c.id = j.id\n RETURNING c.id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "1d819b829cd92995c39d29540df8cffbcc3334bada244a331a0bd8db06029d42"
}

View File

@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT last_value FROM http_trigger_version_seq",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_value",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "20fd50949796913dd48f67ee75b11272ce5b3046f87b9efd504e013fba9724f5"
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1\n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17"
}

View File

@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT created_by, coalesce(job_logs.logs, '') as logs, job_logs.log_offset, job_logs.log_file_index\n FROM v2_as_completed_job\n LEFT JOIN job_logs ON job_logs.job_id = v2_as_completed_job.id\n WHERE v2_as_completed_job.id = $1 AND v2_as_completed_job.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "log_offset",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "log_file_index",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
true,
null,
false,
true
]
},
"hash": "282afbff89d3186d47ef5dbd0b65026ad37fb31b485fc44b6ec257dd77825428"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DROP INDEX CONCURRENTLY IF EXISTS log_file_hostname_log_ts_idx",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "2a33a35afc1ba4c31a5713cfd1a2c662f25cda387197aaf9f35000df31b8b07d"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n path, \n script_path, \n is_flow, \n route_path, \n authentication_resource_path,\n workspace_id, \n is_async, \n authentication_method AS \"authentication_method: _\", \n edited_by, \n email, \n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website\n FROM \n http_trigger \n WHERE \n http_method = $1\n ",
"query": "\n SELECT \n path, \n script_path, \n is_flow, \n route_path, \n authentication_resource_path,\n workspace_id, \n is_async, \n authentication_method AS \"authentication_method: _\", \n edited_by, \n email, \n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website\n FROM \n http_trigger \n WHERE \n http_method = $1\n ",
"describe": {
"columns": [
{
@@ -129,5 +129,5 @@
false
]
},
"hash": "1eeb218c30c0a6b0f7633813c764f57f8968894b4786b94109a057796ff500e4"
"hash": "4053f0bb30f651ddf2214115748daca0ea457da8252394eeeead0897d184f6da"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_job_completed_completed_at ON v2_job_completed (completed_at DESC)",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "49943f69ed74bc889120dcd2571e8e868a4f4795933044ff95b20d8df45cd145"
}

View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "WITH job_result AS (\n SELECT result \n FROM v2_job_completed \n WHERE id = $1\n )\n UPDATE v2_job \n SET args = COALESCE(\n CASE \n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object' \n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END, \n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "52ad0c838d19cbd9e90b8368abe71dd12655179f41f43896e7d30fdfb3ae5939"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job_completed c\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval \n RETURNING c.id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "5820d34be1a7f7b72e656c692f53146f45ad4a6e584e917a0a86280d8f473c10"
}

View File

@@ -1,248 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n cj.id AS \"id!\",\n cj.workspace_id AS \"workspace_id!\",\n cj.parent_job,\n cj.created_by AS \"created_by!\",\n cj.duration_ms AS \"duration_ms!\",\n cj.success AS \"success!\",\n cj.script_hash AS \"script_hash!: Option<ScriptHash>\",\n cj.script_path,\n cj.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n cj.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n cj.deleted AS \"deleted!\",\n cj.canceled AS \"canceled!\",\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind AS \"job_kind!: JobKind\",\n cj.schedule_path,\n cj.permissioned_as AS \"permissioned_as!\",\n cj.is_flow_step AS \"is_flow_step!\",\n cj.language AS \"language: ScriptLang\",\n cj.is_skipped AS \"is_skipped!\",\n cj.email AS \"email!\",\n cj.visible_to_owner AS \"visible_to_owner!\",\n cj.mem_peak,\n cj.tag AS \"tag!\",\n cj.created_at AS \"created_at!\",\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset AS \"log_offset?\",\n job_logs.log_file_index\n\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE (cj.created_at > $1 AND cj.created_at < $3)\n OR cj.id = ANY($2)\n ORDER BY cj.created_at ASC LIMIT $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "parent_job",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "success!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_hash!: Option<ScriptHash>",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "args: sqlx::types::Json<HashMap<String, Box<RawValue>>>",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 10,
"name": "deleted!",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "canceled!",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "canceled_by",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "canceled_reason",
"type_info": "Text"
},
{
"ordinal": 14,
"name": "job_kind!: JobKind",
"type_info": {
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
}
},
{
"ordinal": 15,
"name": "schedule_path",
"type_info": "Varchar"
},
{
"ordinal": 16,
"name": "permissioned_as!",
"type_info": "Varchar"
},
{
"ordinal": 17,
"name": "is_flow_step!",
"type_info": "Bool"
},
{
"ordinal": 18,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java"
]
}
}
}
},
{
"ordinal": 19,
"name": "is_skipped!",
"type_info": "Bool"
},
{
"ordinal": 20,
"name": "email!",
"type_info": "Varchar"
},
{
"ordinal": 21,
"name": "visible_to_owner!",
"type_info": "Bool"
},
{
"ordinal": 22,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 23,
"name": "tag!",
"type_info": "Varchar"
},
{
"ordinal": 24,
"name": "created_at!",
"type_info": "Timestamptz"
},
{
"ordinal": 25,
"name": "started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 26,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 27,
"name": "log_offset?",
"type_info": "Int4"
},
{
"ordinal": 28,
"name": "log_file_index",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Timestamptz",
"UuidArray",
"Timestamptz",
"Int8"
]
},
"nullable": [
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
false,
true
]
},
"hash": "6e2564dc37ee967c634deade67ceabc4c516418e595dee9cd7d651acc1f4af6e"
}

View File

@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n value->'preprocessor_module'->'value' as \"preprocessor_module: _\",\n schema as \"schema: _\"\n FROM flow \n WHERE workspace_id = $1\n AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "preprocessor_module: _",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "schema: _",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null,
true
]
},
"hash": "72916f8e490f8252e0a51b7f562ccc3be832b12102eb86a07d8405a4fa9287d5"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT result as \"result: Json<HashMap<String, Box<RawValue>>>\"\n FROM v2_job_completed \n WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: Json<HashMap<String, Box<RawValue>>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true
]
},
"hash": "91f23fcc27777c279c79e2682fc15c026e55f9ec3799be65a2e8920fe6174a17"
}

View File

@@ -0,0 +1,134 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n path, \n script_path, \n is_flow, \n route_path, \n workspace_id, \n is_async, \n authentication_method AS \"authentication_method: _\", \n edited_by, \n email,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website,\n authentication_resource_path\n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n http_method = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "route_path",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "authentication_method: _",
"type_info": {
"Custom": {
"name": "authentication_method",
"kind": {
"Enum": [
"none",
"windmill",
"api_key",
"basic_http",
"custom_script",
"signature"
]
}
}
}
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "static_asset_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 10,
"name": "wrap_body",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "raw_string",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "workspaced_route",
"type_info": "Bool"
},
{
"ordinal": 13,
"name": "is_static_website",
"type_info": "Bool"
},
{
"ordinal": 14,
"name": "authentication_resource_path",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
false,
false,
false,
false,
true
]
},
"hash": "927149213e0f8ae983652ef80464f646d6be80e702193f1acdd40dd6033652e4"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_by_workspace ON alerts (workspace_id);",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "a8ca4e588e0bf3c4bba2fe4b68a5364e4cba99964513599f8a012a5680d3dca8"
}

View File

@@ -18,8 +18,8 @@
"Left": []
},
"nullable": [
false,
true
true,
false
]
},
"hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76"

View File

@@ -1,46 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH j AS (\n SELECT \n raw_flow->>'concurrency_key' as concurrency_key, \n raw_flow->>'concurrency_time_window_s' as concurrency_time_window_s,\n raw_flow->>'concurrency_limit' as concurrent_limit,\n runnable_path, \n runnable_id as version FROM v2_job\n WHERE id = $1\n )\n SELECT tag, j.concurrency_key, j.concurrency_time_window_s::int, j.concurrent_limit::int, j.version\n FROM flow, j\n WHERE path = j.runnable_path\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "concurrency_key",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "concurrency_time_window_s",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "concurrent_limit",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "version",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
true,
null,
null,
null,
true
]
},
"hash": "b7335ac24702c86fbb4ab95916a6aa1648082287b09122755df2462dc71ce831"
}

View File

@@ -1,18 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH job_result AS (\n SELECT result \n FROM v2_job_completed \n WHERE id = $1\n ),\n updated_queue AS (\n UPDATE v2_job_queue\n SET running = false,\n tag = COALESCE($3, tag)\n WHERE id = $2\n )\n UPDATE v2_job \n SET \n tag = COALESCE($3, tag),\n concurrent_limit = COALESCE($4, concurrent_limit),\n concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),\n args = COALESCE(\n CASE \n WHEN job_result.result IS NULL THEN NULL\n WHEN jsonb_typeof(job_result.result) = 'object' \n THEN job_result.result\n WHEN jsonb_typeof(job_result.result) = 'null'\n THEN NULL\n ELSE jsonb_build_object('value', job_result.result)\n END, \n '{}'::jsonb\n ),\n preprocessed = TRUE\n FROM job_result\n WHERE v2_job.id = $2;\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Varchar",
"Int4",
"Int4"
]
},
"nullable": []
},
"hash": "e07660e8d2a265cb6a83f3a2bb8e7e6330f09ab116e9837f6f16f8fdef938004"
}

View File

@@ -1,246 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n cj.id AS \"id!\",\n cj.workspace_id AS \"workspace_id!\",\n cj.parent_job,\n cj.created_by AS \"created_by!\",\n cj.duration_ms AS \"duration_ms!\",\n cj.success AS \"success!\",\n cj.script_hash AS \"script_hash!: Option<ScriptHash>\",\n cj.script_path,\n cj.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",\n cj.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n cj.deleted AS \"deleted!\",\n cj.canceled AS \"canceled!\",\n cj.canceled_by,\n cj.canceled_reason,\n cj.job_kind AS \"job_kind!: JobKind\",\n cj.schedule_path,\n cj.permissioned_as AS \"permissioned_as!\",\n cj.is_flow_step AS \"is_flow_step!\",\n cj.language AS \"language: ScriptLang\",\n cj.is_skipped AS \"is_skipped!\",\n cj.email AS \"email!\",\n cj.visible_to_owner AS \"visible_to_owner!\",\n cj.mem_peak,\n cj.tag AS \"tag!\",\n cj.created_at AS \"created_at!\",\n cj.started_at,\n job_logs.logs,\n job_logs.log_offset AS \"log_offset?\",\n job_logs.log_file_index\n\n FROM v2_as_completed_job AS cj\n LEFT JOIN job_logs ON cj.id = job_logs.job_id\n WHERE cj.created_at < $1\n ORDER BY cj.created_at ASC LIMIT $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "parent_job",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "success!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_hash!: Option<ScriptHash>",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "args: sqlx::types::Json<HashMap<String, Box<RawValue>>>",
"type_info": "Jsonb"
},
{
"ordinal": 9,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 10,
"name": "deleted!",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "canceled!",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "canceled_by",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "canceled_reason",
"type_info": "Text"
},
{
"ordinal": 14,
"name": "job_kind!: JobKind",
"type_info": {
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript"
]
}
}
}
},
{
"ordinal": 15,
"name": "schedule_path",
"type_info": "Varchar"
},
{
"ordinal": 16,
"name": "permissioned_as!",
"type_info": "Varchar"
},
{
"ordinal": 17,
"name": "is_flow_step!",
"type_info": "Bool"
},
{
"ordinal": 18,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java"
]
}
}
}
},
{
"ordinal": 19,
"name": "is_skipped!",
"type_info": "Bool"
},
{
"ordinal": 20,
"name": "email!",
"type_info": "Varchar"
},
{
"ordinal": 21,
"name": "visible_to_owner!",
"type_info": "Bool"
},
{
"ordinal": 22,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 23,
"name": "tag!",
"type_info": "Varchar"
},
{
"ordinal": 24,
"name": "created_at!",
"type_info": "Timestamptz"
},
{
"ordinal": 25,
"name": "started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 26,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 27,
"name": "log_offset?",
"type_info": "Int4"
},
{
"ordinal": 28,
"name": "log_file_index",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Timestamptz",
"Int8"
]
},
"nullable": [
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
true,
false,
true
]
},
"hash": "f22964772dc2d67aee437bbbd08b64792c00da1d713d7ca8f9904ccce7bfdae7"
}

View File

@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT EXISTS(\n SELECT 1 \n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "f632d08a8d3df691fff9f57fdf926f787287c3cd181a6853056077edba10473d"
}

View File

@@ -41,11 +41,11 @@
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
true,
true,
true,
true,
true
]
},

240
backend/Cargo.lock generated
View File

@@ -744,9 +744,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "aws-config"
version = "1.6.3"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02a18fd934af6ae7ca52410d4548b98eb895aab0f1ea417d168d85db1434a141"
checksum = "b6fcc63c9860579e4cb396239570e979376e70aab79e496621748a09913f8b36"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -833,9 +833,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sqs"
version = "1.68.0"
version = "1.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b484821a335b02b109c17623b8347e692583c2229f8db2f029edd0fdbbd3bea"
checksum = "c6f15bedfb1c4385fccc474f0fe46dffb0335d0b3d6b4413df06fb30d90caba8"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -849,15 +849,16 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-sso"
version = "1.68.0"
version = "1.67.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd5f01ea61fed99b5fe4877abff6c56943342a56ff145e9e0c7e2494419008be"
checksum = "0d4863da26489d1e6da91d7e12b10c17e86c14f94c53f416bd10e0a9c34057ba"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -871,15 +872,16 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-ssooidc"
version = "1.69.0"
version = "1.68.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27454e4c55aaa4ef65647e3a1cf095cb834ca6d54e959e2909f1fef96ad87860"
checksum = "95caa3998d7237789b57b95a8e031f60537adab21fa84c91e35bef9455c652e4"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -893,15 +895,16 @@ dependencies = [
"bytes",
"fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-sts"
version = "1.69.0"
version = "1.68.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffd6ef5d00c94215960fabcdf2d9fe7c090eed8be482d66d47b92d4aba1dd4aa"
checksum = "4939f6f449a37308a78c5a910fd91265479bd2bb11d186f0b8fc114d89ec828d"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -916,15 +919,16 @@ dependencies = [
"aws-types",
"fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sigv4"
version = "1.3.2"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3734aecf9ff79aa401a6ca099d076535ab465ff76b46440cf567c8e70b65dc13"
checksum = "3503af839bd8751d0bdc5a46b9cac93a003a353e635b0c12cf2376b5b53e41ea"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
@@ -1299,7 +1303,7 @@ version = "0.69.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"cexpr",
"clang-sys",
"itertools 0.12.1",
@@ -1322,7 +1326,7 @@ version = "0.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"cexpr",
"clang-sys",
"itertools 0.13.0",
@@ -1374,9 +1378,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.9.1"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
dependencies = [
"serde",
]
@@ -1839,9 +1843,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.23"
version = "1.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766"
checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1"
dependencies = [
"jobserver",
"libc",
@@ -2434,7 +2438,7 @@ version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b28bfe653d79bd16c77f659305b195b82bb5ce0c0eb2a4846b82ddbd77586813"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"libloading 0.8.7",
"winapi",
]
@@ -3211,7 +3215,7 @@ dependencies = [
"once_cell",
"percent-encoding",
"serde",
"sourcemap 9.2.1",
"sourcemap 9.2.0",
"swc_atoms",
"swc_common",
"swc_config",
@@ -4148,9 +4152,9 @@ dependencies = [
[[package]]
name = "deno_unsync"
version = "0.4.3"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47c618b51088b3ac67f15c69b3ed7620ba3a7d495e5a090186df9424b5ab623e"
checksum = "d774fd83f26b24f0805a6ab8b26834a0d06ceac0db517b769b1e4633c96a2057"
dependencies = [
"futures",
"parking_lot 0.12.3",
@@ -4867,9 +4871,9 @@ dependencies = [
[[package]]
name = "errno"
version = "0.3.12"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18"
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
dependencies = [
"libc",
"windows-sys 0.59.0",
@@ -5071,7 +5075,7 @@ version = "25.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"rustc_version 0.4.1",
]
@@ -5891,7 +5895,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbcd2dba93594b227a1f57ee09b8b9da8892c34d55aa332e034a228d0fe6a171"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"gpu-alloc-types",
]
@@ -5901,7 +5905,7 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98ff03b468aa837d70984d55f5d3f846f6ec31fe34bbb97c4f85219caeee1ca4"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
]
[[package]]
@@ -5910,7 +5914,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf29e94d6d243368b7a56caa16bc213e4f9f8ed38c4d9557069527b5d5281ca"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"gpu-descriptor-types",
"hashbrown 0.15.3",
]
@@ -5921,7 +5925,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
]
[[package]]
@@ -6490,7 +6494,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.61.1",
"windows-core 0.61.0",
]
[[package]]
@@ -7201,7 +7205,7 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"libc",
"redox_syscall 0.5.12",
]
@@ -7630,7 +7634,7 @@ version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5637e166ea14be6063a3f8ba5ccb9a4159df7d8f6d61c02fc3d480b1f90dcfcb"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"block",
"core-graphics-types",
"foreign-types 0.5.0",
@@ -7871,7 +7875,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e0ec195e788c95f36b7cf88127d538465fc2f7773e6e47af01834738eab0aee"
dependencies = [
"base64 0.22.1",
"bitflags 2.9.1",
"bitflags 2.9.0",
"btoi",
"byteorder",
"bytes",
@@ -7900,7 +7904,7 @@ checksum = "e536ae46fcab0876853bd4a632ede5df4b1c2527a58f6c5a4150fe86be858231"
dependencies = [
"arrayvec",
"bit-set 0.5.3",
"bitflags 2.9.1",
"bitflags 2.9.0",
"codespan-reporting",
"hexf-parse",
"indexmap 2.9.0",
@@ -7992,7 +7996,7 @@ version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"cfg-if",
"libc",
]
@@ -8003,7 +8007,7 @@ version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"cfg-if",
"cfg_aliases 0.2.1",
"libc",
@@ -8076,7 +8080,7 @@ version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"crossbeam-channel",
"filetime",
"fsevent-sys",
@@ -8590,7 +8594,7 @@ version = "0.10.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"cfg-if",
"foreign-types 0.3.2",
"libc",
@@ -8819,9 +8823,9 @@ dependencies = [
[[package]]
name = "owo-colors"
version = "4.2.1"
version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec"
checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564"
[[package]]
name = "p224"
@@ -9462,7 +9466,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"chrono",
"flate2",
"hex",
@@ -9476,7 +9480,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"chrono",
"hex",
]
@@ -9602,7 +9606,7 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"getopts",
"memchr",
"unicase",
@@ -9869,7 +9873,7 @@ version = "11.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
]
[[package]]
@@ -9991,7 +9995,7 @@ version = "0.5.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
]
[[package]]
@@ -10355,7 +10359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94"
dependencies = [
"base64 0.21.7",
"bitflags 2.9.1",
"bitflags 2.9.0",
"serde",
"serde_derive",
]
@@ -10406,7 +10410,7 @@ version = "0.32.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"fallible-iterator 0.3.0",
"fallible-streaming-iterator",
"hashlink 0.9.1",
@@ -10527,7 +10531,7 @@ version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"errno",
"libc",
"linux-raw-sys 0.4.15",
@@ -10540,7 +10544,7 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"errno",
"libc",
"linux-raw-sys 0.9.4",
@@ -10768,7 +10772,7 @@ version = "13.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"cfg-if",
"clipboard-win",
"fd-lock",
@@ -10961,7 +10965,7 @@ version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"core-foundation 0.9.4",
"core-foundation-sys",
"libc",
@@ -10974,7 +10978,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"core-foundation 0.10.0",
"core-foundation-sys",
"libc",
@@ -11493,9 +11497,9 @@ dependencies = [
[[package]]
name = "sourcemap"
version = "9.2.1"
version = "9.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdee719193ae5c919a3ee43f64c2c0dd87f9b9a451d67918a2a5ec2e3c70561c"
checksum = "dd430118acc9fdd838557649b9b43fd0a78e3834d84a283b466f8e84720d6101"
dependencies = [
"base64-simd 0.8.0",
"bitvec",
@@ -11530,7 +11534,7 @@ version = "0.3.0+sdk-1.3.268.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eda41003dc44290527a59b13432d4a0379379fa074b70174882adfbdfd917844"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
]
[[package]]
@@ -11693,7 +11697,7 @@ dependencies = [
"atoi",
"base64 0.22.1",
"bigdecimal",
"bitflags 2.9.1",
"bitflags 2.9.0",
"byteorder",
"bytes",
"chrono",
@@ -11738,7 +11742,7 @@ dependencies = [
"atoi",
"base64 0.22.1",
"bigdecimal",
"bitflags 2.9.1",
"bitflags 2.9.0",
"byteorder",
"chrono",
"crc",
@@ -11976,7 +11980,7 @@ dependencies = [
"rustc-hash 1.1.0",
"serde",
"siphasher 0.3.11",
"sourcemap 9.2.1",
"sourcemap 9.2.0",
"swc_allocator",
"swc_atoms",
"swc_eq_ignore_macros",
@@ -12018,7 +12022,7 @@ version = "0.118.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6f866d12e4d519052b92a0a86d1ac7ff17570da1272ca0c89b3d6f802cd79df"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"is-macro",
"num-bigint",
"phf",
@@ -12040,7 +12044,7 @@ dependencies = [
"num-bigint",
"once_cell",
"serde",
"sourcemap 9.2.1",
"sourcemap 9.2.0",
"swc_allocator",
"swc_atoms",
"swc_common",
@@ -12104,7 +12108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65f21494e75d0bd8ef42010b47cabab9caaed8f2207570e809f6f4eb51a710d1"
dependencies = [
"better_scoped_tls",
"bitflags 2.9.1",
"bitflags 2.9.0",
"indexmap 2.9.0",
"once_cell",
"phf",
@@ -12372,7 +12376,7 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"byteorder",
"enum-as-inner",
"libc",
@@ -12386,7 +12390,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"byteorder",
"enum-as-inner",
"libc",
@@ -12425,7 +12429,7 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"core-foundation 0.9.4",
"system-configuration-sys 0.6.0",
]
@@ -13308,7 +13312,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e"
dependencies = [
"async-compression",
"bitflags 2.9.1",
"bitflags 2.9.0",
"bytes",
"futures-core",
"http 1.3.1",
@@ -13941,7 +13945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a511192602f7b435b0a241c1947aa743eb7717f20a9195f4b5e8ed1952e01db1"
dependencies = [
"bindgen 0.70.1",
"bitflags 2.9.1",
"bitflags 2.9.0",
"fslock",
"gzip-header",
"home",
@@ -13957,7 +13961,7 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97599c400fc79925922b58303e98fcb8fa88f573379a08ddb652e72cbd2e70f6"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"encoding_rs",
"indexmap 2.9.0",
"num-bigint",
@@ -14248,7 +14252,7 @@ checksum = "d50819ab545b867d8a454d1d756b90cd5f15da1f2943334ca314af10583c9d39"
dependencies = [
"arrayvec",
"bit-vec 0.6.3",
"bitflags 2.9.1",
"bitflags 2.9.0",
"cfg_aliases 0.1.1",
"codespan-reporting",
"document-features",
@@ -14279,7 +14283,7 @@ dependencies = [
"arrayvec",
"ash",
"bit-set 0.5.3",
"bitflags 2.9.1",
"bitflags 2.9.0",
"block",
"cfg_aliases 0.1.1",
"core-graphics-types",
@@ -14317,7 +14321,7 @@ version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1353d9a46bff7f955a680577f34c69122628cc2076e1d6f3a9be6ef00ae793ef"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
"js-sys",
"serde",
"web-sys",
@@ -14397,7 +14401,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"axum",
@@ -14446,7 +14450,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"argon2",
@@ -14555,7 +14559,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -14570,7 +14574,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"chrono",
"serde",
@@ -14583,7 +14587,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"serde",
@@ -14597,7 +14601,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"async-stream",
@@ -14667,7 +14671,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"regex",
"serde",
@@ -14681,7 +14685,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"bytes",
@@ -14704,7 +14708,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -14716,7 +14720,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -14725,7 +14729,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14737,7 +14741,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14749,7 +14753,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"gosyn",
@@ -14761,7 +14765,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14773,7 +14777,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14785,7 +14789,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -14796,7 +14800,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14807,7 +14811,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14818,7 +14822,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -14838,7 +14842,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -14855,7 +14859,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14867,7 +14871,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14885,7 +14889,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"getrandom 0.2.16",
@@ -14909,7 +14913,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14919,7 +14923,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -14952,7 +14956,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -14962,7 +14966,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.491.5"
version = "1.491.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15079,7 +15083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419"
dependencies = [
"windows-collections",
"windows-core 0.61.1",
"windows-core 0.61.0",
"windows-future",
"windows-link",
"windows-numerics",
@@ -15091,7 +15095,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
dependencies = [
"windows-core 0.61.1",
"windows-core 0.61.0",
]
[[package]]
@@ -15120,26 +15124,25 @@ dependencies = [
[[package]]
name = "windows-core"
version = "0.61.1"
version = "0.61.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46ec44dc15085cea82cf9c78f85a9114c463a369786585ad2882d1ff0b0acf40"
checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980"
dependencies = [
"windows-implement 0.60.0",
"windows-interface 0.59.1",
"windows-link",
"windows-result 0.3.3",
"windows-strings 0.4.1",
"windows-result 0.3.2",
"windows-strings 0.4.0",
]
[[package]]
name = "windows-future"
version = "0.2.1"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
checksum = "7a1d6bbefcb7b60acd19828e1bc965da6fcf18a7e39490c5f8be71e54a19ba32"
dependencies = [
"windows-core 0.61.1",
"windows-core 0.61.0",
"windows-link",
"windows-threading",
]
[[package]]
@@ -15220,7 +15223,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
dependencies = [
"windows-core 0.61.1",
"windows-core 0.61.0",
"windows-link",
]
@@ -15230,7 +15233,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
dependencies = [
"windows-result 0.3.3",
"windows-result 0.3.2",
"windows-strings 0.3.1",
"windows-targets 0.53.0",
]
@@ -15246,9 +15249,9 @@ dependencies = [
[[package]]
name = "windows-result"
version = "0.3.3"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b895b5356fc36103d0f64dd1e94dfa7ac5633f1c9dd6e80fe9ec4adef69e09d"
checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252"
dependencies = [
"windows-link",
]
@@ -15264,9 +15267,9 @@ dependencies = [
[[package]]
name = "windows-strings"
version = "0.4.1"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a7ab927b2637c19b3dbe0965e75d8f2d30bdd697a1516191cad2ec4df8fb28a"
checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97"
dependencies = [
"windows-link",
]
@@ -15345,15 +15348,6 @@ dependencies = [
"windows_x86_64_msvc 0.53.0",
]
[[package]]
name = "windows-threading"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
dependencies = [
"windows-link",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
@@ -15541,7 +15535,7 @@ version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags 2.9.1",
"bitflags 2.9.0",
]
[[package]]

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.491.5"
version = "1.491.0"
authors.workspace = true
edition.workspace = true
@@ -32,7 +32,7 @@ members = [
]
[workspace.package]
version = "1.491.5"
version = "1.491.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"

View File

@@ -1 +1 @@
3efa7fa51e9f93f60e141fef5b8b9338528cf955
0a4b3d0ecc9976951517672702e948651806f40b

View File

@@ -1,4 +0,0 @@
-- Add down migration script here
DROP TRIGGER http_trigger_change_trigger ON http_trigger;
DROP FUNCTION notify_http_trigger_change();
DROP SEQUENCE http_trigger_version_seq;

View File

@@ -1,15 +0,0 @@
-- Add up migration script here
CREATE OR REPLACE FUNCTION notify_http_trigger_change()
RETURNS TRIGGER AS $$
BEGIN
PERFORM pg_notify('notify_http_trigger_change', NEW.workspace_id || ':' || NEW.path);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER http_trigger_change_trigger
AFTER INSERT OR UPDATE OR DELETE ON http_trigger
FOR EACH ROW
EXECUTE FUNCTION notify_http_trigger_change();
CREATE SEQUENCE http_trigger_version_seq;

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,3 +0,0 @@
-- Add up migration script here
GRANT ALL ON dependency_map TO windmill_user;
GRANT ALL ON dependency_map TO windmill_admin;

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,9 +0,0 @@
-- Add up migration script here
-- this makes sure that the first time nextval is called, 2 is returned
-- otherwise, `SELECT last_value from http_trigger_version_seq;` would return 1 before and after the first nextval call
-- which would not refresh the routers cache after the first create/update/delete
SELECT setval(
'http_trigger_version_seq',
(SELECT last_value FROM http_trigger_version_seq),
true
);

View File

@@ -810,21 +810,6 @@ Windmill Community Edition {GIT_VERSION}
}
}
},
#[cfg(feature = "http_trigger")]
"notify_http_trigger_change" => {
tracing::info!("HTTP trigger change detected: {}", n.payload());
match windmill_api::http_triggers::refresh_routers(&db).await {
Ok((true, _)) => {
tracing::info!("Refreshed HTTP routers (trigger change)");
},
Ok((false, _)) => {
tracing::warn!("Should have refreshed HTTP routers (trigger change) but did not");
},
Err(err) => {
tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}");
}
};
},
"notify_global_setting_change" => {
tracing::info!("Global setting change detected: {}", n.payload());
match n.payload() {
@@ -1148,10 +1133,6 @@ async fn listen_pg(url: &str) -> Option<PgListener> {
"notify_workspace_envs_change",
"notify_runnable_version_change",
];
#[cfg(feature = "http_trigger")]
channels.push("notify_http_trigger_change");
#[cfg(feature = "cloud")]
channels.push("notify_workspace_premium_change");

View File

@@ -62,11 +62,7 @@ use windmill_common::{
users::truncate_token,
utils::{empty_as_none, now_from_db, rd_string, report_critical_error, Mode},
worker::{
load_env_vars, load_init_bash_from_env, load_whitelist_env_vars_from_env,
load_worker_config, reload_custom_tags_setting, store_pull_query,
store_suspended_pull_query, update_min_version, Connection, WorkerConfig,
DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY,
SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP,
load_env_vars, load_init_bash_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP
},
KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED,
CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
@@ -833,7 +829,11 @@ pub async fn delete_expired_items(db: &DB) -> () {
Ok(mut tx) => {
let deleted_jobs = sqlx::query_scalar!(
"DELETE FROM v2_job_completed c
WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
USING v2_job j
WHERE
created_at <= now() - ($1::bigint::text || ' s')::interval
AND completed_at + ($1::bigint::text || ' s')::interval <= now()
AND c.id = j.id
RETURNING c.id",
job_retention_secs
)

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.491.5
version: 1.491.0
title: Windmill API
contact:
@@ -11028,8 +11028,7 @@ paths:
description: a config
content:
application/json:
schema:
$ref: "#/components/schemas/Configs"
schema: {}
/configs/update/{name}:
post:
@@ -13184,37 +13183,6 @@ components:
code_completion_model:
$ref: "#/components/schemas/AIProviderModel"
Alert:
type: object
properties:
name:
type: string
tags_to_monitor:
type: array
items:
type: string
jobs_num_threshold:
type: integer
alert_cooldown_seconds:
type: integer
alert_time_threshold_seconds:
type: integer
required:
- name
- tags_to_monitor
- jobs_num_threshold
- alert_cooldown_seconds
- alert_time_threshold_seconds
Configs:
type: object
nullable: true
properties:
alerts:
type: array
items:
$ref: '#/components/schemas/Alert'
Script:
type: object
properties:

View File

@@ -28,7 +28,7 @@ pub fn workspaced_service(
use windmill_worker::JobCompletedSender;
let (job_completed_tx, _job_completed_rx) =
JobCompletedSender::new(&Connection::Sql(db.clone()), 10);
JobCompletedSender::new(&Connection::Sql(db.clone()), 100);
let router = Router::new();

View File

@@ -29,14 +29,14 @@ pub enum RawBody {
Empty,
}
#[derive(Debug, Clone, Serialize)]
#[derive(Clone, Serialize)]
#[serde(untagged)]
pub enum Body {
HashMap(HashMap<String, Box<RawValue>>),
NoHashMap(Box<RawValue>),
}
#[derive(Debug, Clone, Default)]
#[derive(Clone, Default)]
pub struct WebhookArgsMetadata {
pub raw_string: Option<String>,
pub headers: HashMap<String, Box<RawValue>>,
@@ -51,7 +51,7 @@ pub struct RawWebhookArgs {
pub metadata: WebhookArgsMetadata,
}
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct WebhookArgs {
pub body: Body,
pub metadata: WebhookArgsMetadata,

View File

@@ -14,18 +14,11 @@ use {
};
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
use {
crate::gcp_triggers_ee::{
manage_google_subscription, process_google_push_request, validate_jwt_token,
CreateUpdateConfig, SubscriptionMode,
},
axum::extract::Request,
http::HeaderMap,
use crate::gcp_triggers_ee::{
manage_google_subscription, process_google_push_request, validate_jwt_token,
CreateUpdateConfig, SubscriptionMode,
};
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
use windmill_common::utils::empty_as_none;
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
use windmill_common::auth::aws::AwsAuthResourceType;
@@ -33,7 +26,12 @@ use windmill_common::auth::aws::AwsAuthResourceType;
feature = "http_trigger",
all(feature = "enterprise", feature = "gcp_trigger")
))]
use {serde::de::DeserializeOwned, windmill_common::error::Error};
use {
axum::extract::Request,
http::HeaderMap,
serde::de::DeserializeOwned,
windmill_common::{error::Error, utils::empty_as_none},
};
#[cfg(all(feature = "enterprise", feature = "kafka"))]
use crate::kafka_triggers_ee::KafkaTriggerConfigConnection;

View File

@@ -782,32 +782,10 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
.execute(db)
.await?;
});
run_windmill_migration!("job_completed_completed_at", db, |tx| {
sqlx::query!(
"CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_job_completed_completed_at ON v2_job_completed (completed_at DESC)"
)
.execute(db)
.await?;
});
run_windmill_migration!("alerts_by_workspace", db, |tx| {
sqlx::query!(
"CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_by_workspace ON alerts (workspace_id);"
)
.execute(db)
.await?;
});
run_windmill_migration!("remove_redundant_log_file_index", db, |tx| {
sqlx::query!("DROP INDEX CONCURRENTLY IF EXISTS log_file_hostname_log_ts_idx")
.execute(db)
.await?;
});
Ok(())
}
#[derive(Clone, Debug, Hash, Eq, PartialEq)]
#[derive(Clone, Debug)]
pub struct ApiAuthed {
pub email: String,
pub username: String,

View File

@@ -17,7 +17,7 @@ use crate::{
pub struct RawHttpTriggerArgs(pub RawWebhookArgs);
#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Hash, Eq, PartialEq)]
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
@@ -56,7 +56,7 @@ where
}
}
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct HttpTriggerArgs(pub WebhookArgs);
impl RawHttpTriggerArgs {

View File

@@ -414,7 +414,7 @@ pub enum Encoding {
Base64Uri,
Hex,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize)]
pub struct SignatureAuthenticationMethod {
algorithm: HmacAlgorithm,
encoding: Encoding,
@@ -426,20 +426,20 @@ pub struct SignatureConfigData<'config> {
secret_key: &'config str,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize)]
pub struct SignatureAuthentication {
signature_provider: WebhookType,
secret_key: String,
authentication_config: Option<SignatureAuthenticationMethod>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize)]
pub struct BasicAuthAuthentication {
pub username: String,
pub password: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize)]
pub struct ApiKeyAuthentication {
api_key_header: String,
api_key_secret: String,
@@ -558,7 +558,7 @@ pub fn verify_hmac_signature(
Ok(())
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum AuthenticationMethod {
Signature(SignatureAuthentication),

View File

@@ -4,7 +4,7 @@ use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs};
use crate::job_helpers_ee::get_workspace_s3_resource;
use crate::resources::try_get_resource_from_db_as;
use crate::trigger_helpers::{get_runnable_format, RunnableId};
use crate::utils::{non_empty_str, ExpiringCacheEntry};
use crate::utils::non_empty_str;
use crate::{
auth::{AuthCache, OptTokened},
db::{ApiAuthed, DB},
@@ -24,14 +24,11 @@ use axum::{
#[cfg(feature = "parquet")]
use http::header::IF_NONE_MATCH;
use http::{HeaderMap, StatusCode};
use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize};
use sql_builder::{bind::Bind, SqlBuilder};
use sqlx::prelude::FromRow;
use sqlx::PgTransaction;
use std::borrow::Cow;
use std::{collections::HashMap, sync::Arc};
use tokio::sync::{RwLock, RwLockReadGuard};
use tower_http::cors::CorsLayer;
use windmill_audit::{audit_ee::audit_log, ActionKind};
use windmill_common::error::Error;
@@ -286,16 +283,6 @@ fn validate_authentication_method(
Ok(())
}
async fn increase_trigger_version_and_commit(mut tx: PgTransaction<'_>) -> error::Result<()> {
sqlx::query!("SELECT nextval('http_trigger_version_seq')",)
.fetch_one(&mut *tx)
.await?;
tx.commit().await?;
Ok(())
}
async fn create_trigger(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -395,7 +382,7 @@ async fn create_trigger(
)
.await?;
increase_trigger_version_and_commit(tx).await?;
tx.commit().await?;
Ok((StatusCode::CREATED, format!("{}", ct.path)))
}
@@ -551,7 +538,7 @@ async fn update_trigger(
)
.await?;
increase_trigger_version_and_commit(tx).await?;
tx.commit().await?;
Ok(path.to_string())
}
@@ -585,7 +572,7 @@ async fn delete_trigger(
)
.await?;
increase_trigger_version_and_commit(tx).await?;
tx.commit().await?;
Ok(format!("HTTP trigger {path} deleted"))
}
@@ -698,8 +685,8 @@ async fn exists_route(
Ok(Json(exists))
}
#[derive(Debug, Deserialize, Clone)]
pub struct TriggerRoute {
#[derive(Debug, Deserialize)]
struct TriggerRoute {
path: String,
script_path: String,
is_flow: bool,
@@ -717,145 +704,6 @@ pub struct TriggerRoute {
raw_string: bool,
}
pub struct RoutersCache {
routers: HashMap<HttpMethod, matchit::Router<TriggerRoute>>,
version: i64,
}
lazy_static::lazy_static! {
static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100);
static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<crate::http_trigger_auth::AuthenticationMethod>> = Cache::new(100);
static ref HTTP_ROUTERS_CACHE: RwLock<RoutersCache> = RwLock::new(RoutersCache {
routers: HashMap::new(),
version: 0,
});
}
pub async fn refresh_routers_loop(
db: &DB,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> () {
match refresh_routers(db).await {
Ok(_) => {
tracing::info!("Loaded HTTP routers");
}
Err(err) => {
tracing::error!("Error loading HTTP routers: {err:#}");
}
};
let db = db.clone();
tokio::spawn(async move {
loop {
tokio::select! {
_ = killpill_rx.recv() => {
break;
}
_ = tokio::time::sleep(std::time::Duration::from_secs(60)) => {
match refresh_routers(&db).await {
Ok((true, _)) => {
tracing::info!("Refreshed HTTP routers");
}
Err(err) => {
tracing::error!("Error refreshing HTTP routers: {err:#}");
}
_ => {}
}
}
}
}
});
}
pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>), Error> {
let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",)
.fetch_one(db)
.await?;
let routers_cache = HTTP_ROUTERS_CACHE.read().await;
if routers_cache.version == 0 || version > routers_cache.version {
drop(routers_cache);
let mut routers = HashMap::new();
for http_method in [
HttpMethod::Get,
HttpMethod::Post,
HttpMethod::Put,
HttpMethod::Patch,
HttpMethod::Delete,
] {
let triggers = sqlx::query_as!(
TriggerRoute,
r#"
SELECT
path,
script_path,
is_flow,
route_path,
authentication_resource_path,
workspace_id,
is_async,
authentication_method AS "authentication_method: _",
edited_by,
email,
static_asset_config AS "static_asset_config: _",
wrap_body,
raw_string,
workspaced_route,
is_static_website
FROM
http_trigger
WHERE
http_method = $1
"#,
&http_method as &HttpMethod
)
.fetch_all(db)
.await?;
let mut router = matchit::Router::new();
for trigger in triggers {
let full_path = if trigger.workspaced_route || *CLOUD_HOSTED {
format!("/{}/{}", trigger.workspace_id, trigger.route_path)
} else {
format!("/{}", trigger.route_path)
};
if trigger.is_static_website {
router
.insert(format!("{}/*wm_subpath", full_path), trigger.clone())
.unwrap_or_else(|e| {
tracing::warn!(
"Failed to consider http trigger route {}/*wm_subpath: {:?}",
full_path,
e,
);
});
}
router
.insert(full_path.clone(), trigger.clone())
.unwrap_or_else(|e| {
tracing::warn!(
"Failed to consider http trigger route {}: {:?}",
full_path,
e,
);
});
}
routers.insert(http_method, router);
}
let mut routers_cache = HTTP_ROUTERS_CACHE.write().await;
*routers_cache = RoutersCache { routers, version };
Ok((true, routers_cache.downgrade()))
} else {
tracing::debug!("No HTTP routers refresh needed");
Ok((false, routers_cache))
}
}
async fn get_http_route_trigger(
route_path: &str,
auth_cache: &Arc<AuthCache>,
@@ -865,30 +713,111 @@ async fn get_http_route_trigger(
method: &http::Method,
) -> error::Result<(TriggerRoute, String, HashMap<String, String>, ApiAuthed)> {
let http_method: HttpMethod = method.try_into()?;
let requested_path = format!("/{}", route_path);
let routers_cache = HTTP_ROUTERS_CACHE.read().await;
let routers_cache = if routers_cache.routers.is_empty() {
tracing::warn!("HTTP routers are not loaded, loading from db");
let (_, routers_cache) = refresh_routers(db).await?;
routers_cache
let (mut triggers, route_path) = if *CLOUD_HOSTED {
let mut splitted = route_path.split("/");
let w_id = splitted.next().ok_or_else(|| {
error::Error::BadRequest("Missing workspace id in route path".to_string())
})?;
let route_path = StripPath(splitted.collect::<Vec<_>>().join("/"));
let triggers = sqlx::query_as!(
TriggerRoute,
r#"
SELECT
path,
script_path,
is_flow,
route_path,
workspace_id,
is_async,
authentication_method AS "authentication_method: _",
edited_by,
email,
static_asset_config AS "static_asset_config: _",
wrap_body,
raw_string,
workspaced_route,
is_static_website,
authentication_resource_path
FROM
http_trigger
WHERE
workspace_id = $1 AND
http_method = $2
"#,
w_id,
http_method as HttpMethod
)
.fetch_all(db)
.await?;
(triggers, route_path)
} else {
routers_cache
let triggers = sqlx::query_as!(
TriggerRoute,
r#"
SELECT
path,
script_path,
is_flow,
route_path,
authentication_resource_path,
workspace_id,
is_async,
authentication_method AS "authentication_method: _",
edited_by,
email,
static_asset_config AS "static_asset_config: _",
wrap_body,
raw_string,
workspaced_route,
is_static_website
FROM
http_trigger
WHERE
http_method = $1
"#,
http_method as HttpMethod
)
.fetch_all(db)
.await?;
(triggers, StripPath(route_path.to_string()))
};
let router = routers_cache
.routers
.get(&http_method)
.ok_or(error::Error::internal_err(
"HTTP routers could not be loaded".to_string(),
))?;
let mut router = matchit::Router::new();
let trigger_match = router.at(requested_path.as_str()).ok();
for (idx, trigger) in triggers.iter().enumerate() {
let route_path = match trigger.workspaced_route {
true => format!("{}/{}", &trigger.workspace_id, &trigger.route_path),
_ => trigger.route_path.clone(),
};
if trigger.is_static_website {
router
.insert(format!("/{}/*wm_subpath", route_path), idx)
.unwrap_or_else(|e| {
tracing::warn!(
"Failed to consider http trigger route {}: {:?}",
route_path,
e,
);
});
}
router
.insert(format!("/{}", route_path), idx)
.unwrap_or_else(|e| {
tracing::warn!(
"Failed to consider http trigger route {}: {:?}",
route_path,
e,
);
});
}
let matchit::Match { value: trigger, params } =
not_found_if_none(trigger_match, "Trigger", requested_path.as_str())?;
let requested_path = format!("/{}", route_path.0);
let trigger_idx = router.at(requested_path.as_str()).ok();
let matchit::Match { value: trigger_idx, params } =
not_found_if_none(trigger_idx, "Trigger", requested_path.as_str())?;
let trigger = triggers.remove(trigger_idx.to_owned());
let params: HashMap<String, String> = params
.iter()
@@ -905,49 +834,25 @@ async fn get_http_route_trigger(
};
if let Some(authed) = opt_authed {
// check that the user has access to the trigger
let cache_key = (
trigger.workspace_id.clone(),
trigger.path.clone(),
authed.clone(),
);
let exists = match HTTP_ACCESS_CACHE.get(&cache_key) {
Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => {
tracing::debug!("HTTP access cache hit for trigger {}", trigger.path);
true
}
_ => {
tracing::debug!("HTTP access cache miss for trigger {}", trigger.path);
let mut tx = user_db.begin(&authed).await?;
let exists = sqlx::query_scalar!(
r#"
SELECT EXISTS(
SELECT 1
FROM
http_trigger
WHERE
workspace_id = $1 AND
path = $2
)
"#,
trigger.workspace_id,
trigger.path
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
if exists {
HTTP_ACCESS_CACHE.insert(
cache_key,
ExpiringCacheEntry {
value: (),
expiry: std::time::Instant::now()
+ std::time::Duration::from_secs(10),
},
);
}
exists
}
};
let mut tx = user_db.begin(&authed).await?;
let exists = sqlx::query_scalar!(
r#"
SELECT EXISTS(
SELECT 1
FROM
http_trigger
WHERE
workspace_id = $1 AND
path = $2
)
"#,
trigger.workspace_id,
trigger.path
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
tx.commit().await?;
if exists {
Some(authed.display_username().to_owned())
} else {
@@ -971,7 +876,7 @@ async fn get_http_route_trigger(
)
.await?;
Ok((trigger.clone(), route_path.to_string(), params, authed))
Ok((trigger, route_path.0, params, authed))
}
async fn route_job(
@@ -996,15 +901,7 @@ async fn route_job(
.map_err(|e| e.into_response())?;
let args = args
.process_args(
&authed,
&db,
&trigger.workspace_id,
match trigger.authentication_method {
AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true,
_ => trigger.raw_string,
},
)
.process_args(&authed, &db, &trigger.workspace_id, trigger.raw_string)
.await
.map_err(|e| e.into_response())?;
@@ -1023,45 +920,31 @@ async fn route_job(
}
};
let cache_key = (
trigger.workspace_id.clone(),
resource_path.clone(),
authed.clone(),
);
let authentication_method =
try_get_resource_from_db_as::<crate::http_trigger_auth::AuthenticationMethod>(
authed.clone(),
Some(user_db.clone()),
&db,
&resource_path,
&trigger.workspace_id,
)
.await
.map_err(|e| e.into_response())?;
let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) {
Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => {
tracing::debug!("HTTP auth method cache hit for trigger {}", trigger.path);
cache_entry.value
}
_ => {
tracing::debug!("HTTP auth method cache miss for trigger {}", trigger.path);
let auth_method = try_get_resource_from_db_as::<
crate::http_trigger_auth::AuthenticationMethod,
>(
authed.clone(),
Some(user_db.clone()),
&db,
&resource_path,
&trigger.workspace_id,
)
.await
.map_err(|e| e.into_response())?;
HTTP_AUTH_CACHE.insert(
cache_key,
ExpiringCacheEntry {
value: auth_method.clone(),
expiry: std::time::Instant::now() + std::time::Duration::from_secs(60),
},
);
auth_method
}
};
let raw_payload = args.0.metadata.raw_string.as_ref();
let raw_payload = args
.0
.metadata
.raw_string
.as_ref()
.map(|raw_payload| serde_json::from_str::<String>(raw_payload))
.transpose()
.map_err(|e| {
windmill_common::error::Error::SerdeJson { location: e.to_string(), error: e }
.into_response()
})?;
let response = authentication_method
.authenticate_http_request(&headers, raw_payload)
.authenticate_http_request(&headers, raw_payload.as_ref())
.map_err(|e| e.into_response())?;
if let Some(response) = response {

View File

@@ -85,14 +85,13 @@ mod http_trigger_args;
#[cfg(feature = "http_trigger")]
mod http_trigger_auth;
#[cfg(feature = "http_trigger")]
pub mod http_triggers;
mod http_triggers;
mod indexer_ee;
mod inputs;
mod integration;
#[cfg(feature = "postgres_trigger")]
mod postgres_triggers;
mod approvals;
#[cfg(feature = "enterprise")]
mod apps_ee;
#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))]
@@ -121,11 +120,12 @@ mod scripts;
mod service_logs;
mod settings;
mod slack_approvals;
mod approvals;
mod teams_approvals_ee;
#[cfg(feature = "smtp")]
mod smtp_server_ee;
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
mod sqs_triggers_ee;
mod teams_approvals_ee;
mod trigger_helpers;
mod static_assets;
@@ -406,12 +406,6 @@ pub async fn run_server(
Router::new()
};
#[cfg(feature = "http_trigger")]
{
let http_killpill_rx = killpill_rx.resubscribe();
http_triggers::refresh_routers_loop(&db, http_killpill_rx).await;
}
let postgres_triggers_service = {
#[cfg(feature = "postgres_trigger")]
{

View File

@@ -4,7 +4,6 @@ use serde_json::value::RawValue;
use std::collections::HashMap;
use windmill_common::{
error::Result,
flows::FlowModuleValue,
get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path,
scripts::{ScriptHash, ScriptLang},
worker::to_raw_value,
@@ -39,6 +38,12 @@ struct ScriptInfo {
schema: Option<sqlx::types::Json<PartialSchema>>,
}
struct FlowInfo {
has_preprocessor: Option<bool>,
is_v1_preprocessor: Option<bool>,
schema: Option<sqlx::types::Json<PartialSchema>>,
}
#[derive(Debug, Deserialize)]
struct PropertyDefinition {
r#type: Option<String>,
@@ -99,8 +104,9 @@ async fn get_script_info(
.await
}
fn runnable_format_from_schema_without_preprocessor(
fn runnable_format_from_schema(
trigger_kind: &TriggerKind,
has_preprocessor: bool,
schema: Option<sqlx::types::Json<PartialSchema>>,
) -> RunnableFormat {
match trigger_kind {
@@ -113,7 +119,7 @@ fn runnable_format_from_schema_without_preprocessor(
})
}) =>
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false }
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor }
}
TriggerKind::Kafka | TriggerKind::Nats
if schema.as_ref().is_some_and(|schema| {
@@ -123,46 +129,18 @@ fn runnable_format_from_schema_without_preprocessor(
.is_some_and(|properties| properties.keys().any(|key| key == "msg"))
}) =>
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false }
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor }
}
_ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false },
_ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor },
}
}
fn runnable_format_from_preprocessor_args(
args: Option<Vec<windmill_parser::Arg>>,
) -> RunnableFormat {
if let Some(args) = args {
if args.iter().any(|arg| arg.name == "wm_trigger")
|| (args.len() > 0 && args.iter().all(|arg| arg.name != "event"))
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
} else {
RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true }
}
} else {
RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true }
}
}
enum PreprocessorInfo {
Preprocessor { content: String, language: ScriptLang },
NoPreprocessor { schema: Option<sqlx::types::Json<PartialSchema>> },
}
#[derive(Debug, Deserialize)]
struct FlowInfo {
preprocessor_module: Option<sqlx::types::Json<FlowModuleValue>>,
schema: Option<sqlx::types::Json<PartialSchema>>,
}
pub async fn get_runnable_format(
runnable_id: RunnableId,
workspace_id: &str,
db: &DB,
trigger_kind: &TriggerKind,
) -> Result<RunnableFormat> {
let (key, preprocessor_info) = match runnable_id {
match runnable_id {
RunnableId::FlowPath(path) => {
let FlowVersionInfo { version, .. } =
get_latest_flow_version_info_for_path(db, workspace_id, &path, true).await?;
@@ -179,10 +157,11 @@ pub async fn get_runnable_format(
let flow_info = sqlx::query_as!(
FlowInfo,
"SELECT
value->'preprocessor_module'->'value' as \"preprocessor_module: _\",
value->'preprocessor_module' IS NOT NULL as has_preprocessor,
value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,
schema as \"schema: _\"
FROM flow
WHERE workspace_id = $1
WHERE workspace_id = $1
AND path = $2",
workspace_id,
path
@@ -190,40 +169,18 @@ pub async fn get_runnable_format(
.fetch_one(db)
.await?;
if let Some(preprocessor_module) = flow_info.preprocessor_module {
match preprocessor_module.0 {
FlowModuleValue::RawScript { content, language, .. } => {
(key, PreprocessorInfo::Preprocessor { content, language })
}
FlowModuleValue::Script { path, hash, .. } => {
let hash = if let Some(hash) = hash {
hash.0
} else {
let script_hash =
get_latest_deployed_hash_for_path(db, workspace_id, &path).await?;
script_hash.hash
};
let script_info = get_script_info(db, workspace_id, hash).await?;
(
key,
PreprocessorInfo::Preprocessor {
content: script_info.content,
language: script_info.language,
},
)
}
_ => {
return Err(windmill_common::error::Error::internal_err(
"Unsupported preprocessor module".to_string(),
));
}
}
let has_preprocessor = flow_info.has_preprocessor.unwrap_or(false);
let is_v1_preprocessor = flow_info.is_v1_preprocessor.unwrap_or(false);
let runnable_format = if has_preprocessor && is_v1_preprocessor {
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
} else {
(
key,
PreprocessorInfo::NoPreprocessor { schema: flow_info.schema },
)
}
runnable_format_from_schema(trigger_kind, has_preprocessor, flow_info.schema)
};
RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
Ok(runnable_format)
}
RunnableId::ScriptId(script_id) => {
let hash = script_id.get_script_hash(workspace_id, db).await?;
@@ -237,59 +194,47 @@ pub async fn get_runnable_format(
let script_info = get_script_info(db, workspace_id, hash).await?;
if script_info.has_preprocessor.unwrap_or(false) {
(
key,
PreprocessorInfo::Preprocessor {
content: script_info.content,
language: script_info.language,
},
)
} else {
(
key,
PreprocessorInfo::NoPreprocessor { schema: script_info.schema },
)
}
}
};
let has_preprocessor = script_info.has_preprocessor.unwrap_or(false);
let runnable_format = match preprocessor_info {
PreprocessorInfo::Preprocessor { content, language } => {
let args = match language {
ScriptLang::Bun
| ScriptLang::Bunnative
| ScriptLang::Deno
| ScriptLang::Nativets => {
let args = windmill_parser_ts::parse_deno_signature(
&content,
true,
false,
Some("preprocessor".to_string()),
)?;
Some(args.args)
let runnable_format = if has_preprocessor {
let args = match script_info.language {
ScriptLang::Bun
| ScriptLang::Bunnative
| ScriptLang::Deno
| ScriptLang::Nativets => {
let args = windmill_parser_ts::parse_deno_signature(
&script_info.content,
true,
false,
Some("preprocessor".to_string()),
)?;
Some(args.args)
}
ScriptLang::Python3 => {
let args = windmill_parser_py::parse_python_signature(
&script_info.content,
Some("preprocessor".to_string()),
false,
)?;
Some(args.args)
}
_ => None,
};
if args.is_some_and(|args| args.iter().any(|arg| arg.name == "wm_trigger")) {
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
} else {
runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema)
}
ScriptLang::Python3 => {
let args = windmill_parser_py::parse_python_signature(
&content,
Some("preprocessor".to_string()),
false,
)?;
Some(args.args)
}
_ => None,
} else {
runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema)
};
runnable_format_from_preprocessor_args(args)
}
PreprocessorInfo::NoPreprocessor { schema } => {
runnable_format_from_schema_without_preprocessor(trigger_kind, schema)
}
};
RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
Ok(runnable_format)
Ok(runnable_format)
}
}
}
#[allow(dead_code)]

View File

@@ -8,8 +8,6 @@
#![allow(non_snake_case)]
use quick_cache::sync::Cache;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::Duration;
@@ -22,8 +20,7 @@ use crate::utils::{
generate_instance_wide_unique_username, get_instance_username_or_create_pending,
};
use crate::{
auth::ExpiringAuthCache, db::DB, utils::require_super_admin, webhook_util::WebhookShared,
COOKIE_DOMAIN, IS_SECURE,
db::DB, utils::require_super_admin, webhook_util::WebhookShared, COOKIE_DOMAIN, IS_SECURE,
};
use argon2::{Argon2, PasswordHash, PasswordVerifier};
use axum::{
@@ -217,10 +214,6 @@ pub async fn fetch_api_authed(
fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await
}
lazy_static::lazy_static! {
static ref API_AUTHED_CACHE: Cache<(String,String,String), ExpiringAuthCache> = Cache::new(300);
}
#[allow(unused)]
pub async fn fetch_api_authed_from_permissioned_as(
permissioned_as: String,
@@ -229,43 +222,18 @@ pub async fn fetch_api_authed_from_permissioned_as(
db: &DB,
username_override: Option<String>,
) -> error::Result<ApiAuthed> {
let key = (w_id.to_string(), permissioned_as.clone(), email.clone());
let mut api_authed = match API_AUTHED_CACHE.get(&key) {
Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => {
tracing::debug!("API authed cache hit for user {}", email);
expiring_authed.authed
}
_ => {
tracing::debug!("API authed cache miss for user {}", email);
let authed =
fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
let api_authed = ApiAuthed {
username: authed.username,
email: email,
is_admin: authed.is_admin,
is_operator: authed.is_operator,
groups: authed.groups,
folders: authed.folders,
scopes: authed.scopes,
username_override: None,
};
API_AUTHED_CACHE.insert(
key,
ExpiringAuthCache {
authed: api_authed.clone(),
expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(),
},
);
api_authed
}
};
api_authed.username_override = username_override;
Ok(api_authed)
let authed =
fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
Ok(ApiAuthed {
username: authed.username,
email: email,
is_admin: authed.is_admin,
is_operator: authed.is_operator,
groups: authed.groups,
folders: authed.folders,
scopes: authed.scopes,
username_override: username_override,
})
}
#[derive(FromRow, Serialize)]

View File

@@ -5,6 +5,7 @@
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{body::Body, response::Response};
use regex::Regex;
use serde::{Deserialize, Deserializer};
@@ -414,10 +415,3 @@ pub async fn acknowledge_all_critical_alerts(
);
Ok("All unacknowledged critical alerts acknowledged".to_string())
}
#[cfg(feature = "http_trigger")]
#[derive(Clone)]
pub struct ExpiringCacheEntry<T> {
pub value: T,
pub expiry: std::time::Instant,
}

View File

@@ -140,7 +140,6 @@ pub struct CanceledBy {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JobCompleted {
pub job: Arc<MiniPulledJob>,
pub preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
pub result: Arc<Box<RawValue>>,
pub result_columns: Option<Vec<String>>,
pub mem_peak: i32,
@@ -2665,7 +2664,7 @@ async fn concurrency_key(db: &Pool<Postgres>, id: &Uuid) -> windmill_common::err
)
}
pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
// Save this value to avoid parsing twice
let workspaced = x.as_str().replace("$workspace", workspace_id).to_string();
if RE_ARG_TAG.is_match(&workspaced) {
@@ -2703,6 +2702,7 @@ pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> Strin
.trim_matches('"')
.to_string()
};
tracing::error!("arg_value: {}", arg_value);
interpolated =
interpolated.replace(format!("$args[{}]", arg_name).as_str(), &arg_value);
}
@@ -3887,13 +3887,11 @@ pub async fn push<'c, 'd>(
let cache_ttl = value.cache_ttl.map(|x| x as i32);
let custom_concurrency_key = value.concurrency_key.clone();
let concurrency_time_window_s = value.concurrency_time_window_s;
let mut concurrent_limit = value.concurrent_limit;
let concurrent_limit = value.concurrent_limit;
if !apply_preprocessor {
value.preprocessor_module = None;
} else {
tag = None;
concurrent_limit = None;
preprocessed = Some(false);
}
@@ -4175,7 +4173,27 @@ pub async fn push<'c, 'd>(
};
if concurrent_limit.is_some() {
insert_concurrency_key(workspace_id, &args, &script_path, job_kind, custom_concurrency_key, &mut tx, job_id).await?;
let concurrency_key = custom_concurrency_key
.map(|x| interpolate_args(x, &args, workspace_id))
.unwrap_or(fullpath_with_workspace(
workspace_id,
script_path.as_ref(),
&job_kind,
));
sqlx::query!(
"WITH inserted_concurrency_counter AS (
INSERT INTO concurrency_counter (concurrency_id, job_uuids)
VALUES ($1, '{}'::jsonb)
ON CONFLICT DO NOTHING
)
INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)",
concurrency_key,
job_id,
)
.execute(&mut *tx)
.warn_after_seconds(3)
.await
.map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?;
}
let stringified_args = if *JOB_ARGS_AUDIT_LOGS {
@@ -4193,7 +4211,6 @@ pub async fn push<'c, 'd>(
Some("preprocessor") => Some(false),
_ => None,
});
let job_authed = match authed {
Some(authed)
@@ -4416,31 +4433,6 @@ pub async fn push<'c, 'd>(
Ok((job_id, tx))
}
pub async fn insert_concurrency_key<'d, 'c>(workspace_id: &str, args: &PushArgs<'d>, script_path: &Option<String>, job_kind: JobKind, custom_concurrency_key: Option<String>, tx: &mut Transaction<'c, Postgres>, job_id: Uuid) -> Result<(), Error> {
let concurrency_key = custom_concurrency_key
.map(|x| interpolate_args(x, args, workspace_id))
.unwrap_or(fullpath_with_workspace(
workspace_id,
script_path.as_ref(),
&job_kind,
));
sqlx::query!(
"WITH inserted_concurrency_counter AS (
INSERT INTO concurrency_counter (concurrency_id, job_uuids)
VALUES ($1, '{}'::jsonb)
ON CONFLICT DO NOTHING
)
INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)",
concurrency_key,
job_id,
)
.execute(&mut **tx)
.warn_after_seconds(3)
.await
.map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?;
Ok(())
}
pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value {
let reason = job
.canceled_reason

View File

@@ -206,7 +206,7 @@ fn do_bigquery_inner<'a>(
convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
s3.upload(stream.boxed()).await?;
return Ok(to_raw_value(&s3.to_return_s3_obj()));
return Ok(to_raw_value(&s3.object_key));
}
Ok(to_raw_value(&rows))

View File

@@ -1591,7 +1591,7 @@ pub struct S3ModeWorkerData {
}
impl S3ModeWorkerData {
pub async fn upload<S>(&self, stream: S) -> error::Result<()>
pub async fn upload<S>(&self, stream: S) -> error::Result<reqwest::Response>
where
S: futures::stream::TryStream + Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
@@ -1606,14 +1606,6 @@ impl S3ModeWorkerData {
)
.await
}
pub fn to_return_s3_obj(&self) -> windmill_common::s3_helpers::S3Object {
windmill_common::s3_helpers::S3Object {
s3: self.object_key.clone(),
storage: self.storage.clone(),
..Default::default()
}
}
}
pub fn s3_mode_args_to_worker_data(

View File

@@ -187,14 +187,14 @@ pub async fn handle_dedicated_process(
let result = Arc::new(result);
append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await;
if line.starts_with("wm_res[success]:") {
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap()
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
} else {
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap()
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
}
},
Err(e) => {
tracing::error!("Could not deserialize job result `{line}`: {e:?}");
job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap();
job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap();
},
};
logs = init_log.clone();

View File

@@ -214,7 +214,7 @@ pub async fn do_mssql(
let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
s3.upload(stream.boxed()).await?;
Ok(to_raw_value(&s3.to_return_s3_obj()))
Ok(serde_json::value::to_raw_value(&s3.object_key)?)
} else {
let stream = prepared_query.query(&mut client).await.map_err(to_anyhow)?;
let results = stream.into_results().await.map_err(to_anyhow)?;

View File

@@ -105,7 +105,7 @@ fn do_mysql_inner<'a>(
let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
s3.upload(stream.boxed()).await?;
Ok(to_raw_value(&s3.to_return_s3_obj()))
Ok(serde_json::value::to_raw_value(&s3.object_key)?)
} else {
let rows: Vec<Row> = conn
.lock()

View File

@@ -159,7 +159,7 @@ pub fn do_oracledb_inner<'a>(
if let Some(s3) = s3 {
let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
s3.upload(stream.boxed()).await?;
return Ok(to_raw_value(&s3.to_return_s3_obj()));
return Ok(serde_json::value::to_raw_value(&s3.object_key)?);
} else {
let rows: Vec<_> = rows_stream.collect().await;
Ok(to_raw_value(

View File

@@ -124,7 +124,7 @@ fn do_postgresql_inner<'a>(
let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
s3.upload(stream.boxed()).await?;
return Ok(to_raw_value(&s3.to_return_s3_obj()));
return Ok(serde_json::value::to_raw_value(&s3.object_key)?);
} else {
let rows = client
.query_raw(&query, query_params)

View File

@@ -29,13 +29,12 @@ use windmill_common::{
use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
use windmill_queue::{
append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob,
WrappedError,
append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError,
};
use serde_json::{json, value::RawValue};
use tokio::task::JoinHandle;
use tokio::{sync::broadcast, task::JoinHandle};
use windmill_queue::{add_completed_job, add_completed_job_error};
@@ -44,8 +43,7 @@ use crate::{
common::{error_to_value, read_result, save_in_cache},
otel_ee::add_root_flow_job_to_otlp,
worker_flow::update_flow_status_after_job_completion,
AuthedClient, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult,
UpdateFlow, INIT_SCRIPT_TAG,
AuthedClient, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG,
};
async fn process_jc(
@@ -120,7 +118,7 @@ async fn process_jc(
}
pub fn start_background_processor(
job_completed_rx: JobCompletedReceiver,
job_completed_rx: flume::Receiver<SendResult>,
job_completed_sender: JobCompletedSender,
same_worker_queue_size: Arc<AtomicU16>,
job_completed_processor_is_done: Arc<AtomicBool>,
@@ -129,14 +127,13 @@ pub fn start_background_processor(
worker_dir: String,
same_worker_tx: SameWorkerSender,
worker_name: String,
mut killpill_rx: broadcast::Receiver<()>,
killpill_tx: KillpillSender,
is_dedicated_worker: bool,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut has_been_killed = false;
let JobCompletedReceiver { bounded_rx, mut killpill_rx, unbounded_rx } = job_completed_rx;
#[cfg(feature = "benchmark")]
let mut infos = BenchmarkInfo::new();
@@ -147,21 +144,15 @@ pub fn start_background_processor(
//if we have been killed, we want to drain the queue of jobs
while let Some(sr) = {
if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 {
unbounded_rx
job_completed_rx
.try_recv()
.ok()
.map(JobCompletedRx::JobCompleted)
.or_else(|| bounded_rx.try_recv().ok().map(JobCompletedRx::JobCompleted))
} else {
tokio::select! {
biased;
result = unbounded_rx.recv_async() => {
result = job_completed_rx.recv_async() => {
result.ok().map(JobCompletedRx::JobCompleted)
}
result = bounded_rx.recv_async() => {
result.ok().map(JobCompletedRx::JobCompleted)
}
_ = killpill_rx.recv() => {
Some(JobCompletedRx::Killpill)
}
@@ -216,7 +207,7 @@ pub fn start_background_processor(
infos.add_iter(bench, true);
}
}
JobCompletedRx::JobCompleted(SendResult::UpdateFlow(UpdateFlow {
JobCompletedRx::JobCompleted(SendResult::UpdateFlow {
flow,
w_id,
success,
@@ -224,7 +215,7 @@ pub fn start_background_processor(
worker_dir,
stop_early_override,
token,
})) => {
}) => {
// let r;
tracing::info!(parent_flow = %flow, "updating flow status");
if let Err(e) = update_flow_status_after_job_completion(
@@ -275,11 +266,29 @@ pub fn start_background_processor(
async fn send_job_completed(
job_completed_tx: JobCompletedSender,
jc: JobCompleted,
job: Arc<MiniPulledJob>,
result: Arc<Box<RawValue>>,
result_columns: Option<Vec<String>>,
mem_peak: i32,
canceled_by: Option<CanceledBy>,
success: bool,
cached_res_path: Option<String>,
token: &str,
duration: Option<i64>,
) {
let jc = JobCompleted {
job,
result,
result_columns,
mem_peak,
canceled_by,
success,
cached_res_path,
token: token.to_string(),
duration,
};
job_completed_tx
.send_job(jc, true)
.send_job(jc)
.with_context(windmill_common::otel_ee::otel_ctx())
.await
.expect("send job completed")
@@ -294,28 +303,37 @@ pub async fn process_result(
canceled_by: Option<CanceledBy>,
cached_res_path: Option<String>,
token: &str,
result_columns: Option<Vec<String>>,
preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
column_order: Option<Vec<String>>,
new_args: Option<HashMap<String, Box<RawValue>>>,
conn: &Connection,
duration: Option<i64>,
) -> error::Result<bool> {
match result {
Ok(result) => {
Ok(r) => {
// Update script args to preprocessed args
if let Connection::Sql(db) = conn {
if let Some(preprocessed_args) = new_args {
sqlx::query!(
"UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2",
Json(preprocessed_args) as Json<HashMap<String, Box<RawValue>>>,
job.id
)
.execute(db)
.await?;
}
}
send_job_completed(
job_completed_tx,
JobCompleted {
job,
preprocessed_args,
result,
result_columns,
mem_peak,
canceled_by,
success: true,
cached_res_path,
token: token.to_string(),
duration,
},
job,
r,
column_order,
mem_peak,
canceled_by,
true,
cached_res_path,
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
@@ -366,18 +384,15 @@ pub async fn process_result(
send_job_completed(
job_completed_tx,
JobCompleted {
job,
result: Arc::new(to_raw_value(&error_value)),
result_columns: None,
preprocessed_args: None,
mem_peak,
canceled_by,
success: false,
cached_res_path,
token: token.to_string(),
duration,
},
job,
Arc::new(to_raw_value(&error_value)),
None,
mem_peak,
canceled_by,
false,
cached_res_path,
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
@@ -453,7 +468,6 @@ pub async fn process_completed_job(
canceled_by,
duration,
result_columns,
preprocessed_args,
..
}: JobCompleted,
client: &AuthedClient,
@@ -478,7 +492,6 @@ pub async fn process_completed_job(
if job.flow_step_id.as_deref() == Some("preprocessor") {
// Do this before inserting to `v2_job_completed` for backwards compatibility
// when we set `flow_status->_metadata->preprocessed_args` to true.
sqlx::query!(
r#"UPDATE v2_job SET
args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb,
@@ -493,15 +506,6 @@ pub async fn process_completed_job(
"error while deleting args of preprocessing step: {e:#}"
))
})?;
} else if let Some(preprocessed_args) = preprocessed_args {
// Update script args to preprocessed args
sqlx::query!(
"UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2",
Json(preprocessed_args) as Json<HashMap<String, Box<RawValue>>>,
job.id
)
.execute(db)
.await?;
}
add_time!(bench, "pre add_completed_job");

View File

@@ -256,7 +256,7 @@ fn do_snowflake_inner<'a>(
rows_stream.map(|r| serde_json::value::to_value(&r?).map_err(to_anyhow));
let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?;
s3.upload(stream.boxed()).await?;
Ok(to_raw_value(&s3.to_return_s3_obj()))
Ok(to_raw_value(&s3.object_key))
} else {
let rows = rows_stream
.collect::<Vec<_>>()

View File

@@ -527,7 +527,7 @@ impl AuthedClient {
object_key: String,
storage: Option<String>,
body: S,
) -> error::Result<()>
) -> error::Result<Response>
where
S: futures::stream::TryStream + Send + 'static,
S::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
@@ -537,8 +537,7 @@ impl AuthedClient {
if let Some(storage) = storage {
query.push(("storage", storage));
}
let response = self
.force_client
self.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.post(format!(
@@ -559,12 +558,7 @@ impl AuthedClient {
.send()
.await
.context(format!("Sent upload_s3_file request",))
.map_err(error::Error::from)?;
match response.status().as_u16() {
200u16 => Ok(()),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
}
.map_err(error::Error::from)
}
}
@@ -574,44 +568,26 @@ pub struct SameWorkerSender(pub Sender<SameWorkerPayload>, pub Arc<AtomicU16>);
#[allow(dead_code)]
#[derive(Clone)]
pub enum JobCompletedSender {
Sql(SqlJobCompletedSender),
Sql(flume::Sender<SendResult>, broadcast::Sender<()>),
Http(HttpClient),
NeverUsed,
}
#[derive(Clone)]
pub struct SqlJobCompletedSender {
sender: flume::Sender<SendResult>,
unbounded_sender: flume::Sender<SendResult>,
killpill_tx: broadcast::Sender<()>,
}
pub struct JobCompletedReceiver {
pub bounded_rx: flume::Receiver<SendResult>,
pub killpill_rx: broadcast::Receiver<()>,
pub unbounded_rx: flume::Receiver<SendResult>,
}
impl JobCompletedReceiver {
pub fn clone(&self) -> Self {
Self {
bounded_rx: self.bounded_rx.clone(),
killpill_rx: self.killpill_rx.resubscribe(),
unbounded_rx: self.unbounded_rx.clone(),
}
}
}
impl JobCompletedSender {
pub fn new(conn: &Connection, buffer_size: u8) -> (Self, Option<JobCompletedReceiver>) {
pub fn new(
conn: &Connection,
buffer_size: usize,
) -> (
Self,
Option<(flume::Receiver<SendResult>, broadcast::Receiver<()>)>,
) {
match conn {
Connection::Sql(_) => {
let (sender, receiver) = flume::bounded::<SendResult>(buffer_size as usize);
let (unbounded_sender, unbounded_rx) = flume::unbounded::<SendResult>();
let (killpill_tx, killpill_rx) = broadcast::channel::<()>(10);
let (sender, receiver) = flume::bounded::<SendResult>(buffer_size);
let (killpill_tx, killpill_rx) = broadcast::channel::<()>(buffer_size);
(
Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, killpill_tx }),
Some(JobCompletedReceiver { bounded_rx: receiver, killpill_rx, unbounded_rx }),
Self::Sql(sender, killpill_tx),
Some((receiver, killpill_rx)),
)
}
Connection::Http(client) => (Self::Http(client.clone()), None),
@@ -621,20 +597,14 @@ impl JobCompletedSender {
(Self::NeverUsed, None)
}
pub async fn send_job(&self, jc: JobCompleted, wait_for_capacity: bool) -> anyhow::Result<()> {
pub async fn send_job(&self, jc: JobCompleted) -> anyhow::Result<()> {
match self {
Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => {
if wait_for_capacity {
sender
} else {
unbounded_sender
}
Self::Sql(sender, _) => sender
.send_async(SendResult::JobCompleted(jc))
.await
.map_err(|_e| {
anyhow::anyhow!("Failed to send job completed to background processor")
})
}
}),
Self::Http(client) => {
crate::agent_workers::send_result(client, jc).await?;
Ok(())
@@ -648,19 +618,9 @@ impl JobCompletedSender {
}
}
pub async fn send(
&self,
send_result: SendResult,
wait_for_capacity: bool,
) -> Result<(), flume::SendError<SendResult>> {
pub async fn send(&self, send_result: SendResult) -> Result<(), flume::SendError<SendResult>> {
match self {
Self::Sql(SqlJobCompletedSender { sender, unbounded_sender, .. }) => {
if wait_for_capacity {
sender.send_async(send_result).await
} else {
unbounded_sender.send_async(send_result).await
}
}
Self::Sql(sender, _) => sender.send_async(send_result).await,
Self::Http(_) => {
tracing::error!("Sending job completed to http client, this should not happen");
Ok(())
@@ -676,7 +636,7 @@ impl JobCompletedSender {
pub async fn kill(&self) -> Result<(), broadcast::error::SendError<()>> {
match self {
Self::Sql(SqlJobCompletedSender { killpill_tx, .. }) => {
Self::Sql(_, killpill_tx) => {
tracing::info!("Sending killpill to bg processors");
killpill_tx.send(())?;
Ok(())
@@ -1097,7 +1057,7 @@ pub async fn run_worker(
let (same_worker_tx, mut same_worker_rx) = mpsc::channel::<SameWorkerPayload>(5);
let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 10);
let (job_completed_tx, job_completed_rx) = JobCompletedSender::new(&conn, 3);
let same_worker_queue_size = Arc::new(AtomicU16::new(0));
let same_worker_tx = SameWorkerSender(same_worker_tx, same_worker_queue_size.clone());
@@ -1105,19 +1065,22 @@ pub async fn run_worker(
Arc::new(AtomicBool::new(matches!(conn, Connection::Http(_))));
let send_result = match (conn, job_completed_rx) {
(Connection::Sql(db), Some(job_completed_receiver)) => Some(start_background_processor(
job_completed_receiver,
job_completed_tx.clone(),
same_worker_queue_size.clone(),
job_completed_processor_is_done.clone(),
base_internal_url.to_string(),
db.clone(),
worker_dir.clone(),
same_worker_tx.clone(),
worker_name.clone(),
killpill_tx.clone(),
is_dedicated_worker,
)),
(Connection::Sql(db), Some((job_completed_rx, bg_killpill_rx))) => {
Some(start_background_processor(
job_completed_rx,
job_completed_tx.clone(),
same_worker_queue_size.clone(),
job_completed_processor_is_done.clone(),
base_internal_url.to_string(),
db.clone(),
worker_dir.clone(),
same_worker_tx.clone(),
worker_name.clone(),
bg_killpill_rx,
killpill_tx.clone(),
is_dedicated_worker,
))
}
_ => None,
};
@@ -1519,21 +1482,17 @@ pub async fn run_worker(
if matches!(job.kind, JobKind::Noop) {
add_time!(bench, "send job completed START");
job_completed_tx
.send_job(
JobCompleted {
preprocessed_args: None,
job: Arc::new(job.job()),
success: true,
result: Arc::new(empty_result()),
result_columns: None,
mem_peak: 0,
cached_res_path: None,
token: "".to_string(),
canceled_by: None,
duration: None,
},
true,
)
.send_job(JobCompleted {
job: Arc::new(job.job()),
success: true,
result: Arc::new(empty_result()),
result_columns: None,
mem_peak: 0,
cached_res_path: None,
token: "".to_string(),
canceled_by: None,
duration: None,
})
.await
.expect("send job completed END");
add_time!(bench, "sent job completed");
@@ -1747,25 +1706,21 @@ pub async fn run_worker(
}
Connection::Http(_) => {
job_completed_tx
.send_job(
JobCompleted {
preprocessed_args: None,
job: arc_job.clone(),
result: Arc::new(
windmill_common::worker::to_raw_value(
&error_to_value(err),
),
.send_job(JobCompleted {
job: arc_job.clone(),
result: Arc::new(
windmill_common::worker::to_raw_value(
&error_to_value(err),
),
result_columns: None,
mem_peak: 0,
canceled_by: None,
success: false,
cached_res_path: None,
token: authed_client.token.clone(),
duration: None,
},
false,
)
),
result_columns: None,
mem_peak: 0,
canceled_by: None,
success: false,
cached_res_path: None,
token: authed_client.token.clone(),
duration: None,
})
.await
.expect("send job completed");
}
@@ -1930,17 +1885,15 @@ async fn queue_init_bash_maybe<'c>(
pub enum SendResult {
JobCompleted(JobCompleted),
UpdateFlow(UpdateFlow),
}
pub struct UpdateFlow {
pub flow: Uuid,
pub w_id: String,
pub success: bool,
pub result: Box<RawValue>,
pub worker_dir: String,
pub stop_early_override: Option<bool>,
pub token: String,
UpdateFlow {
flow: Uuid,
w_id: String,
success: bool,
result: Box<RawValue>,
worker_dir: String,
stop_early_override: Option<bool>,
token: String,
},
}
async fn do_nativets(
@@ -2112,21 +2065,17 @@ async fn handle_queued_job(
append_logs(&job.id, &job.workspace_id, logs, conn).await;
}
job_completed_tx
.send_job(
JobCompleted {
preprocessed_args: None,
job,
result,
result_columns: None,
mem_peak: 0,
canceled_by: None,
success: true,
cached_res_path: None,
token: client.token.clone(),
duration: None,
},
true,
)
.send_job(JobCompleted {
job,
result,
result_columns: None,
mem_peak: 0,
canceled_by: None,
success: true,
cached_res_path: None,
token: client.token.clone(),
duration: None,
})
.await
.expect("send job completed");

View File

@@ -13,10 +13,8 @@ use std::time::Duration;
use crate::common::{cached_result_path, save_in_cache};
use crate::js_eval::{eval_timeout, IdContext};
use crate::worker_utils::get_tag_and_concurrency;
use crate::{
AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, UpdateFlow,
KEEP_JOB_DIR,
AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, KEEP_JOB_DIR,
};
use anyhow::Context;
use futures::TryFutureExt;
@@ -60,8 +58,8 @@ use windmill_queue::flow_status::Step;
use windmill_queue::schedule::get_schedule_opt;
use windmill_queue::{
add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job,
handle_maybe_scheduled_job, insert_concurrency_key, interpolate_args, CanceledBy,
MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError,
handle_maybe_scheduled_job, CanceledBy, MiniPulledJob, PushArgs, PushIsolationLevel,
SameWorkerPayload, WrappedError,
};
type DB = sqlx::Pool<sqlx::Postgres>;
@@ -163,10 +161,6 @@ pub async fn update_flow_status_after_job_completion(
add_time!(bench, "update flow status internal END");
return Ok(None);
}
UpdateFlowStatusAfterJobCompletion::PreprocessingStep => {
add_time!(bench, "update flow status preprocessing step END");
return Ok(None);
}
}
}
}
@@ -176,7 +170,6 @@ pub enum UpdateFlowStatusAfterJobCompletion {
Done(Arc<MiniPulledJob>),
NotDone,
NonLastParallelBranch,
PreprocessingStep,
}
pub struct RecUpdateFlowStatusAfterJobCompletion {
flow: uuid::Uuid,
@@ -432,6 +425,41 @@ pub async fn update_flow_status_after_job_completion_internal(
_ => false,
};
if matches!(module_step, Step::PreprocessorStep) {
sqlx::query!(
"WITH job_result AS (
SELECT result
FROM v2_job_completed
WHERE id = $1
)
UPDATE v2_job
SET args = COALESCE(
CASE
WHEN job_result.result IS NULL THEN NULL
WHEN jsonb_typeof(job_result.result) = 'object'
THEN job_result.result
WHEN jsonb_typeof(job_result.result) = 'null'
THEN NULL
ELSE jsonb_build_object('value', job_result.result)
END,
'{}'::jsonb
),
preprocessed = TRUE
FROM job_result
WHERE v2_job.id = $2;
",
job_id_for_status,
flow
)
.execute(db)
.await
.map_err(|e| {
Error::internal_err(format!(
"error while updating args in preprocessing step: {e:#}"
))
})?;
}
let mut tx = db.begin().await?;
add_time!(bench, "process module status START");
@@ -981,128 +1009,6 @@ pub async fn update_flow_status_after_job_completion_internal(
.ok_or_else(|| Error::internal_err(format!("requiring flow to be in the queue")))?;
tx.commit().await?;
if matches!(module_step, Step::PreprocessorStep) {
let tag_and_concurrency_key = get_tag_and_concurrency(&flow, db).await;
let require_args = tag_and_concurrency_key.as_ref().is_some_and(|x| {
x.tag.as_ref().is_some_and(|t| t.contains("$args"))
|| x.concurrency_key
.as_ref()
.is_some_and(|ck| ck.contains("$args"))
});
let mut tag = tag_and_concurrency_key
.as_ref()
.map(|x| x.tag.clone())
.flatten();
let concurrency_key = tag_and_concurrency_key
.as_ref()
.map(|x| x.concurrency_key.clone())
.flatten();
let concurrent_limit = tag_and_concurrency_key
.as_ref()
.map(|x| x.concurrent_limit)
.flatten();
let concurrency_time_window_s = tag_and_concurrency_key
.as_ref()
.map(|x| x.concurrency_time_window_s)
.flatten();
if require_args {
let args = sqlx::query_scalar!(
"SELECT result as \"result: Json<HashMap<String, Box<RawValue>>>\"
FROM v2_job_completed
WHERE id = $1",
job_id_for_status
)
.fetch_one(db)
.await
.map_err(|e| {
Error::internal_err(format!("error while fetching preprocessing args: {e:#}"))
})?;
let args_hm = args.unwrap_or_default().0;
let args = PushArgs::from(&args_hm);
if let Some(ck) = concurrency_key {
let mut tx = db.begin().await?;
insert_concurrency_key(
&flow_job.workspace_id,
&args,
&flow_job.runnable_path,
JobKind::Flow,
Some(ck),
&mut tx,
flow,
)
.await?;
tx.commit().await?;
}
if let Some(t) = tag {
tag = Some(interpolate_args(t, &args, &flow_job.workspace_id));
}
} else if let Some(ck) = concurrency_key {
let mut tx = db.begin().await?;
insert_concurrency_key(
&flow_job.workspace_id,
&PushArgs::from(&HashMap::new()),
&flow_job.runnable_path,
JobKind::Flow,
Some(ck),
&mut tx,
flow,
)
.await?;
tx.commit().await?;
}
// let tag = tag_and_concurrency_key.and_then(|tc| tc.tag.map(|t| interpolate_args(t.clone(), &args, &workspace_id)));
// let concurrency_key = tag_and_concurrency_key.and_then(|tc| tc.concurrency_key.map(|ck| interpolate_args(&ck, &args, &workspace_id)));
sqlx::query!(
"WITH job_result AS (
SELECT result
FROM v2_job_completed
WHERE id = $1
),
updated_queue AS (
UPDATE v2_job_queue
SET running = false,
tag = COALESCE($3, tag)
WHERE id = $2
)
UPDATE v2_job
SET
tag = COALESCE($3, tag),
concurrent_limit = COALESCE($4, concurrent_limit),
concurrency_time_window_s = COALESCE($5, concurrency_time_window_s),
args = COALESCE(
CASE
WHEN job_result.result IS NULL THEN NULL
WHEN jsonb_typeof(job_result.result) = 'object'
THEN job_result.result
WHEN jsonb_typeof(job_result.result) = 'null'
THEN NULL
ELSE jsonb_build_object('value', job_result.result)
END,
'{}'::jsonb
),
preprocessed = TRUE
FROM job_result
WHERE v2_job.id = $2;
",
job_id_for_status,
flow,
tag,
concurrent_limit,
concurrency_time_window_s,
)
.execute(db)
.await
.map_err(|e| {
Error::internal_err(format!(
"error while updating args in preprocessing step: {e:#}"
))
})?;
if success {
return Ok(UpdateFlowStatusAfterJobCompletion::PreprocessingStep);
}
}
let job_root = flow_job
.flow_innermost_root_job
.map(|x| x.to_string())
@@ -1633,47 +1539,22 @@ pub async fn handle_flow(
);
}
}
let mut rec = PushNextFlowJobRec { flow_job: flow_job, status: status };
loop {
let PushNextFlowJobRec { flow_job, status } = rec;
let next = push_next_flow_job(
flow_job,
status,
let mut rec = Some(PushNextFlowJobRec { flow_job: flow_job, status: status });
while let Some(nrec) = rec {
rec = push_next_flow_job(
nrec.flow_job,
nrec.status,
flow,
db,
client,
last_result.clone(),
same_worker_tx.clone(),
worker_dir,
job_completed_tx.clone(),
worker_name,
)
.warn_after_seconds(10)
.await?;
match next {
PushNextFlowJob::Rec(nrec) => {
tracing::info!("recursively pushing next flow job {}", nrec.flow_job.id);
rec = nrec;
}
PushNextFlowJob::Done(update_flow) => {
if let Some(update_flow) = update_flow {
tracing::info!(
"sending flow status update {} with success {} to job completed channel",
update_flow.flow,
update_flow.success
);
job_completed_tx
.send(SendResult::UpdateFlow(update_flow), false)
.warn_after_seconds(3)
.await
.map_err(|e| {
Error::internal_err(format!(
"error sending update flow message to job completed channel: {e:#}"
))
})?;
}
break;
}
}
}
Ok(())
@@ -1719,10 +1600,6 @@ lazy_static::lazy_static! {
pub static ref EHM: HashMap<String, Box<RawValue>> = HashMap::new();
}
enum PushNextFlowJob {
Rec(PushNextFlowJobRec),
Done(Option<UpdateFlow>),
}
struct PushNextFlowJobRec {
flow_job: Arc<MiniPulledJob>,
status: FlowStatus,
@@ -1738,8 +1615,9 @@ async fn push_next_flow_job(
last_job_result: Option<Arc<Box<RawValue>>>,
same_worker_tx: SameWorkerSender,
worker_dir: &str,
job_completed_tx: JobCompletedSender,
worker_name: &str,
) -> error::Result<PushNextFlowJob> {
) -> error::Result<Option<PushNextFlowJobRec>> {
let job_root = flow_job
.flow_innermost_root_job
.map(|x| x.to_string())
@@ -1772,20 +1650,30 @@ async fn push_next_flow_job(
// if this is an empty module of if the module has already been completed, successfully, update the parent flow
if flow.modules.is_empty() || matches!(status_module, FlowStatusModule::Success { .. }) {
return Ok(PushNextFlowJob::Done(Some(UpdateFlow {
flow: flow_job.id,
success: true,
result: if flow.modules.is_empty() {
to_raw_value(arc_flow_job_args.as_ref())
} else {
// it has to be an empty for loop event
serde_json::from_str("[]").unwrap()
},
stop_early_override: None,
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})));
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: true,
result: if flow.modules.is_empty() {
to_raw_value(arc_flow_job_args.as_ref())
} else {
// it has to be an empty for loop event
serde_json::from_str("[]").unwrap()
},
stop_early_override: None,
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.warn_after_seconds(3)
.await
.map_err(|e| {
Error::internal_err(format!(
"error sending update flow message to job completed channel: {e:#}"
))
})?;
return Ok(None);
}
if matches!(step, Step::Step(0)) {
@@ -1826,21 +1714,28 @@ async fn push_next_flow_job(
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(", ");
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: true,
result: serde_json::from_str(
&format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""),
)
.unwrap(),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.warn_after_seconds(3)
.await
.map_err(|e| {
Error::internal_err(format!(
"error sending update flow message to job completed channel: {e:#}"
))
})?;
return Ok(PushNextFlowJob::Done(Some(
UpdateFlow {
flow: flow_job.id,
success: true,
result: serde_json::from_str(
&format!("\"not allowed to overlap with {overlapping_str}, scheduling next iteration\""),
)
.unwrap(),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
}
)));
return Ok(None);
}
}
}
@@ -1861,15 +1756,25 @@ async fn push_next_flow_job(
.warn_after_seconds(3)
.await?;
if skip {
return Ok(PushNextFlowJob::Done(Some(UpdateFlow {
flow: flow_job.id,
success: true,
result: serde_json::from_str("\"stopped early\"").unwrap(),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})));
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: true,
result: serde_json::from_str("\"stopped early\"").unwrap(),
stop_early_override: Some(true),
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.warn_after_seconds(3)
.await
.map_err(|e| {
Error::internal_err(format!(
"error sending update flow message to job completed channel: {e:#}"
))
})?;
return Ok(None);
}
}
}
@@ -2127,7 +2032,7 @@ async fn push_next_flow_job(
.await?;
tx.commit().warn_after_seconds(3).await?;
return Ok(PushNextFlowJob::Done(None));
return Ok(None);
/* cancelled or we're WaitingForEvents but we don't have enough messages (timed out) */
} else {
@@ -2175,15 +2080,25 @@ async fn push_next_flow_job(
.warn_after_seconds(3)
.await;
return Ok(PushNextFlowJob::Done(Some(UpdateFlow {
flow: flow_job.id,
success: false,
result: to_raw_value(&result),
stop_early_override: None,
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})));
job_completed_tx
.send(SendResult::UpdateFlow {
flow: flow_job.id,
success: false,
result: to_raw_value(&result),
stop_early_override: None,
w_id: flow_job.workspace_id.clone(),
worker_dir: worker_dir.to_string(),
token: client.token.clone(),
})
.warn_after_seconds(3)
.await
.map_err(|e| {
Error::internal_err(format!(
"error sending update flow message to job completed channel: {e:#}"
))
})?;
return Ok(None);
}
}
}
@@ -2522,13 +2437,13 @@ async fn push_next_flow_job(
if let Some(status) = status {
// // flow is reprocessed by the worker in a state where the module has completed successfully.
return Ok(PushNextFlowJob::Rec(PushNextFlowJobRec {
return Ok(Some(PushNextFlowJobRec {
flow_job: flow_job,
status: status,
}));
} else {
return Err(Error::BadRequest(
"impossible to parse new flow status after applying inner flows".to_string(),
"impossible to parse new flow status after applying innr flows".to_string(),
));
}
}
@@ -2742,8 +2657,8 @@ async fn push_next_flow_job(
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}");
let tag = if !matches!(step, Step::PreprocessorStep)
&& (flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id))
let tag = if flow_job.tag == "flow"
|| flow_job.tag == format!("flow-{}", flow_job.workspace_id)
{
payload_tag.tag.clone()
} else {
@@ -3041,7 +2956,7 @@ async fn push_next_flow_job(
.await
.map_err(to_anyhow)?;
}
return Ok(PushNextFlowJob::Done(None));
return Ok(None);
}
// async fn jump_to_next_step(

View File

@@ -3,14 +3,13 @@ use tracing::Instrument;
use uuid::Uuid;
use windmill_common::{
agent_workers::{PingJobStatus, PingJobStatusResponse},
cache,
worker::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
update_worker_ping_main_loop_query, Connection, Ping, PingType, WORKER_CONFIG,
WORKER_GROUP,
},
KillpillSender, DB,
KillpillSender,
};
use crate::{
@@ -321,71 +320,3 @@ pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname:
}
}
}
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct TagAndConcurrencyKey {
pub tag: Option<String>,
pub concurrency_key: Option<String>,
pub concurrent_limit: Option<i32>,
pub concurrency_time_window_s: Option<i32>,
pub version: Option<i64>,
}
pub async fn get_tag_and_concurrency(job_id: &Uuid, db: &DB) -> Option<TagAndConcurrencyKey> {
let r = sqlx::query_as!(
TagAndConcurrencyKey,
"
WITH j AS (
SELECT
raw_flow->>'concurrency_key' as concurrency_key,
raw_flow->>'concurrency_time_window_s' as concurrency_time_window_s,
raw_flow->>'concurrency_limit' as concurrent_limit,
runnable_path,
runnable_id as version FROM v2_job
WHERE id = $1
)
SELECT tag, j.concurrency_key, j.concurrency_time_window_s::int, j.concurrent_limit::int, j.version
FROM flow, j
WHERE path = j.runnable_path
",
job_id
)
.fetch_optional(db)
.await
.ok()
.flatten();
if let Some(tag_and_concurrency_key) = r {
if tag_and_concurrency_key.concurrency_key.as_ref().is_some()
|| tag_and_concurrency_key.version.as_ref().is_none()
{
return Some(tag_and_concurrency_key);
} else {
let version = tag_and_concurrency_key.version.unwrap();
let r = cache::flow::fetch_version_lite(db, version).await;
let flow = match r {
Ok(data) => Ok(data),
Err(_) => cache::flow::fetch_version(db, version).await,
};
let flow_value = flow.map(|f| f.value().clone()).ok();
let concurrency_key = flow_value
.as_ref()
.map(|fv| fv.concurrency_key.clone())
.flatten();
let concurrent_limit = flow_value.as_ref().map(|fv| fv.concurrent_limit).flatten();
let concurrent_time_window_s = flow_value
.as_ref()
.map(|fv| fv.concurrency_time_window_s)
.flatten();
Some(TagAndConcurrencyKey {
tag: tag_and_concurrency_key.tag,
concurrency_key,
concurrent_limit,
concurrency_time_window_s: concurrent_time_window_s,
version: None,
})
}
} else {
None
}
}

View File

@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.491.5";
export const VERSION = "v1.491.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({

View File

@@ -63,7 +63,7 @@ export {
// }
// });
export const VERSION = "1.491.5";
export const VERSION = "1.491.0";
const command = new Command()
.name("wmill")
@@ -94,7 +94,6 @@ const command = new Command()
"Specify headers to use for all requests. e.g: \"HEADERS='h1: v1, h2: v2'\""
)
.version(VERSION)
.versionOption(false)
.command("init", "Bootstrap a windmill project with a wmill.yaml file")
.action(async () => {
if (await Deno.stat("wmill.yaml").catch(() => null)) {
@@ -135,34 +134,15 @@ const command = new Command()
.command("worker-groups", workerGroups)
.command("workers", workers)
.command("queues", queues)
.command("version --version", "Show version information")
.command("version", "Show version information")
.action(async (opts) => {
console.log("CLI version: " + VERSION);
try {
const provider = new NpmProvider({ package: "windmill-cli" });
const versions = await provider.getVersions("windmill-cli");
if (versions.latest !== VERSION) {
console.log(
`CLI is outdated. Latest version ${versions.latest} is available. Run \`wmill upgrade\` to update.`
);
} else {
console.log("CLI is up to date");
}
} catch (e) {
console.warn(
`Cannot fetch latest CLI version on npmjs to check if up-to-date: ${e}`
);
}
console.log("CLI build against " + VERSION);
const workspace = await getActiveWorkspace(opts as GlobalOptions);
if (workspace) {
try {
const backendVersion = await fetchVersion(workspace.remote);
console.log("Backend Version: " + backendVersion);
} catch (e) {
console.warn("Cannot fetch backend version: " + e);
}
const backendVersion = await fetchVersion(workspace.remote);
console.log("Backend Version: " + backendVersion);
} else {
console.warn(
console.log(
"Cannot fetch backend version: no active workspace selected, choose one to pick a remote to fetch version of"
);
}

View File

@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.491.5",
"version": "1.491.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.491.5",
"version": "1.491.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {

View File

@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.491.5",
"version": "1.491.0",
"scripts": {
"dev": "vite dev",
"build": "vite build",

View File

@@ -69,7 +69,7 @@
? truncateContent.substring(
s3LogPrefixes[prefixIndex]?.length,
end == -1 ? undefined : end + 1
)
)
: undefined
}
@@ -109,7 +109,7 @@
? truncatedContent.substring(
truncatedContent.substring(1).indexOf('\n') + 2,
truncatedContent.length
)
)
: truncatedContent
)
export function scrollToBottom() {

View File

@@ -2,7 +2,7 @@
import { onMount } from 'svelte'
import { Drawer, DrawerContent, Button } from './common'
import QueueMetricsDrawerInner from './QueueMetricsDrawerInner.svelte'
import { ConfigService, type Alert } from '$lib/gen'
import { ConfigService } from '$lib/gen'
import Section from './Section.svelte'
import { sendUserToast } from '$lib/toast'
import { Pencil, Trash, Check, PlusCircle, SaveIcon } from 'lucide-svelte'
@@ -21,6 +21,14 @@
}
}
type Alert = {
name: string
tags_to_monitor: string[]
jobs_num_threshold: number
alert_cooldown_seconds: number
alert_time_threshold_seconds: number
}
let drawer: Drawer
export function openDrawer() {
drawer?.openDrawer()
@@ -47,8 +55,8 @@
async function fetchConfig() {
try {
const response = await ConfigService.getConfig({ name: configName })
alerts = response?.alerts || []
const response = (await ConfigService.getConfig({ name: configName })) as { alerts: Alert[] }
alerts = response.alerts || []
originalAlerts = JSON.parse(JSON.stringify(alerts))
} catch (error) {
console.error('Failed to fetch config:', error)
@@ -290,9 +298,7 @@
<input
type="text"
bind:value={newTag}
placeholder={workerTags.length === alert.tags_to_monitor.length
? 'All tags already added'
: 'Add tag from dropdown'}
placeholder="{workerTags.length === alert.tags_to_monitor.length ? 'All tags already added' : 'Add tag from dropdown' }"
on:input={(e) => filterTags(e)}
disabled={workerTags.length === alert.tags_to_monitor.length}
class="p-1 flex-grow mr-1"
@@ -315,15 +321,15 @@
>
{#each filteredTags as tag}
{#if !alert.tags_to_monitor.includes(tag)}
<li>
<button
type="button"
class="w-full text-left p-2 cursor-pointer hover:bg-slate-200 dark:hover:bg-slate-700"
on:click={() => addTag(index, tag)}
>
{tag}
</button>
</li>
<li>
<button
type="button"
class="w-full text-left p-2 cursor-pointer hover:bg-slate-200 dark:hover:bg-slate-700"
on:click={() => addTag(index, tag)}
>
{tag}
</button>
</li>
{/if}
{/each}
</ul>

View File

@@ -41,9 +41,9 @@
export let fromWorkspaceSettings: boolean = false
export let readOnlyMode: boolean
export let initialFileKey: { s3: string; storage?: string } | undefined = undefined
let initialFileKeyInternalCopy: { s3: string; storage?: string }
export let selectedFileKey: { s3: string; storage?: string } | undefined = undefined
export let initialFileKey: { s3: string } | undefined = undefined
let initialFileKeyInternalCopy: { s3: string }
export let selectedFileKey: { s3: string } | undefined = undefined
export let folderOnly = false
export let regexFilter: RegExp | undefined = undefined
@@ -298,7 +298,7 @@
deletionModalOpen = false
}
sendUserToast(`${fileKey} deleted from S3 bucket`)
selectedFileKey = { s3: '', storage }
selectedFileKey = { s3: '' }
const currentPage = page
await clearAndLoadFiles()
for (let i = 0; i < currentPage; i++) {
@@ -359,7 +359,7 @@
moveModalOpen = false
}
sendUserToast(`${srcFileKey} moved to ${destFileKey}`)
selectedFileKey = { s3: destFileKey!, storage }
selectedFileKey = { s3: destFileKey! }
await clearAndLoadFiles()
await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
}
@@ -397,7 +397,7 @@
await clearAndLoadFiles()
if (selectedFileKey !== undefined) {
if (allFilesByKey[selectedFileKey.s3] === undefined) {
selectedFileKey = { s3: '', storage }
selectedFileKey = { s3: '' }
} else {
loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
}
@@ -421,8 +421,7 @@
if (item.type === 'folder') {
if (folderOnly) {
selectedFileKey = {
s3: item_key,
storage
s3: item_key
}
}
if (toggleCollapsed) {
@@ -457,8 +456,7 @@
displayedFileKeys = displayedFileKeys.sort()
} else {
selectedFileKey = {
s3: item_key,
storage
s3: item_key
}
loadFileMetadataPlusPreviewAsync(selectedFileKey.s3)
}
@@ -843,7 +841,7 @@
on:close={async (evt) => {
uploadModalOpen = false
if (evt.detail !== undefined && evt.detail !== null) {
selectedFileKey = { s3: evt.detail, storage }
selectedFileKey = { s3: evt.detail }
await clearAndLoadFiles()
loadFileMetadataPlusPreviewAsync(evt.detail)
}

View File

@@ -199,22 +199,24 @@
<span class="mr-2 w-8 font-mono">{selected == res ? '-' : '+'}</span>
{res}
</button>
<div class={selected == res ? 'border-t' : ''}>
<SubGridEditor
{id}
visible={render && index === selectedIndex}
subGridId={`${id}-${index}`}
class={twMerge(css?.container?.class, 'wm-tabs-container')}
style={css?.container?.style}
containerHeight={componentContainerHeight - (titleBarHeight * tabs.length + 40)}
on:focus={() => {
if (!$connectingInput.opened) {
$selectedComponent = [id]
handleTabSelection()
}
}}
/>
</div>
{#if selected == res}
<div class="border-t">
<SubGridEditor
{id}
visible={render && index === selectedIndex}
subGridId={`${id}-${index}`}
class={twMerge(css?.container?.class, 'wm-tabs-container')}
style={css?.container?.style}
containerHeight={componentContainerHeight - (titleBarHeight * tabs.length + 40)}
on:focus={() => {
if (!$connectingInput.opened) {
$selectedComponent = [id]
handleTabSelection()
}
}}
/>
</div>
{/if}
</div>
{/each}
</div>

View File

@@ -262,7 +262,6 @@
}
let selectedItem: any
async function handleKeydown(event: KeyboardEvent) {
if ((!isMac() ? event.ctrlKey : event.metaKey) && event.key === 'k') {
event.preventDefault()
@@ -642,11 +641,11 @@
<div class="w-4/12 overflow-y-auto max-h-[70vh]">
{#each itemMap['runs'] ?? [] as r}
<QuickMenuItem
on:select={() => {
on:hover={() => {
selectedItem = r
selectedWorkspace = r?.document.workspace_id[0]
}}
on:keyboardOnlySelect={() => {
on:select={() => {
open = false
goto(`/run/${r?.document.id[0]}`)
}}
@@ -658,7 +657,10 @@
>
<svelte:fragment slot="itemReplacement">
<div
class="w-full flex flex-row items-center gap-4 transition-all"
class={twMerge(
`w-full flex flex-row items-center gap-4 transition-all`,
r?.document.id === selectedItem?.document?.id ? 'bg-surface-hover' : ''
)}
>
<div
class="rounded-full w-2 h-2 {r?.document.success[0]

View File

@@ -24,7 +24,6 @@
async function handleKeydown(event: KeyboardEvent) {
if (hovered && event.key === 'Enter') {
event.preventDefault()
dispatch('keyboardOnlySelect')
runAction()
}
}

View File

@@ -263,7 +263,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F19662'}
href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F11446'}
target="_blank">Create from template</Button
>
{/if}

View File

@@ -268,7 +268,6 @@
bind:selectedFileKey={static_asset_config}
on:close={() => {
s3Editor?.setCode(JSON.stringify(static_asset_config, null, 2))
s3FileUploadRawMode = true
}}
readOnlyMode={false}
/>
@@ -367,7 +366,6 @@
disabled={!can_write}
/>
{/if}
{s3FileUploadRawMode}
{#if s3FileUploadRawMode}
{#if can_write}
<JsonEditor
@@ -398,6 +396,7 @@
s3: evt.detail?.path ?? '',
filename: evt.detail?.filename ?? undefined
}
s3FileUploadRawMode = true
}}
on:deletion={(evt) => {
static_asset_config = {
@@ -448,7 +447,7 @@
size="xs"
href={itemKind === 'flow'
? '/flows/add?hub=62'
: '/scripts/add?hub=hub%2F19669'}
: '/scripts/add?hub=hub%2F11627'}
target="_blank">Create from template</Button
>
{/if}
@@ -623,9 +622,7 @@
{/if}
{/if}
{#if !static_asset_config}
<RouteBodyTransformerOption bind:raw_string bind:wrap_body />
{/if}
<RouteBodyTransformerOption bind:raw_string bind:wrap_body />
</div>
</Section>
{/if}

View File

@@ -2,7 +2,7 @@ import { base } from '$lib/base'
import { isCloudHosted } from '$lib/cloud'
export const SECRET_KEY_PATH = 'secret_key_path'
export const HUB_SCRIPT_ID = 19670
export const HUB_SCRIPT_ID = 19661
export const SIGNATURE_TEMPLATE_SCRIPT_HUB_PATH: string = `hub/${HUB_SCRIPT_ID}`
export const SIGNATURE_TEMPLATE_FLOW_HUB_ID = '67'

View File

@@ -660,94 +660,86 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/**
export const TS_PREPROCESSOR_MODULE_CODE = `export async function preprocessor(
event:
| {
kind: "webhook";
body: any;
raw_string: string | null;
query: Record<string, string>;
headers: Record<string, string>;
}
kind: "webhook";
body: any,
raw_string: string | null,
query: Record<string, string>;
headers: Record<string, string>;
}
| {
kind: "http";
body: any;
raw_string: string | null;
route: string;
path: string;
method: string;
params: Record<string, string>;
query: Record<string, string>;
headers: Record<string, string>;
}
kind: "http";
body: any,
raw_string: string | null,
route: string;
path: string;
method: string;
params: Record<string, string>;
query: Record<string, string>;
headers: Record<string, string>;
}
| {
kind: "email";
parsed_email: any;
raw_email: string;
}
kind: "email";
parsed_email: any,
raw_email: string,
}
| { kind: "websocket"; msg: string; url: string }
| {
kind: "kafka";
payload: string;
brokers: string[];
topic: string;
group_id: string;
}
kind: "kafka";
payload: string;
brokers: string[];
topic: string;
group_id: string;
}
| {
kind: "nats";
payload: string;
servers: string[];
subject: string;
headers?: Record<string, string[]>;
status?: number;
description?: string;
length: number;
}
kind: "nats";
payload: string;
servers: string[];
subject: string;
headers?: Record<string, string[]>;
status?: number;
description?: string;
length: number;
}
| {
kind: "sqs";
msg: string;
queue_url: string;
message_id?: string;
receipt_handle?: string;
attributes: Record<string, string>;
message_attributes?: Record<
string,
{ string_value?: string; data_type: string }
>;
}
kind: "sqs";
msg: string,
queue_url: string;
message_id?: string;
receipt_handle?: string;
attributes: Record<string, string>;
message_attributes?: Record<
string,
{ string_value?: string; data_type: string }
>;
}
| {
kind: "mqtt";
payload: string;
topic: string;
retain: boolean;
pkid: number;
qos: number;
v5?: {
payload_format_indicator?: number;
topic_alias?: number;
response_topic?: string;
correlation_data?: Array<number>;
user_properties?: Array<[string, string]>;
subscription_identifiers?: Array<number>;
content_type?: string;
};
}
kind: "mqtt";
payload: string,
topic: string;
retain: boolean;
pkid: number;
qos: number;
v5?: {
payload_format_indicator?: number;
topic_alias?: number;
response_topic?: string;
correlation_data?: Array<number>;
user_properties?: Array<[string, string]>;
subscription_identifiers?: Array<number>;
content_type?: string;
};
}
| {
kind: "gcp";
payload: string;
message_id: string;
subscription: string;
ordering_key?: string;
attributes?: Record<string, string>;
delivery_type: "push" | "pull";
headers?: Record<string, string>;
publish_time?: string;
}
| {
kind: "postgres";
transaction_type: "insert" | "update" | "delete",
schema_name: string,
table_name: string,
old_row?: Record<string, any>,
row: Record<string, any>
}
kind: "gcp";
payload: string,
message_id: string;
subscription: string;
ordering_key?: string;
attributes?: Record<string, string>;
delivery_type: "push" | "pull";
headers?: Record<string, string>;
publish_time?: string;
}
) {
return {
// return the args to be passed to the runnable
@@ -906,16 +898,6 @@ class GcpEvent(TypedDict):
headers: Optional[dict[str, str]]
publish_time: Optional[str]
class PostgresEvent(TypedDict):
kind: Literal["postgres"]
transaction_type: Literal["insert", "update", "delete"]
schema_name: str
table_name: str
old_row: Optional[dict[str, any]]
row: dict[str, any]
Event = Union[
WebhookEvent,
HttpEvent,
@@ -926,7 +908,6 @@ Event = Union[
SqsEvent,
MqttEvent,
GcpEvent,
PostgresEvent,
]

View File

@@ -4,8 +4,8 @@ verify_ssl = true
name = "pypi"
[packages]
wmill = ">=1.491.5"
wmill_pg = ">=1.491.5"
wmill = ">=1.491.0"
wmill_pg = ">=1.491.0"
sendgrid = "*"
mysql-connector-python = "*"
pymongo = "*"

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.491.5
version: 1.491.0
title: OpenFlow Spec
contact:
name: Ruben Fiszel

View File

@@ -12,7 +12,7 @@
RootModule = 'WindmillClient.psm1'
# Version number of this module.
ModuleVersion = '1.491.5'
ModuleVersion = '1.491.0'
# Supported PSEditions
# CompatiblePSEditions = @()

View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill"
version = "1.491.5"
version = "1.491.0"
description = "A client library for accessing Windmill server wrapping the Windmill client API"
license = "Apache-2.0"
homepage = "https://windmill.dev"

View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "wmill-pg"
version = "1.491.5"
version = "1.491.0"
description = "An extension client for the wmill client library focused on pg"
license = "Apache-2.0"
homepage = "https://windmill.dev"

View File

@@ -1,6 +1,6 @@
{
"name": "@windmill/windmill",
"version": "1.491.5",
"version": "1.491.0",
"exports": "./src/index.ts",
"publish": {
"exclude": ["!src", "./s3Types.ts", "./client.ts"]

View File

@@ -1,7 +1,7 @@
{
"name": "windmill-client",
"description": "Windmill SDK client for browsers and Node.js",
"version": "1.491.5",
"version": "1.491.0",
"author": "Ruben Fiszel",
"license": "Apache 2.0",
"devDependencies": {

View File

@@ -1 +1 @@
1.491.5
1.491.0