Compare commits

..

3 Commits

Author SHA1 Message Date
Ruben Fiszel
39f0752525 all 2024-08-19 21:43:22 +02:00
Ruben Fiszel
1ee78da4bf Merge branch 'main' into rf/foo2 2024-08-19 12:43:08 +02:00
Ruben Fiszel
0a336467d8 all 2024-08-19 12:32:43 +02:00
353 changed files with 4150 additions and 35371 deletions

View File

@@ -1,5 +1,5 @@
ARG DEBIAN_IMAGE=debian:bookworm-slim
ARG RUST_IMAGE=rust:1.80-slim-bookworm
ARG RUST_IMAGE=rust:1.79-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.4-slim-bookworm
FROM ${DEBIAN_IMAGE} as downloader

View File

@@ -29,10 +29,10 @@ jobs:
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: 1.80.0
toolchain: 1.79.0
# - uses: Swatinem/rust-cache@v2
# with:
# workspaces: |

View File

@@ -16,7 +16,7 @@ jobs:
- uses: depot/setup-action@v1
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -26,7 +26,7 @@ jobs:
type=raw,value=latest,enable={{is_default_branch}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

View File

@@ -34,7 +34,7 @@ jobs:
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-staging-ee
@@ -45,7 +45,7 @@ jobs:
type=ref,event=branch
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

View File

@@ -18,13 +18,13 @@ jobs:
publish_cli:
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: depot/setup-action@v1
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -35,7 +35,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

View File

@@ -16,13 +16,13 @@ jobs:
publish_multiplayer:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: depot/setup-action@v1
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -33,7 +33,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

View File

@@ -9,7 +9,7 @@ jobs:
runs-on: ubicloud
container: node:18
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- run: git config --system --add safe.directory /__w/windmill/windmill
- name: Change versions
run: ./.github/change-versions.sh "$(cat version.txt)"

View File

@@ -10,7 +10,7 @@ jobs:
build_deno_and_push_to_repo:
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: generate_deno
run: |
cd deno-client
@@ -33,7 +33,7 @@ jobs:
needs: [build_deno_and_push_to_repo]
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
with:
repository: ${{ github.repository_owner }}/${{ env.repo }}
token: ${{ secrets.DENO_PAT }}

View File

@@ -37,7 +37,7 @@ jobs:
- uses: depot/setup-action@v1
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -49,7 +49,7 @@ jobs:
- name: Docker meta
id: meta-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

View File

@@ -1,11 +1,7 @@
env:
REGISTRY: ghcr.io
IMAGE_NAME:
${{ github.event_name != 'pull_request' && github.repository ||
'windmill-labs/windmill-test' }}
DEV_SHA:
${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
github.event.number) }}
IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository || 'windmill-labs/windmill-test' }}
DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', github.event.number) }}
name: Build windmill:main
on:
@@ -48,7 +44,7 @@ jobs:
- uses: depot/setup-action@v1
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -60,7 +56,7 @@ jobs:
- name: Docker meta
id: meta-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -110,7 +106,7 @@ jobs:
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
@@ -121,7 +117,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -172,7 +168,7 @@ jobs:
- name: Docker meta
id: meta-ee-public-py312
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-py312
@@ -183,7 +179,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -279,7 +275,7 @@ jobs:
mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
- name: Attach binary to release
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@v1
with:
files: |
${{ steps.extract.outputs.destination }}/*
@@ -323,7 +319,7 @@ jobs:
# mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"
# - name: Attach binary to release
# uses: softprops/action-gh-release@v2
# uses: softprops/action-gh-release@v1
# with:
# files: |
# ${{ steps.extract.outputs.destination }}/*
@@ -362,7 +358,7 @@ jobs:
with:
fetch-depth: 0
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -381,7 +377,7 @@ jobs:
with:
fetch-depth: 0
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -449,7 +445,7 @@ jobs:
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail
@@ -461,7 +457,7 @@ jobs:
type=sha,enable=true,priority=100,prefix=,suffix=,format=short
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -494,15 +490,15 @@ jobs:
- uses: depot/setup-action@v1
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-reports
@@ -540,12 +536,12 @@ jobs:
uses: docker/setup-buildx-action@v2
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: get git hash
if: github.event_name != 'pull_request'
id: git_hash
@@ -561,7 +557,7 @@ jobs:
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-nsjail:${{ steps.git_hash.outputs.GIT_HASH }}
path: "/static_frontend/."
- uses: reggionick/s3-deploy@v4
- uses: reggionick/s3-deploy@v3
if: github.event_name != 'pull_request'
with:
folder: ${{ steps.extract.outputs.destination }}
@@ -584,7 +580,7 @@ jobs:
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda
@@ -595,7 +591,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -614,51 +610,6 @@ jobs:
${{ steps.meta-ee-public.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
build_slim:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
needs: [build]
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim
flavor: |
latest=false
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push publicly ee
uses: depot/build-push-action@v1
with:
context: .
platforms: linux/amd64
push: true
file: "./docker/DockerfileSlim"
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |
${{ steps.meta-ee-public.outputs.labels }}
build_ee_slim:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
needs: [build_ee]
@@ -675,7 +626,7 @@ jobs:
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim
@@ -686,7 +637,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@@ -698,98 +649,7 @@ jobs:
context: .
platforms: linux/amd64
push: true
file: "./docker/DockerfileSlimEe"
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |
${{ steps.meta-ee-public.outputs.labels }}
org.opencontainers.image.licenses=Windmill-Enterprise-License
build_full:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
needs: [build]
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full
flavor: |
latest=false
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push publicly ee
uses: depot/build-push-action@v1
with:
context: .
platforms: linux/amd64
push: true
file: "./docker/DockerfileRust"
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |
${{ steps.meta-ee-public.outputs.labels }}
build_ee_full:
if: ${{ startsWith(github.ref, 'refs/tags/') }}
needs: [build_ee]
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
- uses: depot/setup-action@v1
- name: Docker meta
id: meta-ee-public
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full
flavor: |
latest=false
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push publicly ee
uses: depot/build-push-action@v1
with:
context: .
platforms: linux/amd64
push: true
file: "./docker/DockerfileRustEe"
file: "./docker/DockerfileSlim"
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |

View File

@@ -10,7 +10,7 @@ jobs:
npm_check:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 18

View File

@@ -12,7 +12,7 @@ jobs:
build_go_and_push_to_repo:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
- name: install_jq
run: |
@@ -42,7 +42,7 @@ jobs:
needs: [build_go_and_push_to_repo]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
with:
repository: ${{ github.repository_owner }}/${{ env.repo }}
token: ${{ secrets.DENO_PAT }}

View File

@@ -8,7 +8,7 @@ jobs:
build_npm:
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: "16.x"

View File

@@ -27,13 +27,13 @@ jobs:
needs: [sleep]
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
with:
fetch-depth: 0
- uses: depot/setup-action@v1
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
uses: docker/metadata-action@v4
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
@@ -44,7 +44,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}
- name: Login to registry
uses: docker/login-action@v3
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}

View File

@@ -12,7 +12,7 @@ jobs:
container:
image: ghcr.io/windmill-labs/python-client-builder
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v3
- name: Upload python client
env:
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

View File

@@ -1,170 +1,5 @@
# Changelog
## [1.390.1](https://github.com/windmill-labs/windmill/compare/v1.390.0...v1.390.1) (2024-09-03)
### Bug Fixes
* do not require hasNullParent only if scriptPathExact on runs search ([d172e45](https://github.com/windmill-labs/windmill/commit/d172e45766220c525e91f933f22a32738fa2709e))
* prevent brute force attacks on tokens by slowing unauthorized response ([acfe778](https://github.com/windmill-labs/windmill/commit/acfe7786152f036f2476f93ab5536571514fa9e3))
## [1.390.0](https://github.com/windmill-labs/windmill/compare/v1.389.1...v1.390.0) (2024-09-02)
### Features
* add yaml editor in flow builder ([71470d7](https://github.com/windmill-labs/windmill/commit/71470d7da9d3c7a52d7c5a206849f2e63ac4fa02))
### Bug Fixes
* improve dependency map to handle recusrive loops + handle better flow relative imports ([cdd7349](https://github.com/windmill-labs/windmill/commit/cdd7349f6769a0195f367a1ac27f802287a1f900))
## [1.389.1](https://github.com/windmill-labs/windmill/compare/v1.389.0...v1.389.1) (2024-09-02)
### Bug Fixes
* fix erronous branchone status in flow viewer ([e311684](https://github.com/windmill-labs/windmill/commit/e311684a668c461538a3456c02c82a1ea6fecbc9))
*
## [1.389.0](https://github.com/windmill-labs/windmill/compare/v1.388.0...v1.389.0) (2024-09-01)
### Features
* service logs ([#4244](https://github.com/windmill-labs/windmill/issues/4244)) ([2fe48df](https://github.com/windmill-labs/windmill/commit/2fe48df720b102255832dc6438d5d64f3443207a))
### Bug Fixes
* prevent duplicate worker alerts ([#4309](https://github.com/windmill-labs/windmill/issues/4309)) ([8e30928](https://github.com/windmill-labs/windmill/commit/8e30928a78c20daef81bc4f98487ab601ecc9b58))
## [1.388.0](https://github.com/windmill-labs/windmill/compare/v1.387.1...v1.388.0) (2024-08-30)
### Features
* add rust ([#4253](https://github.com/windmill-labs/windmill/issues/4253)) ([a2beed9](https://github.com/windmill-labs/windmill/commit/a2beed9d7335856bc956ff34343cdaf5a9b8d281))
* min workers in worker group alert + zombie job critical alert ([#4307](https://github.com/windmill-labs/windmill/issues/4307)) ([aa6fd84](https://github.com/windmill-labs/windmill/commit/aa6fd84079b6015d82d7603218b2c8cb7a156c40))
## [1.387.1](https://github.com/windmill-labs/windmill/compare/v1.387.0...v1.387.1) (2024-08-30)
### Bug Fixes
* fix resource list in args being pre-pended with $res ([2397588](https://github.com/windmill-labs/windmill/commit/239758835503ead5bbb1830c04b4fea5d13ad425))
* improve history navigation on the runs page ([487e7ca](https://github.com/windmill-labs/windmill/commit/487e7ca715d37b4b59a3a03ba8ff2bb644118635))
## [1.387.0](https://github.com/windmill-labs/windmill/compare/v1.386.0...v1.387.0) (2024-08-29)
### Features
* **frontend:** add a favorite button on detail pages ([#4297](https://github.com/windmill-labs/windmill/issues/4297)) ([8148518](https://github.com/windmill-labs/windmill/commit/8148518c5c7780d7999b884b0a9e0e540ece09fd))
* **frontend:** add indicator when a component is locked ([#4296](https://github.com/windmill-labs/windmill/issues/4296)) ([d9b358b](https://github.com/windmill-labs/windmill/commit/d9b358bcc20d2603ef94ada765d395e370d58fc7))
## [1.386.0](https://github.com/windmill-labs/windmill/compare/v1.385.0...v1.386.0) (2024-08-28)
### Features
* add vim support for monaco/webeditor ([1ec45e5](https://github.com/windmill-labs/windmill/commit/1ec45e5ad6436d36d3951a75663b4f310025d765))
* **frontend:** add support to copy a cell value in the clipboard in aggrid tables ([#4286](https://github.com/windmill-labs/windmill/issues/4286)) ([d1ba9b1](https://github.com/windmill-labs/windmill/commit/d1ba9b14040f9c21612a46b36379cce95448fba6))
* **frontend:** manage ag grid actions programmatically ([#4289](https://github.com/windmill-labs/windmill/issues/4289)) ([a411179](https://github.com/windmill-labs/windmill/commit/a4111798d54f5aa1539dd917eb59ec825c9a4a0b))
* show last job instead of current job on workers page ([#4293](https://github.com/windmill-labs/windmill/issues/4293)) ([84ce3d8](https://github.com/windmill-labs/windmill/commit/84ce3d819d19254e89c0da9ddf0b9f3819fe6025))
### Bug Fixes
* cache hub scripts in more cases + pre-cache hub scripts deps in deno ([16465e4](https://github.com/windmill-labs/windmill/commit/16465e47c8acf6320694a821b375aa20ee51067c))
* items with starred info ([#4298](https://github.com/windmill-labs/windmill/issues/4298)) ([e16bd4a](https://github.com/windmill-labs/windmill/commit/e16bd4a9d25d945b13696f8f7d9e39b10acdb444))
* show vCPU, mem aggregate on top of worker group ([e083177](https://github.com/windmill-labs/windmill/commit/e0831777a1a8798ba4234f299035c6f471078553))
* smtp server build without parquet feature ([#4292](https://github.com/windmill-labs/windmill/issues/4292)) ([906cf10](https://github.com/windmill-labs/windmill/commit/906cf1006e5447be075694d7d4903cea04b85737))
## [1.385.0](https://github.com/windmill-labs/windmill/compare/v1.384.0...v1.385.0) (2024-08-26)
### Features
* s3 image preview ([#4262](https://github.com/windmill-labs/windmill/issues/4262)) ([35d665f](https://github.com/windmill-labs/windmill/commit/35d665f6179e0a52dd30512e378f270c44703f48))
### Bug Fixes
* **frontend:** fix inserting Ws scripts from the search menu ([#4290](https://github.com/windmill-labs/windmill/issues/4290)) ([4efc40f](https://github.com/windmill-labs/windmill/commit/4efc40fbb0214fba65f6ff95782e6ead3b20bc71))
## [1.384.0](https://github.com/windmill-labs/windmill/compare/v1.383.1...v1.384.0) (2024-08-26)
### Features
* **frontend:** Add a toggle to disable breakpoints in the App editor ([#4274](https://github.com/windmill-labs/windmill/issues/4274)) ([5b2dd75](https://github.com/windmill-labs/windmill/commit/5b2dd7573d7ac2bd1b4ff06094a2dfd480ed24a4))
* **frontend:** hide/show app editor panels ([#4266](https://github.com/windmill-labs/windmill/issues/4266)) ([fdfd385](https://github.com/windmill-labs/windmill/commit/fdfd385a68c8e05eff147a9471a9f3f44f39e13e))
* put email triggers attachments on s3 ([#4272](https://github.com/windmill-labs/windmill/issues/4272)) ([5bd38f7](https://github.com/windmill-labs/windmill/commit/5bd38f78089a4beb4a28ba32bc99e649f8119a4c))
### Bug Fixes
* bun 1.1.21->1.1.25 ([7fa648f](https://github.com/windmill-labs/windmill/commit/7fa648f0876f5b5509fed39b314f40f08528b123))
* **lsp:** use ruff server instead of ruff-lsp ([88648af](https://github.com/windmill-labs/windmill/commit/88648af1cd6f9e4436e2fd3100f3ece764da75f1))
* update monaco-editor to latest monaco/language-client ([#4285](https://github.com/windmill-labs/windmill/issues/4285)) ([32c0b89](https://github.com/windmill-labs/windmill/commit/32c0b89729d98a25a5937af1ee41e8ea36334509))
## [1.383.1](https://github.com/windmill-labs/windmill/compare/v1.383.0...v1.383.1) (2024-08-22)
### Bug Fixes
* fix app navbar query reactivity + hash in ctx handling is more consistent ([7882d4e](https://github.com/windmill-labs/windmill/commit/7882d4ecdd1007e73a51d2a408cf2e6dc1f83213))
## [1.383.0](https://github.com/windmill-labs/windmill/compare/v1.382.2...v1.383.0) (2024-08-22)
### Features
* add native html select support + fix mobile scroll on app text component ([d604b6f](https://github.com/windmill-labs/windmill/commit/d604b6f2a0e5755c949f302c0490de47a8d662ca))
* add wrap_body header to webhooks ([9226d6c](https://github.com/windmill-labs/windmill/commit/9226d6cbc119f6a1b16936ebb855ff4f50c0e122))
* improve early stop ([#4257](https://github.com/windmill-labs/windmill/issues/4257)) ([bcde2e6](https://github.com/windmill-labs/windmill/commit/bcde2e62d7846822cfe7f8ef89a9484938829903))
### Bug Fixes
* **frontend:** fix large JSON viewer ([#4273](https://github.com/windmill-labs/windmill/issues/4273)) ([b3eabff](https://github.com/windmill-labs/windmill/commit/b3eabffb76e266269b2f5ff471a053fa065da4d0))
* **python-client:** only require httpx to be >= 0.24 instead of ^0.24 ([fc12aeb](https://github.com/windmill-labs/windmill/commit/fc12aeb3961b8a1a42a8493e3774815c940b75af))
## [1.382.2](https://github.com/windmill-labs/windmill/compare/v1.382.1...v1.382.2) (2024-08-20)
### Bug Fixes
* **app:** database studio/empty table count reset ([d6d3389](https://github.com/windmill-labs/windmill/commit/d6d33898a765f910f66cd0cfbd70a439ab107528))
* **frontend:** Fix initial FlowGraph rendering on Chrome ([#4268](https://github.com/windmill-labs/windmill/issues/4268)) ([84d4e2c](https://github.com/windmill-labs/windmill/commit/84d4e2cb956ab4947b364d64c464fd5cf3c1d16f))
* handle more gracefully worker without tags ([7e1f280](https://github.com/windmill-labs/windmill/commit/7e1f28071a7d6f3f34df417ea746fc9b5dca5157))
## [1.382.1](https://github.com/windmill-labs/windmill/compare/v1.382.0...v1.382.1) (2024-08-20)
### Bug Fixes
* **frontend:** Fix flow graph step preview ([#4264](https://github.com/windmill-labs/windmill/issues/4264)) ([0a6832a](https://github.com/windmill-labs/windmill/commit/0a6832aa4561468fb0150f9eb435b7ad857fc841))
## [1.382.0](https://github.com/windmill-labs/windmill/compare/v1.381.0...v1.382.0) (2024-08-20)
### Features
* **frontend:** improve versions history by adding a diff viewer with… ([#4261](https://github.com/windmill-labs/windmill/issues/4261)) ([c19df12](https://github.com/windmill-labs/windmill/commit/c19df12292241c20c12e3de5d880c04df9b74324))
### Bug Fixes
* add FORCE_WORKER_TAGS & fix workers page when default worker group is missing ([09c6af0](https://github.com/windmill-labs/windmill/commit/09c6af05cd6e0b3b30f771a125b1fa330a938101))
* **bun:** disable large transpiling cache ([1a0e32b](https://github.com/windmill-labs/windmill/commit/1a0e32b40b5f1ac5dcfe3639e697affb6e79a558))
* **cli:** improve error message of cli ([f3bcadb](https://github.com/windmill-labs/windmill/commit/f3bcadbfb1e42b56996e273800aa0dcb01a6868b))
* frontend/package.json & frontend/package-lock.json to reduce vulnerabilities ([#4255](https://github.com/windmill-labs/windmill/issues/4255)) ([7fe0442](https://github.com/windmill-labs/windmill/commit/7fe0442a814ed42b1775386be5f4defdcb5c0a0f))
* **frontend:** nit worker limits ([#4258](https://github.com/windmill-labs/windmill/issues/4258)) ([bb92824](https://github.com/windmill-labs/windmill/commit/bb9282427399f305c3e8874204e6478272c43ff8))
* improve resource picker handling of objects ([2502377](https://github.com/windmill-labs/windmill/commit/250237793b3c2b1afd5ee122b147301bc487eeeb))
* **typescript-client:** runFlowAsync by default assume job doesn't outlive flow ([010e0fd](https://github.com/windmill-labs/windmill/commit/010e0fdc2352d740a72de124cc26eea8fc1915af))
## [1.381.0](https://github.com/windmill-labs/windmill/compare/v1.380.0...v1.381.0) (2024-08-16)

View File

@@ -1,5 +1,5 @@
ARG DEBIAN_IMAGE=debian:bookworm-slim
ARG RUST_IMAGE=rust:1.80-slim-bookworm
ARG RUST_IMAGE=rust:1.79-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
FROM ${RUST_IMAGE} AS rust_base
@@ -70,9 +70,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
COPY ./openflow.openapi.yaml /openflow.openapi.yaml
COPY ./backend ./
RUN mkdir -p /frontend
COPY --from=frontend /frontend/build /frontend/build
COPY --from=frontend /frontend /frontend
COPY --from=frontend /backend/windmill-api/openapi-deref.yaml ./windmill-api/openapi-deref.yaml
COPY .git/ .git/
@@ -163,9 +161,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:1.46.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:1.45.4 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.25 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.21 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
@@ -173,16 +171,15 @@ COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
ENV RUSTUP_HOME="/usr/local/rustup"
ENV CARGO_HOME="/usr/local/cargo"
WORKDIR ${APP}
RUN ln -s ${APP}/windmill /usr/local/bin/windmill
COPY ./frontend/src/lib/hubPaths.json ${APP}/hubPaths.json
RUN windmill cache ${APP}/hubPaths.json && rm ${APP}/hubPaths.json && chmod -R 777 /tmp/windmill
RUN windmill cache ${APP}/hubPaths.json
RUN rm ${APP}/hubPaths.json
EXPOSE 8000

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO alerts (alert_type, message) VALUES ('recovered_critical_error', $1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "00ce4ed3ca0eac7cb6283b047353a64b9e78c4beb423f04baef9a53fbf87e9f9"
}

View File

@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT healthy, created_at FROM healthchecks WHERE check_type = 'min_alive_workers_' || $1 ORDER BY created_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "healthy",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "050fb876e10ad13654dbbde4532f408ff1ac92ed0f5d31a3ef6c58313e1f8671"
}

View File

@@ -56,8 +56,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -50,8 +50,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -1,21 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Varchar",
"Timestamp",
"Varchar",
"Int8",
"Int8",
"Bool"
]
},
"nullable": []
},
"hash": "33c1793e55b1127d88d2509aadd0eb04e042463200f237b4c2cb176612fa16fe"
}

View File

@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT SUBSTRING(name, 9) as \"name!\", (config.config->'min_alive_workers_alert_threshold')::INT as \"threshold!\" \n FROM config \n WHERE name LIKE 'worker__%' AND config->'min_alive_workers_alert_threshold' IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "name!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "threshold!",
"type_info": "Int4"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "354f88b23d20f92c6b6d5bdd8d6c69b08c6a86116cbfd0ecad8f112f7f49d8d1"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO alerts (alert_type, message) VALUES ('critical_error', $1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "4c0067c2135a259aea5cc2db60f7375a9a33671be8ef406427d90f67a98c9c9f"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO healthchecks (check_type, healthy) \n SELECT 'min_alive_workers_' || $1, true \n WHERE NOT EXISTS (\n SELECT 1 FROM healthchecks \n WHERE check_type = 'min_alive_workers_' || $1 AND created_at > NOW() - INTERVAL '2 minutes'\n )\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "4cfa57b2a836242d55071ef237855b21bbc487b80bda7fd6250ac12e93938577"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as current_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as current_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -35,12 +35,12 @@
},
{
"ordinal": 6,
"name": "last_job_id",
"name": "current_job_id",
"type_info": "Uuid"
},
{
"ordinal": 7,
"name": "last_job_workspace_id",
"name": "current_job_workspace_id",
"type_info": "Varchar"
},
{
@@ -111,5 +111,5 @@
true
]
},
"hash": "8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af"
"hash": "55d6c6f887f1cceb311d9ab034d3485b8ccb2d07d723f184cef2696760d3ccf9"
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO healthchecks (check_type, healthy) \n SELECT 'min_alive_workers_' || $1, false\n WHERE NOT EXISTS (\n SELECT 1 FROM healthchecks \n WHERE check_type = 'min_alive_workers_' || $1 AND created_at > NOW() - INTERVAL '2 minutes'\n )\n RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "5f22a7c9170f035779782591b4bfb1d462d5c796f08c7481c42154f9bfacf388"
}

View File

@@ -70,8 +70,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET running = false, started_at = null\n WHERE last_ping < now() - ($1 || ' seconds')::interval\n AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'singlescriptflow') AND same_worker = false RETURNING id, workspace_id, last_ping",
"query": "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' \n WHERE last_ping < now() - ($1 || ' seconds')::interval\n AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'singlescriptflow') AND same_worker = false RETURNING id, workspace_id, last_ping",
"describe": {
"columns": [
{
@@ -30,5 +30,5 @@
true
]
},
"hash": "8c15ea27be82665fe74a4be3182a5049fa937e30b4115caf0ba8d61d34f2089a"
"hash": "7ab02d2c060ed12531e6af98a5d3369afe534519bf48f61f88f60786db870c24"
}

View File

@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM healthchecks WHERE check_type LIKE 'min_alive_workers_%' AND created_at < NOW() - INTERVAL '48 hours'",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "7fa8fca5b5cdf147da0cf8ae5a0d9fdee37dd8df66974c8d33f58a88b1bca537"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a"
}

View File

@@ -55,8 +55,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT count(*) \n FROM worker_ping \n WHERE worker_group LIKE $1 AND ping_at > now() - INTERVAL '2 minutes'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "8dd1de2aca8c6c9ffaddd2c41c3a614a50fa5fd03c2d3b9a41bd85a7f156345e"
}

View File

@@ -36,8 +36,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -55,8 +55,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -50,8 +50,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE password SET name = $1 WHERE email = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "aa5bdf0ab8781cf984711ae4f4351eb3bd5621ae081a418f1d4dbcb8359a8adb"
}

View File

@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_logs (job_id, logs) VALUES ($1,'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n') \n ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "ae25fae1aca2cffc43a6054cd54639ba9deaaef419ad4cb2655e7e95f602a688"
}

View File

@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT healthy, created_at FROM healthchecks WHERE check_type = 'min_alive_workers_' || $1 AND created_at > NOW() - INTERVAL '24 hours' ORDER BY created_at DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "healthy",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "c1ceb03e0eafd2e494c06cef1ec96f78c2cfd42ed7d1ca6cfd35288b561120db"
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, current_job_id = NULL, current_job_workspace_id = NULL, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
@@ -17,5 +17,5 @@
},
"nullable": []
},
"hash": "9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51"
"hash": "c1de24c68933ab35934395d74c5d2747b3da80c96e255ccf0c4e053206a9ad1b"
}

View File

@@ -76,8 +76,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM log_file WHERE hostname = $1 AND log_ts = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Timestamp"
]
},
"nullable": [
null
]
},
"hash": "dee32ce9c4010ae407b4d1ba2ecf2062c36ee5dab3d4894ad4b2aa8b2f5a0db9"
}

View File

@@ -55,8 +55,7 @@
"powershell",
"mssql",
"php",
"bunnative",
"rust"
"bunnative"
]
}
}

View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING",
"query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5)",
"describe": {
"columns": [],
"parameters": {
@@ -14,5 +14,5 @@
},
"nullable": []
},
"hash": "958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c"
"hash": "e87c317550af671e4b65752a9c0b659cedf430fb7242faee4379b4a13e5fd763"
}

787
backend/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.390.1"
version = "1.381.0"
authors.workspace = true
edition.workspace = true
@@ -18,7 +18,6 @@ members = [
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-wasm",
"./parsers/windmill-parser-go",
"./parsers/windmill-parser-rust",
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-imports",
@@ -26,7 +25,7 @@ members = [
]
[workspace.package]
version = "1.390.1"
version = "1.381.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -48,7 +47,7 @@ flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
loki = ["windmill-common/loki"]
pg_embed = ["dep:pg-embed"]
embedding = ["windmill-api/embedding"]
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"]
parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
flow_testing = ["windmill-worker/flow_testing"]
openidconnect = ["windmill-api/openidconnect"]
@@ -85,9 +84,7 @@ gethostname.workspace = true
serde_json.workspace = true
serde.workspace = true
deno_core.workspace = true
object_store = { workspace = true, optional = true }
pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']}
quote.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
@@ -117,7 +114,6 @@ windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" }
windmill-parser-go = { path = "./parsers/windmill-parser-go" }
windmill-parser-rust = { path = "./parsers/windmill-parser-rust" }
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
@@ -140,7 +136,6 @@ chrono = { version = "0.4.35", features = ["serde"] }
chrono-tz = "^0"
tracing = "^0"
tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
tracing-appender = "^0"
prometheus = { version = "^0", default-features = false }
cookie = { version = "0.17.0" }
phf = { version = "0.11", features = ["macros"] }
@@ -271,9 +266,3 @@ tikv-jemalloc-ctl = { version = "^0.5" }
triomphe = "<0.1.12"
tantivy = "0.22.0"
pulldown-cmark = "0.9"
toml = "0.7"
syn = { version = "2.0.74", features = ["full"] }
quote = "1.0.36"
regex-lite = "0.1.6"

View File

@@ -1 +1 @@
04a34e95a377afdc45cf4f354d5009f02260f3b1
7cf76a8c6694b59f4615b0c8fa61fb02411326a7

View File

@@ -1,2 +1 @@
-- Add down migration script here
DROP TABLE public.job_perms;

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,3 +0,0 @@
-- Add up migration script here
ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'rust';
UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["rust"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php"]}'::jsonb AND NOT config->'worker_tags' @> '"rust"'::jsonb;

View File

@@ -1,4 +0,0 @@
-- Add down migration script here
DROP TABLE log_file;
DROP TYPE LOG_MODE;

View File

@@ -1,16 +0,0 @@
-- Add up migration script here
-- Add up migration script here
CREATE TYPE LOG_MODE AS ENUM ('standalone', 'server', 'worker', 'agent', 'indexer');
CREATE TABLE log_file (
hostname VARCHAR(255) NOT NULL,
log_ts TIMESTAMP,
ok_lines BIGINT,
err_lines BIGINT,
mode LOG_MODE NOT NULL,
worker_group VARCHAR(255),
file_path VARCHAR(510) NOT NULL,
PRIMARY KEY (hostname, log_ts)
);
CREATE INDEX log_file_log_ts_idx ON log_file (log_ts);
CREATE INDEX log_file_hostname_log_ts_idx ON log_file (hostname, log_ts);

View File

@@ -1,3 +0,0 @@
-- Add down migration script here
drop table alerts;
drop table healthchecks;

View File

@@ -1,16 +0,0 @@
-- Add up migration script here
create table alerts (
id serial PRIMARY KEY,
alert_type varchar(50) NOT NULL,
message text NOT NULL,
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
);
create table healthchecks (
id bigserial PRIMARY KEY,
check_type varchar(50) NOT NULL,
healthy boolean NOT NULL,
created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
);
create index healthchecks_check_type_created_at on healthchecks(check_type, created_at);

View File

@@ -1 +0,0 @@
-- Add down migration script here

View File

@@ -1,2 +0,0 @@
-- Add up migration script here
ALTER TABLE log_file ADD COLUMN IF NOT EXISTS json_fmt boolean DEFAULT false;

View File

@@ -8,14 +8,9 @@ authors.workspace = true
name = "windmill_parser_bash"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
regex-lite.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
regex.workspace = true
[dependencies]
windmill-parser.workspace = true
anyhow.workspace = true
regex.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
serde_json.workspace = true

View File

@@ -1,12 +1,7 @@
#![allow(non_snake_case)] // TODO: switch to parse_* function naming
use anyhow::anyhow;
#[cfg(not(target_arch = "wasm32"))]
use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use serde_json::json;
use std::collections::HashMap;

View File

@@ -8,14 +8,9 @@ authors.workspace = true
name = "windmill_parser_graphql"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
regex-lite.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
regex.workspace = true
[dependencies]
windmill-parser.workspace = true
anyhow.workspace = true
regex.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
serde_json.workspace = true

View File

@@ -1,12 +1,7 @@
#![allow(non_snake_case)] // TODO: switch to parse_* function naming
use anyhow::anyhow;
#[cfg(not(target_arch = "wasm32"))]
use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use serde_json::json;
use windmill_parser::{Arg, MainArgSignature, Typ};

View File

@@ -14,3 +14,6 @@ itertools.workspace = true
serde_json.workspace = true
anyhow.workspace = true
php-parser-rs.workspace = true
convert_case.workspace = true
lazy_static.workspace = true
regex.workspace = true

View File

@@ -1,5 +1,7 @@
use convert_case::{Case, Casing};
use regex::Regex;
use serde_json::Value;
use windmill_parser::{to_snake_case, Arg, MainArgSignature, Typ};
use windmill_parser::{Arg, MainArgSignature, Typ};
use php_parser_rs::parser::{
self,
@@ -11,6 +13,17 @@ use php_parser_rs::parser::{
},
};
lazy_static::lazy_static! {
static ref RE_SNK_CASE: Regex = Regex::new(r"_(\d)").unwrap();
}
fn to_snake_case(s: String) -> String {
let r = s.to_case(Case::Snake);
// s_3 => s3
RE_SNK_CASE.replace_all(&r, "$1").to_string()
}
fn parse_php_type(e: Type) -> Typ {
match e {
Type::Float(_) => Typ::Float,
@@ -19,7 +32,7 @@ fn parse_php_type(e: Type) -> Typ {
Type::String(_) => Typ::Str(None),
Type::Array(_) => Typ::List(Box::new(Typ::Str(None))),
Type::Object(_) => Typ::Object(vec![]),
Type::Named(_, name) => Typ::Resource(to_snake_case(name.to_string().as_ref())),
Type::Named(_, name) => Typ::Resource(to_snake_case(name.to_string())),
_ => Typ::Unknown,
}
}

View File

@@ -8,18 +8,13 @@ authors.workspace = true
name = "windmill_parser_py_imports"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
regex-lite.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
regex.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-common.workspace = true
rustpython-parser.workspace = true
phf.workspace = true
itertools.workspace = true
regex.workspace = true
serde_json.workspace = true
anyhow.workspace = true
lazy_static.workspace = true

View File

@@ -10,11 +10,7 @@ use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
use phf::phf_map;
#[cfg(not(target_arch = "wasm32"))]
use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use rustpython_parser::{
ast::{Stmt, StmtImport, StmtImportFrom, Suite},

View File

@@ -1,22 +0,0 @@
[package]
name = "windmill-parser-rust"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_rust"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
itertools.workspace = true
serde_json.workspace = true
anyhow.workspace = true
convert_case.workspace = true
lazy_static.workspace = true
regex.workspace = true
pulldown-cmark.workspace = true
toml.workspace = true
syn.workspace = true
quote.workspace = true

View File

@@ -1,16 +0,0 @@
[[bin]]
name = "main"
path = "./main.rs"
[package]
authors = ["Anonymous"]
edition = "2021"
name = "main"
version = "0.1.0"
[profile.release]
strip = true
[dependencies]
serde = { version = "1.0.207", features = ["derive"] }
serde_json = "1.0.124"

View File

@@ -1,501 +0,0 @@
use anyhow::anyhow;
use itertools::Itertools;
use quote::ToTokens;
use regex::Regex;
use windmill_parser::{Arg, MainArgSignature, Typ};
pub fn otyp_to_string(otyp: Option<String>) -> String {
otyp.unwrap()
}
pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
let ast: syn::File = syn::parse_file(code)?;
if let Some(main_fn) = ast.items.iter().find_map(|item| match item {
syn::Item::Fn(f) if f.sig.ident == "main" => Some(f),
_ => None,
}) {
let args = main_fn
.sig
.inputs
.iter()
.map(|param| {
let (otyp, typ, name) = parse_rust_typ(param);
Arg { name, otyp, typ, default: None, has_default: false, oidx: None }
})
.collect_vec();
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: Some(false),
})
} else {
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
})
}
}
pub fn parse_rust_deps_into_manifest(code: &str) -> anyhow::Result<String> {
const MODIFIABLE_MANIFEST_TABLES: &[&str] = &["dependencies"];
let partial_manifest = find_embedded_manifest(code).unwrap_or(Manifest::Toml("".to_string()));
let partial_manifest = partial_manifest.into_toml()?;
let mut manif = default_manifest();
for table_name in MODIFIABLE_MANIFEST_TABLES {
match partial_manifest.get(*table_name) {
Some(toml::Value::Table(tab)) => {
// Merge.
match manif.entry(*table_name) {
toml::map::Entry::Vacant(e) => {
e.insert(toml::Value::Table(tab.to_owned()));
}
toml::map::Entry::Occupied(e) => {
let into_t = match e.into_mut() {
toml::Value::Table(t) => Some(t),
_ => None,
};
into_t.ok_or(anyhow!(""))?.extend(tab.to_owned());
}
}
}
Some(v) => {
// Just replace.
manif.insert(table_name.to_string(), v.to_owned());
}
None => (),
}
}
Ok(manif.to_string())
}
fn parse_pat_type(p: Box<syn::Type>) -> Typ {
match *p {
syn::Type::Array(a) => {
let inner_typ = parse_pat_type(a.elem);
Typ::List(Box::new(inner_typ))
}
// syn::Type::BareFn(_) => todo!(),
// syn::Type::Group(_) => todo!(),
// syn::Type::ImplTrait(_) => todo!(),
// syn::Type::Infer(_) => todo!(),
// syn::Type::Macro(_) => todo!(),
// syn::Type::Never(_) => todo!(),
syn::Type::Paren(e) => parse_pat_type(e.elem),
syn::Type::Path(e) => {
if let Some(u) = e.path.segments.last() {
match u.ident.to_string().as_str() {
"usize" | "u8" | "u16" | "u32" | "u64" | "u128" | "isize" | "i8" | "i16"
| "i32" | "i64" | "i128" => Typ::Int,
"String" | "str" => Typ::Str(None),
"bool" => Typ::Bool,
"f32" | "f64" => Typ::Float,
"Vec" => {
if let syn::PathArguments::AngleBracketed(t) = &u.arguments {
if let Some(syn::GenericArgument::Type(a)) = t.args.last() {
Typ::List(Box::new(parse_pat_type(Box::new(a.clone()))))
} else {
Typ::Unknown
}
} else {
Typ::Unknown
}
}
_ => Typ::Unknown,
}
} else {
Typ::Unknown
}
}
// syn::Type::Ptr(_) => todo!(),
syn::Type::Reference(e) => parse_pat_type(e.elem),
syn::Type::Slice(e) => Typ::List(Box::new(parse_pat_type(e.elem))),
// syn::Type::TraitObject(_) => todo!(),
// syn::Type::Tuple(_) => todo!(),
// syn::Type::Verbatim(_) => todo!(),
_ => Typ::Unknown,
}
}
fn parse_rust_typ(param_typ: &syn::FnArg) -> (Option<String>, Typ, String) {
match param_typ {
syn::FnArg::Receiver(_) => (None, Typ::Unknown, "self".to_string()),
syn::FnArg::Typed(s) => {
let name = match *s.pat.clone() {
syn::Pat::Ident(p) => p.ident.to_string(),
_ => "undefined".to_string(),
};
let otyp = Some(s.ty.to_token_stream().to_string());
let typ = parse_pat_type(s.ty.clone());
(otyp, typ, name)
}
}
}
#[derive(Debug, PartialEq)]
enum Manifest {
Toml(String),
DepList(String),
}
impl Manifest {
pub fn into_toml(self) -> anyhow::Result<toml::value::Table> {
match self {
Manifest::Toml(s) => toml::from_str(&s),
Manifest::DepList(s) => Manifest::dep_list_to_toml(&s),
}
.map_err(|e| anyhow!("Could not parse embedded manifest: {}", e))
}
fn dep_list_to_toml(s: &str) -> ::std::result::Result<toml::value::Table, toml::de::Error> {
let mut r = String::new();
r.push_str("[dependencies]\n");
for dep in s.trim().split(',') {
// If there's no version specified, add one.
match dep.contains('=') {
true => {
r.push_str(dep);
r.push('\n');
}
false => {
r.push_str(dep);
r.push_str("=\"*\"\n");
}
}
}
toml::from_str(&r)
}
}
fn default_manifest() -> toml::value::Table {
toml::from_str(include_str!("../manifest/Cargo.toml.default"))
.expect("Failed to parse Cargo.toml.default")
}
/**
* Get a `Manifest` that can be made into a TOML table. The format and logic are the same as in
* the rust-script or cargo-eval projects.
*/
fn find_embedded_manifest(s: &str) -> Option<Manifest> {
find_short_hand_manifest(s).or_else(|| find_code_block_manifest(s))
}
fn find_short_hand_manifest(s: &str) -> Option<Manifest> {
let re: Regex = Regex::new(r"^(?i)\s*//\s*cargo-deps\s*:(.*?)(\r\n|\n)").unwrap();
/*
This is pretty simple: the only valid syntax for this is for the first, non-blank line to contain a single-line comment whose first token is `cargo-deps:`. That's it.
*/
if let Some(cap) = re.captures(s) {
if let Some(m) = cap.get(1) {
return Some(Manifest::DepList(m.as_str().to_string()));
}
}
None
}
/**
Locates a "code block manifest" in Rust source.
*/
fn find_code_block_manifest(s: &str) -> Option<Manifest> {
let re_crate_comment: Regex = {
Regex::new(
r"(?x)
^\s*
(/\*!|//(!|/))
",
)
.unwrap()
};
let start = match re_crate_comment.captures(s) {
Some(cap) => match cap.get(1) {
Some(m) => m.start(),
None => return None,
},
None => return None,
};
let comment = match extract_comment(&s[start..]) {
Some(s) => s,
None => {
return None;
}
};
scrape_markdown_manifest(&comment).map(|s| Manifest::Toml(s))
}
/**
Extracts the first `Cargo` fenced code block from a chunk of Markdown.
*/
fn scrape_markdown_manifest(content: &str) -> Option<String> {
use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag};
// To match librustdoc/html/markdown.rs, opts.
let exts = Options::ENABLE_TABLES | Options::ENABLE_FOOTNOTES;
let md = Parser::new_ext(content, exts);
let mut found = false;
let mut output = None;
for item in md {
match item {
Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(ref info)))
if info.to_lowercase() == "cargo" && output.is_none() =>
{
found = true;
}
Event::Text(ref text) if found => {
let s = output.get_or_insert(String::new());
s.push_str(text);
}
Event::End(Tag::CodeBlock(_)) if found => {
found = false;
}
_ => (),
}
}
output
}
/**
Extracts the contents of a Rust doc comment.
*/
fn extract_comment(s: &str) -> Option<String> {
use std::cmp::min;
fn extract_block(s: &str) -> Option<String> {
/*
On every line:
- update nesting level and detect end-of-comment
- if margin is None:
- if there appears to be a margin, set margin.
- strip off margin marker
- update the leading space counter
- strip leading space
- append content
*/
let mut r = String::new();
let margin_re: Regex = Regex::new(r"^\s*\*( |$)").unwrap();
let space_re: Regex = Regex::new(r"^(\s+)").unwrap();
let nesting_re: Regex = Regex::new(r"/\*|\*/").unwrap();
let mut leading_space = None;
let mut margin = None;
let mut depth: u32 = 1;
for line in s.lines() {
if depth == 0 {
break;
}
let mut end_of_comment = None;
for (end, marker) in nesting_re.find_iter(line).map(|m| (m.start(), m.as_str())) {
match (marker, depth) {
("/*", _) => depth += 1,
("*/", 1) => {
end_of_comment = Some(end);
depth = 0;
break;
}
("*/", _) => depth -= 1,
_ => return None,
}
}
let line = end_of_comment.map(|end| &line[..end]).unwrap_or(line);
margin = margin.or_else(|| margin_re.find(line).map(|m| m.as_str()));
let line = if let Some(margin) = margin {
let end = line
.char_indices()
.take(margin.len())
.map(|(i, c)| i + c.len_utf8())
.last()
.unwrap_or(0);
&line[end..]
} else {
line
};
leading_space = leading_space.or_else(|| space_re.find(line).map(|m| m.end()));
let strip_len = min(leading_space.unwrap_or(0), line.len());
let line = &line[strip_len..];
r.push_str(line);
r.push('\n');
}
Some(r)
}
fn extract_line(s: &str) -> Option<String> {
let mut r = String::new();
let comment_re = Regex::new(r"^\s*//(!|/)").unwrap();
let space_re = Regex::new(r"^(\s+)").unwrap();
let mut leading_space = None;
for line in s.lines() {
let content = match comment_re.find(line) {
Some(m) => &line[m.end()..],
None => break,
};
leading_space = leading_space.or_else(|| {
space_re
.captures(content)
.and_then(|c| c.get(1))
.map(|m| m.end())
});
let strip_len = min(leading_space.unwrap_or(0), content.len());
let content = &content[strip_len..];
r.push_str(content);
r.push('\n');
}
Some(r)
}
if let Some(stripped) = s.strip_prefix("/*!") {
extract_block(stripped)
} else if s.starts_with("//!") || s.starts_with("///") {
extract_line(s)
} else {
None
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_parse_rust_signature() {
let code = r#"
// commenting comments
fn main(
my_int: i8,
my_vec: Vec<u8>,
mut my_arr: [u8; 9],
my_complex: Vec<Result<MyStruct, anyhow::Error>>,
) -> Result<String, String> {
println!("My int is {}", my_int);
}"#;
let ret = parse_rust_signature(code).unwrap();
assert_eq!(ret.args.len(), 4);
assert_eq!(ret.args[0].name, "my_int");
assert_eq!(ret.args[0].otyp, Some("i8".to_string()));
assert_eq!(ret.args[0].typ, Typ::Int);
assert_eq!(ret.args[1].name, "my_vec");
assert_eq!(ret.args[1].otyp, Some("Vec < u8 >".to_string()));
assert_eq!(ret.args[1].typ, Typ::List(Box::new(Typ::Int)));
assert_eq!(ret.args[2].name, "my_arr");
assert_eq!(ret.args[2].otyp, Some("[u8 ; 9]".to_string()));
assert_eq!(ret.args[2].typ, Typ::List(Box::new(Typ::Int)));
assert_eq!(ret.args[3].name, "my_complex");
assert_eq!(
ret.args[3].otyp,
Some("Vec < Result < MyStruct , anyhow :: Error > >".to_string())
);
assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Unknown)));
let code = r#"
// commenting comments
fn main(
my_str_slice: &str,
my_String: String,
mut my_mut_ref_to_string: &mut String,
my_string_vec: Vec<String>,
) -> Result<String, String> {
println!("My int is {}", my_int);
}"#;
let ret = parse_rust_signature(code).unwrap();
assert_eq!(ret.args.len(), 4);
assert_eq!(ret.args[0].name, "my_str_slice");
assert_eq!(ret.args[0].otyp, Some("& str".to_string()));
assert_eq!(ret.args[0].typ, Typ::Str(None));
assert_eq!(ret.args[1].name, "my_String");
assert_eq!(ret.args[1].otyp, Some("String".to_string()));
assert_eq!(ret.args[1].typ, Typ::Str(None));
assert_eq!(ret.args[2].name, "my_mut_ref_to_string");
assert_eq!(ret.args[2].otyp, Some("& mut String".to_string()));
assert_eq!(ret.args[2].typ, Typ::Str(None));
assert_eq!(ret.args[3].name, "my_string_vec");
assert_eq!(
ret.args[3].otyp,
Some("Vec < String >".to_string())
);
assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None))));
}
#[test]
fn test_parse_rust_manifest() {
assert_eq!(find_embedded_manifest("fn main() {}"), None);
let code = r#"
//! Crate doc comment right here
//! We need to use the `cargo` language in the code bloc
//!
//! ```cargo
//! [dependencies]
//! time = "0.1.25"
//! ```
fn main() {
println!("{}", time::now().rfc822z());
}
"#;
let manif = find_embedded_manifest(code);
assert_eq!(
manif,
Some(Manifest::Toml(
r#"[dependencies]
time = "0.1.25"
"#
.to_string()
))
);
let _full_manif = parse_rust_deps_into_manifest(code).unwrap();
}
#[test]
fn test_default_manifest() {
default_manifest();
}
}

View File

@@ -8,14 +8,9 @@ authors.workspace = true
name = "windmill_parser_sql"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
regex-lite.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
regex.workspace = true
[dependencies]
windmill-parser.workspace = true
anyhow.workspace = true
regex.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
serde_json.workspace = true

View File

@@ -1,12 +1,7 @@
#![allow(non_snake_case)] // TODO: switch to parse_* function naming
use anyhow::anyhow;
#[cfg(not(target_arch = "wasm32"))]
use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use serde_json::json;
use std::{

View File

@@ -22,5 +22,6 @@ swc_ecma_ast.workspace = true
swc_ecma_visit.workspace = true
serde_json.workspace = true
anyhow.workspace = true
convert_case.workspace = true
regex.workspace = true
lazy_static.workspace = true

View File

@@ -1,3 +1,5 @@
use convert_case::{Case, Casing};
use regex::Regex;
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
@@ -9,9 +11,7 @@
use serde_json::Value;
use std::collections::HashSet;
use swc_ecma_visit::{noop_visit_type, Visit, VisitWith};
use windmill_parser::{
json_to_typ, to_snake_case, Arg, MainArgSignature, ObjectProperty, OneOfVariant, Typ,
};
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, OneOfVariant, Typ};
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
use swc_ecma_ast::{
@@ -23,7 +23,6 @@ use swc_ecma_ast::{
};
use swc_ecma_parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax, TsConfig};
use regex::Regex;
#[cfg(target_arch = "wasm32")]
use wasm_bindgen::prelude::*;
@@ -295,6 +294,7 @@ fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String
}
lazy_static::lazy_static! {
static ref RE_SNK_CASE: Regex = Regex::new(r"_(\d)").unwrap();
static ref IMPORTS_VERSION: Regex = Regex::new(r"^((?:\@[^\/\@]+\/[^\/\@]+)|(?:[^\/\@]+))(?:\@(?:[^\/]+))?(.*)$").unwrap();
}
@@ -320,6 +320,13 @@ pub fn remove_pinned_imports(code: &str) -> anyhow::Result<String> {
Ok(content)
}
fn to_snake_case(s: &str) -> String {
let r = s.to_case(Case::Snake);
// s_3 => s3
RE_SNK_CASE.replace_all(&r, "$1").to_string()
}
fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
match ts_type {
TsType::TsKeywordType(t) => (

View File

@@ -12,31 +12,17 @@ path = "./src/lib.rs"
[dev-dependencies]
wasm-bindgen-test.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-bash.workspace = true
[features]
default = []
go-parser = [ "dep:windmill-parser-go"]
bash-parser = [ "dep:windmill-parser-bash"]
sql-parser = [ "dep:windmill-parser-sql"]
py-parser = [ "dep:windmill-parser-py"]
ts-parser = [ "dep:windmill-parser-ts"]
php-parser = [ "dep:windmill-parser-php"]
rust-parser = [ "dep:windmill-parser-rust"]
graphql-parser = [ "dep:windmill-parser-graphql"]
[dependencies]
anyhow.workspace = true
windmill-parser.workspace = true
windmill-parser-go = { workspace = true, optional = true }
windmill-parser-bash = { workspace = true, optional = true }
windmill-parser-sql = { workspace = true, optional = true }
windmill-parser-py = { workspace = true, optional = true }
windmill-parser-ts = { workspace = true, optional = true }
windmill-parser-php = { workspace = true, optional = true }
windmill-parser-graphql = { workspace = true, optional = true }
windmill-parser-rust = { workspace = true, optional = true }
windmill-parser-go.workspace = true
windmill-parser-bash.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-py.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-php.workspace = true
windmill-parser-graphql.workspace = true
wasm-bindgen.workspace = true
serde_json.workspace = true
getrandom = { workspace = true, features = ["js"] }

View File

@@ -1,25 +0,0 @@
### Windmill parser wasm
How to build
```
wasm-pack build --release --target web
```
#### Prerequisite
Install wasm-pack
```
cargo install wasm-pack
```
#### To use it on a dev environment
Go to frontend and run:
```
npm install ../backend/parsers/windmill-parser-wasm/pkg
```
Make sure to not reset the package.json before commiting

View File

@@ -1,44 +0,0 @@
#!/bin/bash
set -eou pipefail
# full pkg
OUT_DIR="pkg"
wasm-pack build --release --target web --out-dir $OUT_DIR --all-features \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
# bun and deno
OUT_DIR="pkg-ts"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "ts-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
# sql languages, graphql and bash/powershell, since they all use regex
OUT_DIR="pkg-regex"
wasm-pack build --release --target web --out-dir $OUT_DIR \
--features "sql-parser,graphql-parser,bash-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
# python
OUT_DIR="pkg-py"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "py-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
# go
OUT_DIR="pkg-go"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "go-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
# php
OUT_DIR="pkg-php"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
# rust
OUT_DIR="pkg-rust"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json

View File

@@ -3,7 +3,7 @@
"collaborators": [
"Ruben Fiszel <ruben@windmill.dev>"
],
"version": "1.385.0",
"version": "1.367.2",
"files": [
"windmill_parser_wasm_bg.wasm",
"windmill_parser_wasm.js",

View File

@@ -75,11 +75,6 @@ export function parse_graphql(code: string): string;
* @returns {string}
*/
export function parse_php(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_rust(code: string): string;
export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
@@ -100,7 +95,6 @@ export interface InitOutput {
readonly parse_db_resource: (a: number, b: number, c: number) => void;
readonly parse_graphql: (a: number, b: number, c: number) => void;
readonly parse_php: (a: number, b: number, c: number) => void;
readonly parse_rust: (a: number, b: number, c: number) => void;
readonly __wbindgen_malloc: (a: number, b: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
@@ -113,18 +107,18 @@ export type SyncInitInput = BufferSource | WebAssembly.Module;
* Instantiates the given `module`, which can either be bytes or
* a precompiled `WebAssembly.Module`.
*
* @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
* @param {SyncInitInput} module
*
* @returns {InitOutput}
*/
export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
export function initSync(module: SyncInitInput): InitOutput;
/**
* If `module_or_path` is {RequestInfo} or {URL}, makes a request and
* for everything else, calls `WebAssembly.instantiate` directly.
*
* @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
* @param {InitInput | Promise<InitInput>} module_or_path
*
* @returns {Promise<InitOutput>}
*/
export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;
export default function __wbg_init (module_or_path?: InitInput | Promise<InitInput>): Promise<InitOutput>;

View File

@@ -22,13 +22,13 @@ function takeObject(idx) {
let WASM_VECTOR_LEN = 0;
let cachedUint8ArrayMemory0 = null;
let cachedUint8Memory0 = null;
function getUint8ArrayMemory0() {
if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
function getUint8Memory0() {
if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8ArrayMemory0;
return cachedUint8Memory0;
}
const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
@@ -51,7 +51,7 @@ function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length, 1) >>> 0;
getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
@@ -59,7 +59,7 @@ function passStringToWasm0(arg, malloc, realloc) {
let len = arg.length;
let ptr = malloc(len, 1) >>> 0;
const mem = getUint8ArrayMemory0();
const mem = getUint8Memory0();
let offset = 0;
@@ -74,7 +74,7 @@ function passStringToWasm0(arg, malloc, realloc) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
offset += ret.written;
@@ -89,13 +89,22 @@ function isLikeNone(x) {
return x === undefined || x === null;
}
let cachedDataViewMemory0 = null;
let cachedInt32Memory0 = null;
function getDataViewMemory0() {
if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
function getInt32Memory0() {
if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) {
cachedInt32Memory0 = new Int32Array(wasm.memory.buffer);
}
return cachedDataViewMemory0;
return cachedInt32Memory0;
}
let cachedFloat64Memory0 = null;
function getFloat64Memory0() {
if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
}
return cachedFloat64Memory0;
}
function addHeapObject(obj) {
@@ -113,7 +122,16 @@ if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
}
let cachedBigInt64Memory0 = null;
function getBigInt64Memory0() {
if (cachedBigInt64Memory0 === null || cachedBigInt64Memory0.byteLength === 0) {
cachedBigInt64Memory0 = new BigInt64Array(wasm.memory.buffer);
}
return cachedBigInt64Memory0;
}
function debugString(val) {
@@ -192,8 +210,8 @@ export function parse_deno(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_deno(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -215,8 +233,8 @@ export function parse_outputs(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_outputs(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -238,8 +256,8 @@ export function parse_ts_imports(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_ts_imports(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -261,8 +279,8 @@ export function parse_bash(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_bash(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -284,8 +302,8 @@ export function parse_powershell(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_powershell(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -307,8 +325,8 @@ export function parse_go(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_go(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -330,8 +348,8 @@ export function parse_python(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_python(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -353,8 +371,8 @@ export function parse_sql(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_sql(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -376,8 +394,8 @@ export function parse_mysql(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_mysql(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -399,8 +417,8 @@ export function parse_bigquery(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_bigquery(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -422,8 +440,8 @@ export function parse_snowflake(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_snowflake(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -445,8 +463,8 @@ export function parse_mssql(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_mssql(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -466,8 +484,8 @@ export function parse_db_resource(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_db_resource(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
let v2;
if (r0 !== 0) {
v2 = getStringFromWasm0(r0, r1).slice();
@@ -491,8 +509,8 @@ export function parse_graphql(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_graphql(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -514,8 +532,8 @@ export function parse_php(code) {
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_php(retptr, ptr0, len0);
var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
@@ -567,7 +585,7 @@ async function __wbg_load(module, imports) {
function __wbg_get_imports() {
const imports = {};
imports.wbg = {};
imports.wbg.__wbg_eval_204ec87eadeaef4a = function(arg0, arg1) {
imports.wbg.__wbg_eval_aa725d466edcea2c = function(arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
@@ -579,8 +597,8 @@ function __wbg_get_imports() {
const ret = typeof(obj) === 'string' ? obj : undefined;
var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
var len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
};
imports.wbg.__wbindgen_boolean_get = function(arg0) {
const v = getObject(arg0);
@@ -594,8 +612,8 @@ function __wbg_get_imports() {
imports.wbg.__wbindgen_number_get = function(arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof(obj) === 'number' ? obj : undefined;
getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true);
getFloat64Memory0()[arg0 / 8 + 1] = isLikeNone(ret) ? 0 : ret;
getInt32Memory0()[arg0 / 4 + 0] = !isLikeNone(ret);
};
imports.wbg.__wbindgen_is_object = function(arg0) {
const val = getObject(arg0);
@@ -626,11 +644,11 @@ function __wbg_get_imports() {
const ret = getObject(arg0) == getObject(arg1);
return ret;
};
imports.wbg.__wbg_get_3baa728f9d58d3f6 = function(arg0, arg1) {
imports.wbg.__wbg_get_bd8e338fbd5f5cc8 = function(arg0, arg1) {
const ret = getObject(arg0)[arg1 >>> 0];
return addHeapObject(ret);
};
imports.wbg.__wbg_length_ae22078168b726f5 = function(arg0) {
imports.wbg.__wbg_length_cd7af8117672b8b8 = function(arg0) {
const ret = getObject(arg0).length;
return ret;
};
@@ -638,39 +656,39 @@ function __wbg_get_imports() {
const ret = typeof(getObject(arg0)) === 'function';
return ret;
};
imports.wbg.__wbg_next_de3e9db4440638b2 = function(arg0) {
imports.wbg.__wbg_next_40fc327bfc8770e6 = function(arg0) {
const ret = getObject(arg0).next;
return addHeapObject(ret);
};
imports.wbg.__wbg_next_f9cb570345655b9a = function() { return handleError(function (arg0) {
imports.wbg.__wbg_next_196c84450b364254 = function() { return handleError(function (arg0) {
const ret = getObject(arg0).next();
return addHeapObject(ret);
}, arguments) };
imports.wbg.__wbg_done_bfda7aa8f252b39f = function(arg0) {
imports.wbg.__wbg_done_298b57d23c0fc80c = function(arg0) {
const ret = getObject(arg0).done;
return ret;
};
imports.wbg.__wbg_value_6d39332ab4788d86 = function(arg0) {
imports.wbg.__wbg_value_d93c65011f51a456 = function(arg0) {
const ret = getObject(arg0).value;
return addHeapObject(ret);
};
imports.wbg.__wbg_iterator_888179a48810a9fe = function() {
imports.wbg.__wbg_iterator_2cee6dadfd956dfa = function() {
const ret = Symbol.iterator;
return addHeapObject(ret);
};
imports.wbg.__wbg_get_224d16597dbbfd96 = function() { return handleError(function (arg0, arg1) {
imports.wbg.__wbg_get_e3c254076557e348 = function() { return handleError(function (arg0, arg1) {
const ret = Reflect.get(getObject(arg0), getObject(arg1));
return addHeapObject(ret);
}, arguments) };
imports.wbg.__wbg_call_1084a111329e68ce = function() { return handleError(function (arg0, arg1) {
imports.wbg.__wbg_call_27c0f87801dedf93 = function() { return handleError(function (arg0, arg1) {
const ret = getObject(arg0).call(getObject(arg1));
return addHeapObject(ret);
}, arguments) };
imports.wbg.__wbg_isArray_8364a5371e9737d8 = function(arg0) {
imports.wbg.__wbg_isArray_2ab64d95e09ea0ae = function(arg0) {
const ret = Array.isArray(getObject(arg0));
return ret;
};
imports.wbg.__wbg_instanceof_ArrayBuffer_61dfc3198373c902 = function(arg0) {
imports.wbg.__wbg_instanceof_ArrayBuffer_836825be07d4c9d2 = function(arg0) {
let result;
try {
result = getObject(arg0) instanceof ArrayBuffer;
@@ -680,7 +698,7 @@ function __wbg_get_imports() {
const ret = result;
return ret;
};
imports.wbg.__wbg_instanceof_Map_763ce0e95960d55e = function(arg0) {
imports.wbg.__wbg_instanceof_Map_87917e0a7aaf4012 = function(arg0) {
let result;
try {
result = getObject(arg0) instanceof Map;
@@ -690,30 +708,30 @@ function __wbg_get_imports() {
const ret = result;
return ret;
};
imports.wbg.__wbg_isSafeInteger_7f1ed56200d90674 = function(arg0) {
imports.wbg.__wbg_isSafeInteger_f7b04ef02296c4d2 = function(arg0) {
const ret = Number.isSafeInteger(getObject(arg0));
return ret;
};
imports.wbg.__wbg_entries_7a0e06255456ebcd = function(arg0) {
imports.wbg.__wbg_entries_95cc2c823b285a09 = function(arg0) {
const ret = Object.entries(getObject(arg0));
return addHeapObject(ret);
};
imports.wbg.__wbg_buffer_b7b08af79b0b0974 = function(arg0) {
imports.wbg.__wbg_buffer_12d079cc21e14bdb = function(arg0) {
const ret = getObject(arg0).buffer;
return addHeapObject(ret);
};
imports.wbg.__wbg_new_ea1883e1e5e86686 = function(arg0) {
imports.wbg.__wbg_new_63b92bc8671ed464 = function(arg0) {
const ret = new Uint8Array(getObject(arg0));
return addHeapObject(ret);
};
imports.wbg.__wbg_set_d1e79e2388520f18 = function(arg0, arg1, arg2) {
imports.wbg.__wbg_set_a47bac70306a19a7 = function(arg0, arg1, arg2) {
getObject(arg0).set(getObject(arg1), arg2 >>> 0);
};
imports.wbg.__wbg_length_8339fcf5d8ecd12e = function(arg0) {
imports.wbg.__wbg_length_c20a40f15020d68a = function(arg0) {
const ret = getObject(arg0).length;
return ret;
};
imports.wbg.__wbg_instanceof_Uint8Array_247a91427532499e = function(arg0) {
imports.wbg.__wbg_instanceof_Uint8Array_2b3bbecd033d19f6 = function(arg0) {
let result;
try {
result = getObject(arg0) instanceof Uint8Array;
@@ -726,15 +744,15 @@ function __wbg_get_imports() {
imports.wbg.__wbindgen_bigint_get_as_i64 = function(arg0, arg1) {
const v = getObject(arg1);
const ret = typeof(v) === 'bigint' ? v : undefined;
getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true);
getBigInt64Memory0()[arg0 / 8 + 1] = isLikeNone(ret) ? BigInt(0) : ret;
getInt32Memory0()[arg0 / 4 + 0] = !isLikeNone(ret);
};
imports.wbg.__wbindgen_debug_string = function(arg0, arg1) {
const ret = debugString(getObject(arg1));
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len1 = WASM_VECTOR_LEN;
getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
};
imports.wbg.__wbindgen_throw = function(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
@@ -747,16 +765,17 @@ function __wbg_get_imports() {
return imports;
}
function __wbg_init_memory(imports, memory) {
function __wbg_init_memory(imports, maybe_memory) {
}
function __wbg_finalize_init(instance, module) {
wasm = instance.exports;
__wbg_init.__wbindgen_wasm_module = module;
cachedDataViewMemory0 = null;
cachedUint8ArrayMemory0 = null;
cachedBigInt64Memory0 = null;
cachedFloat64Memory0 = null;
cachedInt32Memory0 = null;
cachedUint8Memory0 = null;
return wasm;
@@ -765,12 +784,6 @@ function __wbg_finalize_init(instance, module) {
function initSync(module) {
if (wasm !== undefined) return wasm;
if (typeof module !== 'undefined' && Object.getPrototypeOf(module) === Object.prototype)
({module} = module)
else
console.warn('using deprecated parameters for `initSync()`; pass a single object instead')
const imports = __wbg_get_imports();
__wbg_init_memory(imports);
@@ -784,30 +797,24 @@ function initSync(module) {
return __wbg_finalize_init(instance, module);
}
async function __wbg_init(module_or_path) {
async function __wbg_init(input) {
if (wasm !== undefined) return wasm;
if (typeof module_or_path !== 'undefined' && Object.getPrototypeOf(module_or_path) === Object.prototype)
({module_or_path} = module_or_path)
else
console.warn('using deprecated parameters for the initialization function; pass a single object instead')
if (typeof module_or_path === 'undefined') {
module_or_path = new URL('windmill_parser_wasm_bg.wasm', import.meta.url);
if (typeof input === 'undefined') {
input = new URL('windmill_parser_wasm_bg.wasm', import.meta.url);
}
const imports = __wbg_get_imports();
if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) {
module_or_path = fetch(module_or_path);
if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) {
input = fetch(input);
}
__wbg_init_memory(imports);
const { instance, module } = await __wbg_load(await module_or_path, imports);
const { instance, module } = await __wbg_load(await input, imports);
return __wbg_finalize_init(instance, module);
}
export { initSync };
export { initSync }
export default __wbg_init;

View File

@@ -16,7 +16,6 @@ export function parse_mssql(a: number, b: number, c: number): void;
export function parse_db_resource(a: number, b: number, c: number): void;
export function parse_graphql(a: number, b: number, c: number): void;
export function parse_php(a: number, b: number, c: number): void;
export function parse_rust(a: number, b: number, c: number): void;
export function __wbindgen_malloc(a: number, b: number): number;
export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number;
export function __wbindgen_add_to_stack_pointer(a: number): number;

View File

@@ -1,23 +0,0 @@
#!/bin/bash
set -eou pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"
args=${1:-}
pushd "pkg-ts" && npm publish ${args}
popd
pushd "pkg-regex" && npm publish ${args}
popd
pushd "pkg-py" && npm publish ${args}
popd
pushd "pkg-go" && npm publish ${args}
popd
pushd "pkg-php" && npm publish ${args}
popd
pushd "pkg-rust" && npm publish ${args}
popd

View File

@@ -1,12 +1,8 @@
#[cfg(feature = "ts-parser")]
use serde_json::json;
#[allow(unused_imports)]
use wasm_bindgen::prelude::*;
use windmill_parser::MainArgSignature;
#[cfg(feature = "ts-parser")]
use windmill_parser_ts::{parse_expr_for_ids, parse_expr_for_imports};
#[allow(dead_code)]
fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
if let Ok(r) = r {
return serde_json::to_string(&r).unwrap();
@@ -15,13 +11,11 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
}
}
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_deno(code: &str) -> String {
wrap_sig(windmill_parser_ts::parse_deno_signature(code, false, None))
}
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_outputs(code: &str) -> String {
let parsed = parse_expr_for_ids(code);
@@ -33,7 +27,6 @@ pub fn parse_outputs(code: &str) -> String {
return serde_json::to_string(&r).unwrap();
}
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_ts_imports(code: &str) -> String {
let parsed = parse_expr_for_imports(code);
@@ -45,80 +38,62 @@ pub fn parse_ts_imports(code: &str) -> String {
return serde_json::to_string(&r).unwrap();
}
#[cfg(feature = "bash-parser")]
#[wasm_bindgen]
pub fn parse_bash(code: &str) -> String {
wrap_sig(windmill_parser_bash::parse_bash_sig(code))
}
#[cfg(feature = "bash-parser")]
#[wasm_bindgen]
pub fn parse_powershell(code: &str) -> String {
wrap_sig(windmill_parser_bash::parse_powershell_sig(code))
}
#[cfg(feature = "go-parser")]
#[wasm_bindgen]
pub fn parse_go(code: &str) -> String {
wrap_sig(windmill_parser_go::parse_go_sig(code))
}
#[cfg(feature = "py-parser")]
#[wasm_bindgen]
pub fn parse_python(code: &str) -> String {
wrap_sig(windmill_parser_py::parse_python_signature(code, None))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_sql(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_pgsql_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_mysql(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_mysql_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_bigquery(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_bigquery_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_snowflake(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_snowflake_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_mssql(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_mssql_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_db_resource(code: &str) -> Option<String> {
windmill_parser_sql::parse_db_resource(code)
}
#[cfg(feature = "graphql-parser")]
#[wasm_bindgen]
pub fn parse_graphql(code: &str) -> String {
wrap_sig(windmill_parser_graphql::parse_graphql_sig(code))
}
#[cfg(feature = "php-parser")]
#[wasm_bindgen]
pub fn parse_php(code: &str) -> String {
wrap_sig(windmill_parser_php::parse_php_signature(code, None))
}
#[cfg(feature = "rust-parser")]
#[wasm_bindgen]
pub fn parse_rust(code: &str) -> String {
wrap_sig(windmill_parser_rust::parse_rust_signature(code))
}

View File

@@ -11,4 +11,3 @@ path = "./src/lib.rs"
[dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_json.workspace = true
convert_case.workspace = true

View File

@@ -6,7 +6,6 @@
* LICENSE-AGPL for a copy of the license.
*/
use convert_case::{Boundary, Case, Casing};
use serde::Serialize;
use serde_json::Value;
@@ -76,88 +75,3 @@ pub fn json_to_typ(js: &Value) -> Typ {
_ => Typ::Unknown,
}
}
pub fn to_snake_case(s: &str) -> String {
s.with_boundaries(&Boundary::defaults())
.without_boundaries(&Boundary::letter_digit())
.to_case(Case::Snake)
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_snake_case() {
assert_eq!("s3", to_snake_case("S3"));
assert_eq!("s3", to_snake_case("s3"));
assert_eq!("s_3", to_snake_case("S_3"));
assert_eq!("type_name_here", to_snake_case("typeNameHere"));
}
#[test]
fn test_empty_string() {
assert_eq!(to_snake_case(""), "");
}
#[test]
fn test_single_char_lowercase() {
assert_eq!(to_snake_case("a"), "a");
}
#[test]
fn test_single_char_uppercase() {
assert_eq!(to_snake_case("A"), "a");
}
#[test]
fn test_all_uppercase() {
assert_eq!(to_snake_case("TEST"), "test");
}
#[test]
fn test_all_lowercase() {
assert_eq!(to_snake_case("test"), "test");
}
#[test]
fn test_mixed_case() {
assert_eq!(to_snake_case("testCase"), "test_case");
}
#[test]
fn test_mixed_case_with_numbers() {
assert_eq!(to_snake_case("testCase1"), "test_case1");
assert_eq!(to_snake_case("Test123Case"), "test123_case");
}
#[test]
fn test_numbers_with_hyphen() {
assert_eq!(to_snake_case("test-3"), "test_3");
}
#[test]
fn test_string_with_spaces() {
assert_eq!(to_snake_case("This is a Test"), "this_is_a_test");
}
#[test]
fn test_snake_case_input() {
assert_eq!(to_snake_case("already_snake_case"), "already_snake_case");
}
#[test]
fn test_kebab_case_input() {
assert_eq!(to_snake_case("already-kebab-case"), "already_kebab_case");
}
#[test]
fn test_mixed_delimiters() {
assert_eq!(to_snake_case("test-Case_with Spaces"), "test_case_with_spaces");
}
#[test]
fn test_leading_and_trailing_spaces() {
assert_eq!(to_snake_case(" test case "), "test_case");
}
}

View File

@@ -7,8 +7,8 @@
*/
use anyhow::Context;
use gethostname::gethostname;
use git_version::git_version;
use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store};
use rand::Rng;
use sqlx::{postgres::PgListener, Pool, Postgres};
use std::{
@@ -20,7 +20,6 @@ use tokio::{
fs::{create_dir_all, DirBuilder, File},
io::AsyncReadExt,
};
use uuid::Uuid;
use windmill_api::HTTP_CLIENT;
#[cfg(feature = "enterprise")]
@@ -37,10 +36,9 @@ use windmill_common::{
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
},
scripts::ScriptLang,
stats_ee::schedule_stats,
utils::{hostname, rd_string, Mode},
worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, WORKER_GROUP},
utils::{rd_string, Mode},
worker::{reload_custom_tags_setting, WORKER_GROUP},
DB, METRICS_ENABLED,
};
@@ -63,8 +61,8 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
};
use crate::monitor::{
@@ -136,25 +134,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
create_dir_all(HUB_CACHE_DIR).await?;
for path in paths.values() {
tracing::info!("Caching hub script at {path}");
let res = get_hub_script_content_and_requirements(Some(path.to_string()), None).await?;
if res.language.is_some_and(|x| x == ScriptLang::Deno) {
let job_dir = format!("{}/cache_init/{}", TMP_DIR, Uuid::new_v4());
create_dir_all(&job_dir).await?;
let _ = windmill_worker::generate_deno_lock(
&Uuid::nil(),
&res.content,
&mut 0,
&mut None,
"/tmp/windmill/",
None,
"global",
"global",
"",
)
.await?;
tokio::fs::remove_dir_all(job_dir).await?;
}
get_hub_script_content_and_requirements(Some(path.to_string()), None).await?;
}
Ok(())
}
@@ -166,10 +146,8 @@ async fn windmill_main() -> anyhow::Result<()> {
std::env::set_var("RUST_LOG", "info")
}
let hostname = hostname();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname);
windmill_common::tracing_init::initialize_tracing();
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
tracing::info!("jemalloc enabled");
@@ -400,8 +378,6 @@ Windmill Community Edition {GIT_VERSION}
monitor_pool(&db).await;
send_logs_to_object_store(&db, &hostname, &mode);
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
if !worker_mode {
monitor_mem().await;
@@ -487,7 +463,6 @@ Windmill Community Edition {GIT_VERSION}
base_internal_url.clone(),
rsmq.clone(),
mode.clone() == Mode::Agent,
hostname.clone(),
)
.await?;
tracing::info!("All workers exited.");
@@ -718,8 +693,6 @@ Windmill Community Edition {GIT_VERSION}
} else {
tracing::info!("Nothing to do, exiting.");
}
send_current_log_file_to_object_store(&db, &hostname, &mode).await;
tracing::info!("Exiting connection pool");
tokio::select! {
_ = db.close() => {
@@ -791,7 +764,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
base_internal_url: String,
rsmq: Option<R>,
agent_mode: bool,
hostname: String,
) -> anyhow::Result<()> {
let mut killpill_rxs = vec![];
for _ in 0..num_workers {
@@ -802,6 +774,10 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
tracing::info!("Received killpill, exiting");
return Ok(());
}
let hostname = gethostname()
.to_str()
.map(|x| x.to_string())
.unwrap_or_else(|| rd_string(5));
let instance_name = hostname
.clone()
.replace(" ", "")
@@ -836,7 +812,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
BUN_BUNDLE_CACHE_DIR,
GO_CACHE_DIR,
GO_BIN_CACHE_DIR,
RUST_CACHE_DIR,
HUB_CACHE_DIR,
POWERSHELL_CACHE_DIR,
] {

View File

@@ -7,7 +7,6 @@ use std::{
time::Duration,
};
use chrono::{NaiveDateTime, Utc};
use rsmq_async::MultiplexedRsmq;
use serde::de::DeserializeOwned;
use sqlx::{Pool, Postgres};
@@ -22,8 +21,6 @@ use windmill_api::{
oauth2_ee::{build_oauth_clients, OAuthClient},
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
};
#[cfg(feature = "enterprise")]
use windmill_common::ee::worker_groups_alerts;
use windmill_common::{
auth::JWT_SECRET,
ee::CriticalErrorChannel,
@@ -42,13 +39,11 @@ use windmill_common::{
jobs::QueuedJob,
oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
server::load_server_config,
tracing_init::JSON_FMT,
users::truncate_token,
utils::{now_from_db, rd_string, report_critical_error, Mode},
utils::{now_from_db, rd_string},
worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG,
WORKER_GROUP,
},
BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
METRICS_DEBUG_ENABLED, METRICS_ENABLED,
@@ -339,195 +334,6 @@ pub async fn monitor_mem() {
});
}
async fn sleep_until_next_minute_start_plus_one_s() {
let now = Utc::now();
let next_minute = now + Duration::from_secs(60 - now.timestamp() as u64 % 60 + 1);
tokio::time::sleep(tokio::time::Duration::from_secs(
next_minute.timestamp() as u64 - now.timestamp() as u64,
))
.await;
}
use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE;
async fn find_two_highest_files(hostname: &str) -> (Option<String>, Option<String>) {
let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname);
let rd_dir = tokio::fs::read_dir(log_dir).await;
if let Ok(mut log_files) = rd_dir {
let mut highest_file: Option<String> = None;
let mut second_highest_file: Option<String> = None;
while let Ok(Some(file)) = log_files.next_entry().await {
let file_name = file
.file_name()
.to_str()
.map(|x| x.to_string())
.unwrap_or_default();
if file_name > highest_file.clone().unwrap_or_default() {
second_highest_file = highest_file;
highest_file = Some(file_name);
}
}
(highest_file, second_highest_file)
} else {
tracing::error!(
"Error reading log files: {TMP_WINDMILL_LOGS_SERVICE}, {:#?}",
rd_dir.unwrap_err()
);
(None, None)
}
}
fn get_worker_group(mode: &Mode) -> Option<String> {
let worker_group = WORKER_GROUP.clone();
if worker_group.is_empty() || mode == &Mode::Server || mode == &Mode::Indexer {
None
} else {
Some(worker_group)
}
}
pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
let db = db.clone();
let hostname = hostname.to_string();
let mode = mode.clone();
let worker_group = get_worker_group(&mode);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(10));
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
sleep_until_next_minute_start_plus_one_s().await;
loop {
interval.tick().await;
let (_, snd_highest_file) = find_two_highest_files(&hostname).await;
send_log_file_to_object_store(
&hostname,
&mode,
&worker_group,
&db,
snd_highest_file,
false,
)
.await;
}
});
}
pub async fn send_current_log_file_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
tracing::info!("Sending current log file to object store");
let (highest_file, _) = find_two_highest_files(hostname).await;
let worker_group = get_worker_group(&mode);
send_log_file_to_object_store(hostname, mode, &worker_group, db, highest_file, true).await;
}
fn get_now_and_str() -> (NaiveDateTime, String) {
let ts = Utc::now().naive_utc();
(
ts,
ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT)
.to_string(),
)
}
async fn send_log_file_to_object_store(
hostname: &str,
mode: &Mode,
worker_group: &Option<String>,
db: &Pool<Postgres>,
snd_highest_file: Option<String>,
use_now: bool,
) {
if let Some(highest_file) = snd_highest_file {
//parse datetime frome file xxxx.yyyy-MM-dd-HH-mm
let (ts, ts_str) = if use_now {
get_now_and_str()
} else {
highest_file
.split(".")
.last()
.and_then(|x| {
NaiveDateTime::parse_from_str(
x,
windmill_common::tracing_init::LOG_TIMESTAMP_FMT,
)
.ok()
.map(|y| (y, x.to_string()))
})
.unwrap_or_else(get_now_and_str)
};
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM log_file WHERE hostname = $1 AND log_ts = $2)",
hostname,
ts
)
.fetch_one(db)
.await;
match exists {
Ok(Some(true)) => {
return;
}
Err(e) => {
tracing::error!("Error checking if log file exists: {:?}", e);
return;
}
_ => (),
}
#[cfg(feature = "parquet")]
let s3_client = OBJECT_STORE_CACHE_SETTINGS.read().await.clone();
#[cfg(feature = "parquet")]
if let Some(s3_client) = s3_client {
let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE)
.join(hostname)
.join(&highest_file);
//read file as byte stream
let bytes = tokio::fs::read(&path).await;
if let Err(e) = bytes {
tracing::error!("Error reading log file: {:?}", e);
return;
}
let path = object_store::path::Path::from_url_path(format!(
"{}{hostname}/{highest_file}",
windmill_common::tracing_init::LOGS_SERVICE
));
if let Err(e) = path {
tracing::error!("Error creating log file path: {:?}", e);
return;
}
if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await {
tracing::error!("Error sending logs to object store: {:?}", e);
}
}
let (ok_lines, err_lines) = read_log_counters(ts_str);
if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT)
.execute(db)
.await {
tracing::error!("Error inserting log file: {:?}", e);
}
}
}
fn read_log_counters(ts_str: String) -> (usize, usize) {
let counters = windmill_common::tracing_init::LOG_COUNTING_BY_MIN.read();
let mut ok_lines = 0;
let mut err_lines = 0;
if let Ok(ref c) = counters {
let counter = c.get(&ts_str);
if let Some(counter) = counter {
ok_lines = counter.non_error_count;
err_lines = counter.error_count;
} else {
println!("no counter found for {ts_str}");
}
} else {
println!("Error reading log counters 2");
}
(ok_lines, err_lines)
}
pub async fn load_keep_job_dir(db: &DB) {
let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await;
match value {
@@ -662,15 +468,6 @@ pub async fn delete_expired_items(db: &DB) -> () {
{
tracing::error!("Error deleting custom concurrency key: {:?}", e);
}
if let Err(e) = sqlx::query!(
"DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
job_retention_secs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting log file: {:?}", e);
}
}
}
Err(e) => {
@@ -1038,19 +835,11 @@ pub async fn monitor_db(
}
};
let worker_groups_alerts_f = async {
#[cfg(feature = "enterprise")]
if server_mode {
worker_groups_alerts(&db).await;
}
};
join!(
expired_items_f,
zombie_jobs_f,
expose_queue_metrics_f,
verify_license_key_f,
worker_groups_alerts_f
verify_license_key_f
);
}
@@ -1248,7 +1037,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
) {
if *RESTART_ZOMBIE_JOBS {
let restarted = sqlx::query!(
"UPDATE queue SET running = false, started_at = null
"UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n'
WHERE last_ping < now() - ($1 || ' seconds')::interval
AND running = true AND job_kind NOT IN ('flow', 'flowpreview', 'singlescriptflow') AND same_worker = false RETURNING id, workspace_id, last_ping",
*ZOMBIE_JOB_TIMEOUT,
@@ -1261,25 +1050,13 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
}
let base_url = BASE_URL.read().await.clone();
for r in restarted {
let last_ping = if let Some(x) = r.last_ping {
format!("last ping at {x}")
} else {
"no last ping".to_string()
};
let url = format!("{}/run/{}?workspace={}", base_url, r.id, r.workspace_id,);
let error_message = format!(
"Zombie job {} on {} ({}) detected, restarting it, {}",
r.id, r.workspace_id, url, last_ping
tracing::error!(
"Zombie job detected, restarting it: {} {} {:?}",
r.id,
r.workspace_id,
r.last_ping
);
let _ = sqlx::query!("
INSERT INTO job_logs (job_id, logs) VALUES ($1,'Restarted job after not receiving job''s ping for too long the ' || now() || '\n\n')
ON CONFLICT (job_id) DO UPDATE SET logs = job_logs.logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE job_logs.job_id = $1", r.id)
.execute(db).await;
tracing::error!(error_message);
report_critical_error(error_message, db.clone()).await;
}
}
@@ -1382,12 +1159,11 @@ async fn handle_zombie_flows(
.get(0)
.is_some_and(|x| matches!(x, FlowStatusModule::WaitingForPriorSteps { .. }))
}) {
let error_message = format!(
tracing::error!(
"Zombie flow detected: {} in workspace {}. It hasn't started yet, restarting it.",
flow.id, flow.workspace_id
flow.id,
flow.workspace_id
);
tracing::error!(error_message);
report_critical_error(error_message, db.clone()).await;
// if the flow hasn't started and is a zombie, we can simply restart it
sqlx::query!(
"UPDATE queue SET running = false, started_at = null WHERE id = $1 AND canceled = false",
@@ -1407,7 +1183,6 @@ async fn handle_zombie_flows(
format!("Flow {id} was cancelled because it")
}
);
report_critical_error(reason.clone(), db.clone()).await;
cancel_zombie_flow_job(db, flow, &rsmq, reason).await?;
}
}

View File

@@ -75,9 +75,7 @@ async fn initialize_tracing() {
use std::sync::Once;
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let _ = windmill_common::tracing_init::initialize_tracing("test");
});
ONCE.call_once(windmill_common::tracing_init::initialize_tracing);
}
/// it's important this is unique between tests as there is one prometheus registry and
@@ -1115,7 +1113,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
}
.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1155,7 +1152,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
}
.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1170,7 +1166,6 @@ async fn test_deno_flow(db: Pool<Postgres>) {
}
.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1275,7 +1270,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1325,7 +1319,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1361,7 +1354,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1377,7 +1369,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
],
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1420,7 +1411,6 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrency_time_window_s: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
summary: Default::default(),
suspend: Default::default(),
retry: None,
@@ -1743,41 +1733,6 @@ func main(derp string) (string, error) {
assert_eq!(result, serde_json::json!("hello world"));
}
#[sqlx::test(fixtures("base"))]
async fn test_rust_job(db: Pool<Postgres>) {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await;
let port = server.addr.port();
let content = r#"
fn main(world: String) -> Result<String, String> {
println!("Which world to greet today?");
Ok(format!("Hello {}!", world))
}
"#
.to_owned();
let result = RunJob::from(JobPayload::Code(RawCode {
hash: None,
content,
path: None,
lock: None,
language: ScriptLang::Rust,
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: None,
}))
.arg("world", json!("Hyrule"))
.run_until_complete(&db, port)
.await
.json_result()
.unwrap();
assert_eq!(result, serde_json::json!("Hello Hyrule!"));
}
#[sqlx::test(fixtures("base"))]
async fn test_bash_job(db: Pool<Postgres>) {
initialize_tracing().await;
@@ -2712,7 +2667,7 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
in_test_worker(&db, listen_first_job, port).await;
let modules = client
.get_flow_by_path("test-workspace", "g/all/flow_lock_all", None)
.get_flow_by_path("test-workspace", "g/all/flow_lock_all")
.await
.unwrap()
.into_inner()

View File

@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.390.1
version: 1.381.0
title: Windmill API
contact:
@@ -342,8 +342,6 @@ paths:
properties:
is_super_admin:
type: boolean
name:
type: string
responses:
"200":
description: user updated
@@ -3926,10 +3924,6 @@ paths:
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
- name: with_starred_info
in: query
schema:
type: boolean
responses:
"200":
description: script details
@@ -4065,10 +4059,6 @@ paths:
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptHash"
- name: with_starred_info
in: query
schema:
type: boolean
responses:
"200":
description: script details
@@ -4525,10 +4515,6 @@ paths:
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
- name: with_starred_info
in: query
schema:
type: boolean
responses:
"200":
description: flow details
@@ -4946,10 +4932,6 @@ paths:
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
- name: with_starred_info
in: query
schema:
type: boolean
responses:
"200":
description: app details
@@ -6137,7 +6119,7 @@ paths:
/w/{workspace}/jobs_u/queue/cancel/{id}:
post:
summary: cancel queued or running job
summary: cancel queued job
operationId: cancelQueuedJob
tags:
- job
@@ -8603,71 +8585,6 @@ paths:
items:
$ref: "#/components/schemas/TimeseriesMetric"
/service_logs/list_files:
get:
summary: list log files ordered by timestamp
operationId: listLogFiles
tags:
- service_logs
parameters:
- $ref: "#/components/parameters/Before"
- $ref: "#/components/parameters/After"
- name: with_error
in: query
required: false
schema:
type: boolean
responses:
"200":
description: time
content:
application/json:
schema:
type: array
items:
type: object
properties:
hostname:
type: string
mode:
type: string
worker_group:
type: string
log_ts:
type: string
format: date-time
file_path:
type: string
ok_lines:
type: integer
err_lines:
type: integer
json_fmt:
type: boolean
required:
- hostname
- mode
- log_ts
- file_path
- json_fmt
/service_logs/get_log_file/{path}:
get:
summary: get log file by path
operationId: getLogFile
tags:
- service_logs
parameters:
- $ref: "#/components/parameters/Path"
responses:
"200":
description: log stream
content:
text/plain:
schema:
type: string
/concurrency_groups/list:
get:
summary: List all concurrency groups
@@ -9039,13 +8956,6 @@ components:
schema:
type: string
format: date-time
Before:
name: before
description: filter on started before (inclusive) timestamp
in: query
schema:
type: string
format: date-time
CreatedOrStartedAfter:
name: created_or_started_after
description:
@@ -9122,6 +9032,13 @@ components:
schema:
type: string
format: date-time
Before:
name: before
description: filter on created before (exclusive) timestamp
in: query
schema:
type: string
format: date-time
Username:
name: username
description: filter on exact username of user
@@ -9252,7 +9169,6 @@ components:
nativets,
bun,
php,
rust,
]
kind:
type: string
@@ -9351,7 +9267,6 @@ components:
nativets,
bun,
php,
rust,
]
kind:
type: string
@@ -9569,7 +9484,6 @@ components:
nativets,
bun,
php,
rust,
]
email:
type: string
@@ -9685,7 +9599,6 @@ components:
nativets,
bun,
php,
rust,
]
is_skipped:
type: boolean
@@ -10224,8 +10137,7 @@ components:
graphql,
nativets,
bun,
php,
rust,
php
]
tag:
type: string
@@ -10652,9 +10564,9 @@ components:
type: string
wm_version:
type: string
last_job_id:
current_job_id:
type: string
last_job_workspace_id:
current_job_workspace_id:
type: string
occupancy_rate:
type: number
@@ -11355,8 +11267,7 @@ components:
graphql,
nativets,
bun,
php,
rust,
php
]
required:
- raw_code

View File

@@ -11,7 +11,6 @@ use crate::{
db::{ApiAuthed, DB},
resources::get_resource_value_interpolated_internal,
users::{require_owner_of_path, OptAuthed},
utils::WithStarredInfoQuery,
variables::encrypt,
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
@@ -119,15 +118,6 @@ pub struct AppWithLastVersion {
pub extra_perms: Option<serde_json::Value>,
}
#[derive(Serialize, FromRow)]
pub struct AppWithLastVersionAndStarred {
#[sqlx(flatten)]
#[serde(flatten)]
pub app: AppWithLastVersion,
#[serde(skip_serializing_if = "Option::is_none")]
pub starred: Option<bool>,
}
#[derive(Serialize, Deserialize, FromRow)]
pub struct AppWithLastVersionAndDraft {
pub id: i64,
@@ -326,44 +316,20 @@ async fn get_app(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<WithStarredInfoQuery>,
) -> JsonResult<AppWithLastVersionAndStarred> {
) -> JsonResult<AppWithLastVersion> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = if query.with_starred_info.unwrap_or(false) {
sqlx::query_as::<_, AppWithLastVersionAndStarred>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by, favorite.path IS NOT NULL as starred
FROM app
JOIN app_version
ON app_version.id = app.versions[array_upper(app.versions, 1)]
LEFT JOIN favorite
ON favorite.favorite_kind = 'app'
AND favorite.workspace_id = app.workspace_id
AND favorite.path = app.path
AND favorite.usr = $3
WHERE app.path = $1 AND app.workspace_id = $2",
)
.bind(path.to_owned())
.bind(&w_id)
.bind(&authed.username)
.fetch_optional(&mut *tx)
.await?
} else {
sqlx::query_as::<_, AppWithLastVersionAndStarred>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by, NULL as starred
FROM app, app_version
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?
};
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let app = not_found_if_none(app_o, "App", path)?;

View File

@@ -248,78 +248,6 @@ async fn fix_flow_versioning_migration(
Ok(())
}
macro_rules! run_windmill_migration {
($migration_job_name:expr, $db:expr, $code:block) => {
{
let migration_job_name = $migration_job_name;
let db: &Pool<Postgres> = $db;
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
tracing::info!("Applying {migration_job_name} migration");
let mut tx = db.begin().await?;
let mut r = false;
while !r {
r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
sqlx::migrate::MigrateError::Execute(e)
})?
.unwrap_or(false);
if !r {
tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
tracing::info!("acquired lock for {migration_job_name}");
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
$code
sqlx::query!(
"INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
migration_job_name
)
.execute(&mut *tx)
.await?;
tracing::info!("Finished applying {migration_job_name} migration");
} else {
tracing::info!("migration {migration_job_name} already done");
}
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("released lock for {migration_job_name}");
tracing::info!("Finished applying {migration_job_name} migration");
} else {
tracing::info!("migration {migration_job_name} already done");
}
}
};
}
async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
// let has_done_migration = sqlx::query_scalar!(
// "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_job_completed_index')"
@@ -362,124 +290,284 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
// tx.commit().await?;
// }
run_windmill_migration!("fix_job_completed_index_2", &db, {
// sqlx::query(
// "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_created_at_new_2 ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, created_at DESC)"
// ).execute(db).await?;
let migration_job_name = "fix_job_completed_index_2";
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
tracing::info!("Applying {migration_job_name} migration");
let mut tx = db.begin().await?;
let mut r = false;
while !r {
r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
sqlx::migrate::MigrateError::Execute(e)
})?
.unwrap_or(false);
if !r {
tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
// sqlx::query(
// "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_started_at_new ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, started_at DESC)"
// ).execute(db).await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at")
.execute(db)
.await?;
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new",
tracing::info!("acquired lock for {migration_job_name}");
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.execute(db)
.await?;
});
.fetch_one(db)
.await?
.unwrap_or(false);
run_windmill_migration!("fix_job_completed_index_3", &db, {
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_job_on_schedule_path")
if !has_done_migration {
// sqlx::query(
// "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_created_at_new_2 ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, created_at DESC)"
// ).execute(db).await?;
// sqlx::query(
// "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_started_at_new ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, started_at DESC)"
// ).execute(db).await?;
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at",
)
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS concurrency_limit_stats_queue")
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new",
)
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index")
sqlx::query!(
"INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
migration_job_name
)
.execute(&mut *tx)
.await?;
tracing::info!("Finished applying {migration_job_name} migration");
} else {
tracing::info!("migration {migration_job_name} already done");
}
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("released lock for {migration_job_name}");
}
let migration_job_name = "fix_job_completed_index_3";
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
tracing::info!("Applying {migration_job_name} migration");
let mut tx = db.begin().await?;
let mut r = false;
while !r {
r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
sqlx::migrate::MigrateError::Execute(e)
})?
.unwrap_or(false);
if !r {
tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
tracing::info!("acquired lock for {migration_job_name}");
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_job_on_schedule_path")
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS concurrency_limit_stats_queue")
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index")
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_on_created")
.execute(db)
.await?;
sqlx::query!(
"INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
migration_job_name
)
.execute(&mut *tx)
.await?;
tracing::info!("Finished applying {migration_job_name} migration");
} else {
tracing::info!("migration {migration_job_name} already done");
}
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("released lock for {migration_job_name}");
}
let migration_job_name = "fix_job_completed_index_4";
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
tracing::info!("Applying {migration_job_name} migration");
let mut tx = db.begin().await?;
let mut r = false;
while !r {
r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
sqlx::migrate::MigrateError::Execute(e)
})?
.unwrap_or(false);
if !r {
tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
tracing::info!("acquired lock for {migration_job_name}");
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
let mut i = 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_3 ON completed_job (workspace_id, created_at DESC)")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_8 ON completed_job (workspace_id, created_at DESC) where job_kind in ('deploymentcallback') AND parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_on_created")
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_9 ON completed_job (workspace_id, created_at DESC) where job_kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL")
.execute(db)
.await?;
});
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
run_windmill_migration!("fix_job_completed_index_4", &db, {
let migration_job_name = "fix_job_completed_index_4";
let mut i = 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_3 ON completed_job (workspace_id, created_at DESC)")
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_5 ON completed_job (workspace_id, created_at DESC) where job_kind in ('preview', 'flowpreview') AND parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_8 ON completed_job (workspace_id, created_at DESC) where job_kind in ('deploymentcallback') AND parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_9 ON completed_job (workspace_id, created_at DESC) where job_kind in ('dependencies', 'flowdependencies', 'appdependencies') AND parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_5 ON completed_job (workspace_id, created_at DESC) where job_kind in ('preview', 'flowpreview') AND parent_job IS NULL")
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_6 ON completed_job (workspace_id, created_at DESC) where job_kind in ('script', 'flow') AND parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_6 ON completed_job (workspace_id, created_at DESC) where job_kind in ('script', 'flow') AND parent_job IS NULL")
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_7 ON completed_job (workspace_id, success, created_at DESC) where job_kind in ('script', 'flow') AND parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_created_at_new_7 ON completed_job (workspace_id, success, created_at DESC) where job_kind in ('script', 'flow') AND parent_job IS NULL")
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON completed_job (workspace_id, started_at DESC)")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_workspace_id_started_at_new_2 ON completed_job (workspace_id, started_at DESC)")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists root_job_index_by_path_2 ON completed_job (workspace_id, script_path, created_at desc) WHERE parent_job IS NULL")
sqlx::query("create index concurrently if not exists root_job_index_by_path_2 ON completed_job (workspace_id, script_path, created_at desc) WHERE parent_job IS NULL")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("create index concurrently if not exists ix_completed_job_created_at ON completed_job (created_at DESC)")
sqlx::query("create index concurrently if not exists ix_completed_job_created_at ON completed_job (created_at DESC)")
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new_2",
)
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new",
)
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path")
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new_2",
)
.execute(db)
.await?;
});
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query(
"DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_started_at_new",
)
.execute(db)
.await?;
i += 1;
tracing::info!("step {i} of {migration_job_name} migration");
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index_by_path")
.execute(db)
.await?;
sqlx::query!(
"INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
migration_job_name
)
.execute(&mut *tx)
.await?;
tracing::info!("Finished applying {migration_job_name} migration");
} else {
tracing::info!("migration {migration_job_name} already done");
}
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("released lock for {migration_job_name}");
}
Ok(())
}

View File

@@ -9,7 +9,6 @@
use std::collections::HashMap;
use crate::db::ApiAuthed;
use crate::utils::WithStarredInfoQuery;
use crate::{
db::DB,
schedule::clear_schedule,
@@ -36,7 +35,7 @@ use windmill_common::HUB_BASE_URL;
use windmill_common::{
db::UserDB,
error::{self, to_anyhow, Error, JsonResult, Result},
flows::{Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow},
flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
jobs::JobPayload,
schedule::Schedule,
scripts::Schema,
@@ -877,31 +876,13 @@ async fn get_flow_by_path(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<WithStarredInfoQuery>,
) -> JsonResult<FlowWithStarred> {
) -> JsonResult<Flow> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let flow_o = if query.with_starred_info.unwrap_or(false) {
sqlx::query_as::<_, FlowWithStarred>(
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, favorite.path IS NOT NULL as starred
FROM flow
LEFT JOIN favorite
ON favorite.favorite_kind = 'flow'
AND favorite.workspace_id = flow.workspace_id
AND favorite.path = flow.path
AND favorite.usr = $3
LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 AND flow.workspace_id = $2"
)
.bind(path)
.bind(w_id)
.bind(&authed.username)
.fetch_optional(&mut *tx)
.await?
} else {
sqlx::query_as::<_, FlowWithStarred>(
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by, NULL as starred
let flow_o =
sqlx::query_as::<_, Flow>(
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by
FROM flow
LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 AND flow.workspace_id = $2"
@@ -909,8 +890,7 @@ async fn get_flow_by_path(
.bind(path)
.bind(w_id)
.fetch_optional(&mut *tx)
.await?
};
.await?;
tx.commit().await?;
let flow = not_found_if_none(flow_o, "Flow", path)?;
@@ -1164,7 +1144,6 @@ mod tests {
tag_override: None,
}),
stop_after_if: None,
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
@@ -1193,7 +1172,6 @@ mod tests {
expr: "foo = 'bar'".to_string(),
skip_if_stopped: false,
}),
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
@@ -1220,7 +1198,6 @@ mod tests {
expr: "previous.isEmpty()".to_string(),
skip_if_stopped: false,
}),
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,
@@ -1246,7 +1223,6 @@ mod tests {
expr: "previous.isEmpty()".to_string(),
skip_if_stopped: false,
}),
stop_after_all_iters_if: None,
summary: None,
suspend: Default::default(),
retry: None,

View File

@@ -33,7 +33,6 @@ use crate::add_webhook_allowed_origin;
use crate::concurrency_groups::join_concurrency_key;
use crate::db::ApiAuthed;
use crate::utils::content_plain;
use crate::{
db::DB,
users::{check_scopes, require_owner_of_path, OptAuthed},
@@ -848,6 +847,14 @@ async fn get_logs_from_disk(
return None;
}
fn content_plain(body: Body) -> Response {
use axum::http::header;
Response::builder()
.header(header::CONTENT_TYPE, "text/plain")
.body(body)
.unwrap()
}
async fn get_job_logs(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
@@ -1029,17 +1036,17 @@ pub struct ListableCompletedJob {
#[derive(Deserialize, Clone, Default)]
pub struct RunJobQuery {
pub scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
pub scheduled_in_secs: Option<i64>,
pub parent_job: Option<Uuid>,
pub root_job: Option<Uuid>,
pub invisible_to_owner: Option<bool>,
pub queue_limit: Option<i64>,
pub payload: Option<String>,
pub job_id: Option<Uuid>,
pub tag: Option<String>,
pub timeout: Option<i32>,
pub cache_ttl: Option<i32>,
scheduled_for: Option<chrono::DateTime<chrono::Utc>>,
scheduled_in_secs: Option<i64>,
parent_job: Option<Uuid>,
root_job: Option<Uuid>,
invisible_to_owner: Option<bool>,
queue_limit: Option<i64>,
payload: Option<String>,
job_id: Option<Uuid>,
tag: Option<String>,
timeout: Option<i32>,
cache_ttl: Option<i32>,
}
impl RunJobQuery {
@@ -1571,11 +1578,7 @@ async fn list_jobs(
None
};
let sql = if lq.success.is_none()
&& lq.label.is_none()
&& lq.created_or_started_before.is_none()
&& lq.started_before.is_none()
{
let sql = if lq.success.is_none() && lq.label.is_none() {
let mut sqlq = list_queue_jobs_query(
&w_id,
&ListQueueQuery { order_desc: Some(true), ..lq.into() },
@@ -1595,11 +1598,6 @@ async fn list_jobs(
sqlq.limit(per_page).offset(offset).query()?
}
} else {
if sqlc.is_none() {
return Err(error::Error::BadRequest(
"cannot specify success, label, created_or_started_before, or started_before with running".to_string(),
));
}
sqlc.unwrap().limit(per_page).offset(offset).query()?
};
let mut tx = user_db.begin(&authed).await?;

View File

@@ -76,7 +76,6 @@ mod saml_ee;
mod schedule;
mod scim_ee;
mod scripts;
mod service_logs;
mod settings;
pub mod smtp_server_ee;
mod static_assets;
@@ -292,7 +291,6 @@ pub async fn run_server(
)
.nest("/settings", settings::global_service())
.nest("/workers", workers::global_service())
.nest("/service_logs", service_logs::global_service())
.nest("/configs", configs::global_service())
.nest("/scripts", scripts::global_service())
.nest("/integrations", integration::global_service())

View File

@@ -10,7 +10,6 @@ use crate::{
db::{ApiAuthed, DB},
schedule::clear_schedule,
users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
utils::WithStarredInfoQuery,
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
@@ -47,7 +46,7 @@ use windmill_common::{
schedule::Schedule,
scripts::{
to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang, ScriptWithStarred,
ScriptHistory, ScriptHistoryUpdate, ScriptKind, ScriptLang,
},
users::username_to_permissioned_as,
utils::{
@@ -527,8 +526,7 @@ async fn create_script_internal<'c>(
)));
};
let ScriptWithStarred { script: ps, .. } =
get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash, None).await?;
let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
if ps.path != ns.path {
require_owner_of_path(&authed, &ps.path)?;
@@ -580,7 +578,6 @@ async fn create_script_internal<'c>(
|| ns.language == ScriptLang::Bun
|| ns.language == ScriptLang::Bunnative
|| ns.language == ScriptLang::Deno
|| ns.language == ScriptLang::Rust
|| ns.language == ScriptLang::Php)
{
Some(String::new())
@@ -829,40 +826,19 @@ async fn get_script_by_path(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<WithStarredInfoQuery>,
) -> JsonResult<ScriptWithStarred> {
) -> JsonResult<Script> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let script_o = if query.with_starred_info.unwrap_or(false) {
sqlx::query_as::<_, ScriptWithStarred>(
"SELECT s.*, favorite.path IS NOT NULL as starred
FROM script s
LEFT JOIN favorite
ON favorite.favorite_kind = 'script'
AND favorite.workspace_id = s.workspace_id
AND favorite.path = s.path
AND favorite.usr = $3
WHERE s.path = $1
AND s.workspace_id = $2
AND s.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2)",
)
.bind(path)
.bind(w_id)
.bind(&authed.username)
.fetch_optional(&mut *tx)
.await?
} else {
sqlx::query_as::<_, ScriptWithStarred>(
"SELECT *, NULL as starred FROM script WHERE path = $1 AND workspace_id = $2 \
AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
workspace_id = $2)",
)
.bind(path)
.bind(w_id)
.fetch_optional(&mut *tx)
.await?
};
let script_o = sqlx::query_as::<_, Script>(
"SELECT * FROM script WHERE path = $1 AND workspace_id = $2 \
AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
workspace_id = $2)",
)
.bind(path)
.bind(w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let script = not_found_if_none(script_o, "Script", path)?;
@@ -1120,33 +1096,13 @@ async fn get_script_by_hash_internal<'c>(
db: &mut Transaction<'c, Postgres>,
workspace_id: &str,
hash: &ScriptHash,
with_starred_info_for_username: Option<&str>,
) -> Result<ScriptWithStarred> {
let script_o = if let Some(username) = with_starred_info_for_username {
sqlx::query_as::<_, ScriptWithStarred>(
"SELECT s.*, favorite.path IS NOT NULL as starred
FROM script s
LEFT JOIN favorite
ON favorite.favorite_kind = 'script'
AND favorite.workspace_id = s.workspace_id
AND favorite.path = s.path
AND favorite.usr = $1
WHERE s.hash = $2 AND s.workspace_id = $3",
)
.bind(&username)
.bind(hash)
.bind(workspace_id)
.fetch_optional(&mut **db)
.await?
} else {
sqlx::query_as::<_, ScriptWithStarred>(
"SELECT *, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2",
)
.bind(hash)
.bind(workspace_id)
.fetch_optional(&mut **db)
.await?
};
) -> Result<Script> {
let script_o =
sqlx::query_as::<_, Script>("SELECT * FROM script WHERE hash = $1 AND workspace_id = $2")
.bind(hash)
.bind(workspace_id)
.fetch_optional(&mut **db)
.await?;
let script = not_found_if_none(script_o, "Script", hash.to_string())?;
Ok(script)
@@ -1155,23 +1111,9 @@ async fn get_script_by_hash_internal<'c>(
async fn get_script_by_hash(
Extension(db): Extension<DB>,
Path((w_id, hash)): Path<(String, ScriptHash)>,
Query(query): Query<WithStarredInfoQuery>,
Extension(authed): Extension<ApiAuthed>,
) -> JsonResult<ScriptWithStarred> {
) -> JsonResult<Script> {
let mut tx = db.begin().await?;
let r = get_script_by_hash_internal(
&mut tx,
&w_id,
&hash,
query.with_starred_info.and_then(|x| {
if x {
Some(authed.username.as_str())
} else {
None
}
}),
)
.await?;
let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
tx.commit().await?;
Ok(Json(r))
@@ -1185,10 +1127,10 @@ async fn raw_script_by_hash(
let hash = ScriptHash(to_i64(hash_str.strip_suffix(".ts").ok_or_else(|| {
Error::BadRequest("Raw script path must end with .ts".to_string())
})?)?);
let r = get_script_by_hash_internal(&mut tx, &w_id, &hash, None).await?;
let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
tx.commit().await?;
Ok(r.script.content)
Ok(r.content)
}
#[derive(FromRow, Serialize)]

View File

@@ -1,141 +0,0 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::utils::content_plain;
use axum::{body::Body, extract::Query, response::Response, routing::get, Extension, Json, Router};
use serde::Serialize;
use windmill_common::{
error::{Error, JsonResult},
utils::Pagination,
};
use crate::{
db::{ApiAuthed, DB},
utils::require_super_admin,
};
pub fn global_service() -> Router {
Router::new()
.route("/list_files", get(list_files))
.route("/get_log_file/*path", get(get_log_file))
}
use axum::extract::Path;
#[derive(Debug, serde::Deserialize)]
pub struct LogFileQuery {
before: Option<chrono::DateTime<chrono::Utc>>,
after: Option<chrono::DateTime<chrono::Utc>>,
with_error: Option<bool>,
}
#[derive(Debug, sqlx::FromRow, Serialize)]
pub struct LogFile {
pub hostname: String,
pub mode: String,
pub worker_group: Option<String>,
pub log_ts: chrono::NaiveDateTime,
pub file_path: String,
pub ok_lines: Option<i64>,
pub err_lines: Option<i64>,
pub json_fmt: bool,
}
async fn list_files(
ApiAuthed { email, .. }: ApiAuthed,
Extension(db): Extension<DB>,
Query(pagination): Query<Pagination>,
Query(lq): Query<LogFileQuery>,
) -> JsonResult<Vec<LogFile>> {
require_super_admin(&db, &email).await?;
let (per_page, offset) = windmill_common::utils::paginate(pagination);
let mut sqlb = sql_builder::SqlBuilder::select_from("log_file")
.fields(&[
"hostname",
"mode::text",
"worker_group",
"log_ts",
"file_path",
"ok_lines",
"err_lines",
"json_fmt",
])
.order_by("log_ts", true)
.offset(offset)
.limit(per_page)
.clone();
if let Some(dt) = &lq.before {
sqlb.and_where_le(
"log_ts",
format!("to_timestamp({} / 1000.0)", dt.timestamp_millis()),
);
}
if let Some(dt) = &lq.after {
sqlb.and_where_ge(
"log_ts",
format!("to_timestamp({} / 1000.0)", dt.timestamp_millis()),
);
}
if let Some(true) = lq.with_error {
sqlb.and_where("err_lines > 0");
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let rows = sqlx::query_as::<_, LogFile>(&sql).fetch_all(&db).await?;
Ok(Json(rows))
}
async fn get_log_file(
ApiAuthed { email, .. }: ApiAuthed,
Extension(db): Extension<DB>,
Path(path): Path<windmill_common::utils::StripPath>,
) -> windmill_common::error::Result<Response> {
use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE;
require_super_admin(&db, &email).await?;
let path = path.to_path();
#[cfg(feature = "parquet")]
let s3_client = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
.read()
.await
.clone();
#[cfg(feature = "parquet")]
if let Some(s3_client) = s3_client {
let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);
let file = s3_client.get(&object_store::path::Path::from(path)).await;
match file {
Ok(file) => {
let bytes = file.bytes().await;
match bytes {
Ok(bytes) => {
return Ok(content_plain(Body::from(bytes::Bytes::from(bytes))));
}
Err(e) => {
return Err(Error::InternalErr(format!(
"Error pulling the bytes: {}",
e
)));
}
}
}
Err(e) => {
return Err(Error::InternalErr(format!(
"Error fetching the file: {}",
e
)));
}
}
}
let file = tokio::fs::read(format!("{}{}", TMP_WINDMILL_LOGS_SERVICE, path)).await;
if let Ok(bytes) = file {
Ok(content_plain(Body::from(bytes::Bytes::from(bytes))))
} else {
Err(Error::NotFound(format!("File {path} not found")))
}
}

View File

@@ -8,7 +8,6 @@
#![allow(non_snake_case)]
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
use std::sync::Arc;
use crate::db::ApiAuthed;
@@ -146,7 +145,6 @@ pub struct ExpiringAuthCache {
pub authed: ApiAuthed,
pub expiry: chrono::DateTime<chrono::Utc>,
}
pub struct AuthCache {
cache: Cache<(String, String), ExpiringAuthCache>,
db: DB,
@@ -504,32 +502,6 @@ pub struct Tokened {
pub token: String,
}
struct BruteForceCounter {
counter: AtomicU64,
last_reset: AtomicI64,
}
lazy_static! {
static ref BRUTE_FORCE_COUNTER: BruteForceCounter =
BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) };
}
impl BruteForceCounter {
async fn increment(&self) {
let now = time::OffsetDateTime::now_utc().unix_timestamp();
if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 {
tracing::error!(
"Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds"
);
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
}
if now - self.last_reset.load(Ordering::Relaxed) > 60 {
self.counter.store(0, Ordering::Relaxed);
self.last_reset.store(now, Ordering::Relaxed);
}
}
}
#[async_trait]
impl<S> FromRequestParts<S> for Tokened
where
@@ -554,7 +526,6 @@ where
parts.extensions.insert(tokened.clone());
Ok(tokened)
} else {
BRUTE_FORCE_COUNTER.increment().await;
Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
}
}
@@ -654,7 +625,6 @@ where
&& (path_vec.len() < 3
|| (path_vec[4] != "jobs" && path_vec[4] != "jobs_u"))
{
BRUTE_FORCE_COUNTER.increment().await;
return Err((
StatusCode::UNAUTHORIZED,
format!("Unauthorized scoped token: {:?}", authed.scopes),
@@ -670,7 +640,6 @@ where
}
}
}
BRUTE_FORCE_COUNTER.increment().await;
Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned()))
}
}
@@ -787,7 +756,6 @@ pub struct DeclineInvite {
#[derive(Deserialize)]
pub struct EditUser {
pub is_super_admin: Option<bool>,
pub name: Option<String>,
}
#[derive(Deserialize)]
@@ -1735,16 +1703,6 @@ async fn update_user(
.await?;
}
if let Some(n) = eu.name {
sqlx::query_scalar!(
"UPDATE password SET name = $1 WHERE email = $2",
n,
&email_to_update
)
.execute(&mut *tx)
.await?;
}
audit_log(
&mut *tx,
&authed,

View File

@@ -6,9 +6,7 @@
* LICENSE-AGPL for a copy of the license.
*/
use axum::{body::Body, response::Response};
use regex::Regex;
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
use windmill_common::{
auth::is_super_admin_email,
@@ -16,11 +14,6 @@ use windmill_common::{
DB,
};
#[derive(Deserialize)]
pub struct WithStarredInfoQuery {
pub with_starred_info: Option<bool>,
}
pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> {
let is_admin = is_super_admin_email(db, email).await?;
@@ -173,11 +166,3 @@ pub async fn get_and_delete_pending_username_or_generate<'c>(
Ok(username)
}
}
pub fn content_plain(body: Body) -> Response {
use axum::http::header;
Response::builder()
.header(header::CONTENT_TYPE, "text/plain")
.body(body)
.unwrap()
}

View File

@@ -46,8 +46,8 @@ struct WorkerPing {
started_at: chrono::DateTime<chrono::Utc>,
ip: String,
jobs_executed: i32,
last_job_id: Option<Uuid>,
last_job_workspace_id: Option<String>,
current_job_id: Option<Uuid>,
current_job_workspace_id: Option<String>,
custom_tags: Option<Vec<String>>,
worker_group: String,
wm_version: String,
@@ -88,7 +88,7 @@ async fn list_worker_pings(
let rows = sqlx::query_as!(
WorkerPing,
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as current_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as current_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage
FROM worker_ping
WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
ORDER BY ping_at desc LIMIT $2 OFFSET $3",

View File

@@ -2576,7 +2576,6 @@ async fn tarball_workspace(
}
}
ScriptLang::Php => "php",
ScriptLang::Rust => "rs",
};
archive
.write_to_archive(&script.content, &format!("{}.{}", script.path, ext))

View File

@@ -37,8 +37,6 @@ reqwest = { workspace = true }
tracing-subscriber = { workspace = true }
lazy_static.workspace = true
tracing-flame = { version = "^0", optional = true }
tracing-appender.workspace = true
gethostname.workspace = true
itertools.workspace = true
regex.workspace = true
git-version.workspace = true
@@ -54,7 +52,6 @@ bytes.workspace = true
mail-send.workspace = true
futures-core.workspace = true
async-stream.workspace = true
const_format.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }

View File

@@ -1,4 +1,3 @@
use crate::db::DB;
use crate::ee::LicensePlan::Community;
#[cfg(feature = "enterprise")]
use crate::error;
@@ -27,10 +26,7 @@ pub async fn get_license_plan() -> LicensePlan {
#[serde(untagged)]
pub enum CriticalErrorChannel {}
#[cfg(feature = "enterprise")]
pub async fn send_critical_error(_error_message: String) {}
#[cfg(feature = "enterprise")]
pub async fn send_recovered_critical_error(_error_message: String) {}
pub async fn trigger_critical_error_channels(_error_message: String) {}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
@@ -55,6 +51,3 @@ pub async fn create_customer_portal_session(
// Implementation is not open source
Ok("".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn worker_groups_alerts(_db: &DB) {}

Some files were not shown because too many files have changed in this diff Show More