inline script drawer

.devcontainer/Dockerfile

@@ -29,6 +29,7 @@ FROM mcr.microsoft.com/vscode/devcontainers/rust:bullseye
 
 RUN apt update \
     && apt-get install -y \
+    lld \
     python3 \
     libprotobuf-dev \ 
     libnl-route-3-dev \

.env

@@ -1,9 +1,2 @@
 DB_PASSWORD=changeme
-
-# this is the url that your instance is publicly exposed to
-WM_BASE_URL=http://localhost
-
-# To use another port than :80, setup the Caddyfile and the caddy section of the docker-compose to your needs: https://caddyserver.com/docs/getting-started
-
-# License key for enterprise edition
-# WM_LICENSE_KEY="<id>.<expiry>.<signature>"
+WM_BASE_URL=localhost

.github/DockerfileBackendTests

@@ -26,7 +26,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl
+    curl lld
 
 ENV SQLX_OFFLINE=true
 

.github/change-versions.sh

@@ -4,7 +4,7 @@ VERSION=$1
 echo "Updating versions to: $VERSION"
 
 sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
-sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
+sed -i -e "/^const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
 sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
 sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
 sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json

.github/uffizzi/caddy/Caddyfile

@@ -0,0 +1,5 @@
+localhost {
+        bind 0.0.0.0
+        reverse_proxy /ws/* http://0.0.0.0:3001
+        reverse_proxy /* http://0.0.0.0:8000
+}

.github/uffizzi/docker-compose.uffizzi.yml

@@ -0,0 +1,53 @@
+version: '3.7'
+
+x-uffizzi:
+  ingress:
+    service: windmill
+    port: 8000
+
+services:
+  db:
+    image: postgres:14
+    environment:
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_DB: windmill
+
+  windmill:
+    image: '${WINDMILL_IMAGE}'
+    ports:
+      - 8000:8000
+    entrypoint: ['/bin/sh', '-c']
+    command: 'echo ${OAUTH_JSON_BASE64} | base64 --decode > /usr/src/app/oauth.json && ./windmill'
+    environment:
+      - DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
+      - BASE_URL=${EXPECTED_URL}
+      - BASE_INTERNAL_URL=http://localhost:8000
+      - RUST_LOG=info
+      - NUM_WORKERS=3
+      - KEEP_JOB_DIR=false
+      - DENO_PATH=/usr/bin/deno
+      - PYTHON_PATH=/usr/local/bin/python3
+      - METRICS_ADDR=false
+      - OAUTH_JSON_BASE64=${OAUTH_JSON_BASE64}
+    volumes:
+      - worker_dependency_cache:/tmp/windmill/cache
+    deploy:
+      resources:
+        limits:
+          memory: 250M
+
+  lsp:
+    image: '${LSP_IMAGE}'
+    ports:
+      - 3001:3001
+
+  # caddy:
+  #   image: caddy:2.5.2-alpine
+  #   restart: unless-stopped
+  #   volumes:
+  #     - ./.github/uffizzi/caddy:/etc/caddy
+  #   environment:
+  #     - BASE_URL=localhost
+
+volumes:
+  worker_dependency_cache:

.github/workflows/automerge-dependabot.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.5.0
+        uses: dependabot/fetch-metadata@v1.3.6
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Enable auto-merge for Dependabot PRs

.github/workflows/backend-test.yml

@@ -40,4 +40,4 @@ jobs:
             backend -> target
       - name: cargo test
         timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture

.github/workflows/build_ws.yml

@@ -1,79 +0,0 @@
-env:
-  REGISTRY: ghcr.io
-  ECR_REGISTRY: 976079455550.dkr.ecr.us-east-1.amazonaws.com
-  IMAGE_NAME: ${{ github.repository }}-multiplayer
-
-name: Publish websocket multiplayer server
-on:
-  workflow_dispatch:
-
-
-permissions:
-  contents: read
-  id-token: write
-  packages: write
-  
-jobs:
-  publish_multiplayer:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: depot/setup-action@v1
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Login to registry
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push publicly
-        uses: depot/build-push-action@v1
-        with:
-          context: .
-          file: ./docker/DockerfileMultiplayer
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.meta.outputs.tags }}
-          labels: |
-            ${{ steps.meta.outputs.labels }}
-            org.opencontainers.image.licenses=AGPLv3
-
-  publish_privately:
-    needs: [publish_multiplayer]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to ECR
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.ECR_REGISTRY }}
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-
-      - name: Push image to ECR
-        run: |
-          docker buildx imagetools create \
-            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.github/workflows/deploy_to_windmill.yml

@@ -0,0 +1,20 @@
+name: Deploy to windmill.dev
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "community/**"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Deploy to windmill.dev
+        uses: windmill-labs/windmill-gh-action-deploy@v2.0.0
+        with:
+          dry_run: false
+          input_dir: community
+          windmill_workspace: starter
+          windmill_token: ${{ secrets.WINDMILL_API_TOKEN }}

.github/workflows/docker-image.yml

@@ -18,10 +18,10 @@ permissions:
   contents: read
   id-token: write
   packages: write
-
+  
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
         with:
@@ -50,6 +50,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
+
       - name: Build and push publicly
         uses: depot/build-push-action@v1
         with:
@@ -62,9 +63,9 @@ jobs:
           labels: |
             ${{ steps.meta-public.outputs.labels }}
             org.opencontainers.image.licenses=AGPLv3
-
+   
   build_ee:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
         with:
@@ -108,61 +109,40 @@ jobs:
             ${{ steps.meta-ee-public.outputs.labels }}
             org.opencontainers.image.licenses=Windmill-Enterprise-License
 
-  # disabled until we make it 100% reliable and add more meaningful tests
-  # playwright:
-  #   runs-on: [self-hosted, new]
-  #   needs: [build]
-  #   services:
-  #     postgres:
-  #       image: postgres
-  #       env:
-  #         POSTGRES_DB: windmill
-  #         POSTGRES_USER: admin
-  #         POSTGRES_PASSWORD: changeme
-  #       ports:
-  #         - 5432:5432
-  #       options: >-
-  #         --health-cmd pg_isready
-  #         --health-interval 10s
-  #         --health-timeout 5s
-  #         --health-retries 5
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: "Docker"
-  #       run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
-  #       id: docker-container
-  #     - uses: actions/setup-node@v3
-  #       with:
-  #         node-version: 16
-  #     - name: "Playwright run"
-  #       timeout-minutes: 2
-  #       run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
-  #     - name: "Clean up"
-  #       run: docker kill ${{ steps.docker-container.outputs.id }}
-  #       if: always()
 
-  deploy_s3:
-    needs: [build_ee]
-    runs-on: ubuntu-latest
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  playwright:
+    runs-on: [self-hosted, new]
+    needs: [build]
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_USER: admin
+          POSTGRES_PASSWORD: changeme
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v3
+      - name: "Docker"
+        run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
+        id: docker-container
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
-      - uses: shrink/actions-docker-extract@v2
-        id: extract
-        with:
-          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
-          path: "/static_frontend/."
+          node-version: 16
+      - name: "Playwright run"
+        timeout-minutes: 2
+        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=http://localhost:8000 && npm run test
+      - name: "Clean up"
+        run: docker kill ${{ steps.docker-container.outputs.id }}
+        if: always()
+
 
-      - uses: reggionick/s3-deploy@v3
-        with:
-          folder: ${{ steps.extract.outputs.destination }}
-          bucket: windmill-frontend
-          bucket-region: us-east-1
   publish_privately_heavy:
     needs: [build_ee]
     runs-on: [self-hosted, new]
@@ -211,12 +191,8 @@ jobs:
           tags: |
             ${{ steps.meta-heavy.outputs.tags }}
           labels: ${{ steps.meta-heavy.outputs.labels }}
-          cache-from:
-            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
-            }}-heavy:buildcache
-          cache-to:
-            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
-            }}-heavy:buildcache,mode=max
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-heavy:buildcache,mode=max
 
   publish_privately_helm:
     runs-on: [self-hosted, new]
@@ -228,6 +204,7 @@ jobs:
           fetch-depth: 0
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+ 
 
       - name: Login to registry
         uses: docker/login-action@v2
@@ -243,7 +220,7 @@ jobs:
           registry: ${{ env.ECR_REGISTRY }}
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
+          
       - name: Build and push privately
         uses: docker/build-push-action@v4
         if: github.event_name != 'pull_request'
@@ -253,9 +230,5 @@ jobs:
           file: ./docker/DockerfileHelm
           tags: |
             ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:helm
-          cache-from:
-            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
-            }}-helm:buildcache
-          cache-to:
-            type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME
-            }}-helm:buildcache,mode=max
+          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache
+          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-helm:buildcache,mode=max

.github/workflows/frontend-check.yml

@@ -4,8 +4,6 @@ on:
     types: [opened,synchronize,reopened,closed]
     paths:
       - "frontend/**"
-  merge_group:
-
 jobs:
   npm_check:
     runs-on: ubuntu-latest

.github/workflows/go_on_release.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v3
       - name: generate_go
         run: |
           go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.11.0

.github/workflows/pypi_on_release.yml

@@ -10,15 +10,9 @@ on:
       - "v*"
   workflow_dispatch:
 
-permissions:
-  contents: read
-  id-token: write
-  packages: write
-
 jobs:
   publish_pypi:
     runs-on: ubuntu-latest
-    if: startsWith(github.ref, 'refs/tags/v')
     container:
       image: ghcr.io/windmill-labs/python-client-builder
     steps:
@@ -30,25 +24,26 @@ jobs:
           cd python-client
           ./publish.sh
 
-
   publish_lsp:
-    runs-on: ubuntu-latest
+    needs: [publish_pypi]
+    runs-on: [self-hosted, new]
     steps:
-      - name: Sleep for 300 seconds waiting for pypi to update index
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: sleep 300
+      - name: Sleep for 30 seconds waiting for pypi to update index
+        run: sleep 30s
         shell: bash
-        
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: depot/setup-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -62,30 +57,6 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push publicly
-        uses: depot/build-push-action@v1
-        with:
-          context: "{{defaultContext}}:lsp"
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.meta.outputs.tags }}
-          labels: |
-            ${{ steps.meta.outputs.labels }}
-            org.opencontainers.image.licenses=AGPLv3
-
-
-  publish_lsp_private:
-    needs: [publish_lsp]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
       - name: Login to ECR
         uses: docker/login-action@v2
         with:
@@ -93,9 +64,17 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-
-      - name: Push image to ECR
-        run: |
-          docker buildx imagetools create \
-            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+      - name: Build and push publicly
+        uses: docker/build-push-action@v4
+        with:
+          context: "{{defaultContext}}:lsp"
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.metalocal.outputs.tags }}
+            ${{ steps.meta.outputs.tags }}
+            registry.uffizzi.com/windmill-lsp:60d
+          labels: ${{ steps.metalocal.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

.github/workflows/uffizzi-build.yml

@@ -0,0 +1,93 @@
+name: Build PR Image
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+    paths:
+      - "backend/**"
+      - ".github/uffizzi/**"
+      - ".github/workflows/**"
+  workflow_dispatch:
+      
+jobs:
+  build-windmill:
+    name: Build and Push `windmill`
+    runs-on: ubuntu-latest
+    if: ${{ (github.event_name != 'pull_request' || github.event.action != 'closed')}}
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
+          tags: type=raw,value=60d
+      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          context: ./
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+
+  render-compose-file:
+    name: Render Docker Compose File
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs:
+      - build-windmill
+    outputs:
+      compose-file-cache-key: ${{ steps.hash.outputs.hash }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          WINDMILL_IMAGE=${{ needs.build-windmill.outputs.tags }}
+          export WINDMILL_IMAGE
+          LSP_IMAGE=registry.uffizzi.com/windmill-lsp:60d
+          export LSP_IMAGE
+          envsubst '${WINDMILL_IMAGE} ${LSP_IMAGE}' < ./.github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }}
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Call for Preview Deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run: echo '${{ toJSON(github.event) }}' > event.json
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview.yml

@@ -0,0 +1,115 @@
+name: Deploy Uffizzi Preview
+
+on:
+  workflow_run:
+    workflows:
+      - "Build PR Image"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Compose File
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+    steps:
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            if (matchArtifact === undefined) {
+              throw TypeError('Build Artifact not found!');
+            }
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      - name: 'Unzip artifact'
+        run: unzip preview-spec.zip
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+      - name: Predict Deployment URL
+        id: url
+        # Replace dots in the repo name with the plus sign
+        run: |
+          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
+          echo "EXPECTED_URL=https://app.uffizzi.com/github.com/$REPO/pull/$PR_NUMBER" >> $GITHUB_ENV
+
+      - name: Re-Render Compose File
+        run: |
+          OAUTH_JSON_BASE64=${{ secrets.OAUTH_JSON_BASE64 }}
+          export OAUTH_JSON_BASE64
+          envsubst '${OAUTH_JSON_BASE64} ${EXPECTED_URL}' < docker-compose.rendered.yml > docker-compose.uffizzi.yml
+          # cat docker-compose.uffizzi.yml
+
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.uffizzi.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.uffizzi.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+
+  deploy-uffizzi-preview:
+    name: Use Remote Workflow to Preview on Uffizzi
+    needs:
+      - cache-compose-file
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.uffizzi.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write
+
+  playwright:
+    runs-on: ubuntu-latest
+    needs:
+      - deploy-uffizzi-preview
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: "Playwright run"
+        timeout-minutes: 2
+        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=${{ needs.deploy-uffizzi-preview.outputs.url }} && npm run test

Caddyfile

@@ -1,6 +1,15 @@
-{$BASE_URL} {
+{
+        auto_https off
+}
+
+http://{$BASE_URL} {
+        bind {$ADDRESS}
         reverse_proxy /ws/* http://lsp:3001
         reverse_proxy /* http://windmill_server:8000
-        # reverse_proxy /ws_mp/* http://multiplayer:3002
 }
-}
+
+https://{$BASE_URL}  {  
+  bind {$ADDRESS}
+  reverse_proxy /ws/* http://localhost:3001
+}
+}

Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl nodejs npm
+    curl lld nodejs npm
 
 RUN rustup component add rustfmt
 
@@ -39,7 +39,7 @@ WORKDIR /windmill
 ENV SQLX_OFFLINE=true
 ENV CARGO_INCREMENTAL=1
 
-FROM node:20-alpine as frontend
+FROM node:19-alpine as frontend
 
 # install dependencies
 WORKDIR /frontend
@@ -52,13 +52,13 @@ RUN mkdir /backend
 COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
 COPY /openflow.openapi.yaml /openflow.openapi.yaml
 COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
-
 RUN cd /backend/windmill-api && . ./build_openapi.sh
-COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/
 
 RUN npm run generate-backend-client
 ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
+RUN npm run check
+
 
 
 FROM rust_base AS planner
@@ -73,7 +73,7 @@ ARG features=""
 
 COPY --from=planner /windmill/recipe.json recipe.json
 
-RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json
+RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef cook --release --features "$features" --recipe-path recipe.json
 
 COPY ./openflow.openapi.yaml /openflow.openapi.yaml
 COPY ./backend ./
@@ -85,8 +85,7 @@ COPY .git/ .git/
 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features"
 
 
-FROM python:3.11.3-slim-buster
-ARG TARGETPLATFORM
+FROM python:3.11.2-slim-buster
 
 ARG APP=/usr/src/app
 
@@ -124,26 +123,16 @@ ENV TZ=Etc/UTC
 
 RUN /usr/local/bin/python3 -m pip install pip-tools
 
-COPY --from=frontend /frontend/build /static_frontend
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
 COPY --from=nsjail /nsjail/nsjail /bin/nsjail
 
-COPY --from=denoland/deno:1.33.3 /usr/bin/deno /usr/bin/deno
-
-# docker does not support conditional COPY and we want to use the same Dockerfile for both amd64 and arm64 and privilege the official image
-COPY --from=lukechannings/deno:v1.33.3 /usr/bin/deno /usr/bin/deno-arm
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then rm /usr/bin/deno-arm; elif [ "$TARGETPLATFORM" = "linux/arm64" ]; then mv /usr/bin/deno-arm /usr/bin/deno; fi
-
-# add the docker client to call docker from a worker if enabled
-COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 
 RUN mkdir -p ${APP}
 
-RUN ln -s ${APP}/windmill /usr/local/bin/windmill
-
 WORKDIR ${APP}
 
 EXPOSE 8000
 
-CMD ["windmill"]
+CMD ["./windmill"]

LICENSE

@@ -1,16 +1,12 @@
 
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
 
 Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
 Every file is under License AGPL unless otherwise specified 
 or belonging to one of the below cases:
 
-The files under backend/ are AGPLv3 Licensed.
-The files under frontend/ are AGPLv3 Licensed.
-The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
-
-The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.
-
-All third party components incorporated into the Windmill Software are licensed under the
-original license provided by the owner of the applicable component.
+The files under backend/ are AGPL Licensed.
+The files under frontend/ are AGPL Licensed.
+The files under python-client/ are Apache 2.0 Licensed.
+The files under community/ are Apache 2.0 Licensed.

README.md

@@ -1,32 +1,20 @@
 <p align="center">
-  <a href="https://www.windmill.dev/"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
+  <a href="https://app.windmill.dev"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
 </p>
 <p align="center">
     <em>.</em>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
-
-<p align=center>
-Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>
 
 <p align="center">
-<a href="https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL" target="_blank">
-    <img src="https://img.shields.io/badge/License-AGPLv3-blue.svg" alt="Package version">
-</a>
 <a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
     <img src="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml/badge.svg" alt="Docker Image CI">
 </a>
 <a href="https://pypi.org/project/wmill" target="_blank">
     <img src="https://img.shields.io/pypi/v/wmill?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
-</p>
-
-<p align="center">
-<a href="https://img.shields.io/github/commit-activity/m/windmill-labs/windmill" target="_blank">
-    <img src="https://img.shields.io/github/commit-activity/m/windmill-labs/windmill" alt="Commit activity">
-</a>
 <a href="https://discord.gg/V7PM2YHsPB" target="_blank">
   <img src="https://discordapp.com/api/guilds/930051556043276338/widget.png" alt="Discord Shield"/>
 </a>
@@ -38,12 +26,11 @@ Scripts are turned into UIs and no-code modules, no-code modules can be composed
 
 # Windmill - Turn scripts into workflows and UIs that you can share and run at scale
 
-Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
-dedicated instance and commercial support and licenses.
+Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers dedicated instance and commercial support and licenses.
 
-![Windmill Diagram](./imgs/stacks.svg)
+![Windmill Diagram](/imgs/stacks.svg)
 
-https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-822f-0c7ee7104252
+https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2bd-3bacc1f0ec79.mp4
 
 - [Windmill - Turn scripts into workflows and UIs that you can share and run at scale](#windmill---turn-scripts-into-workflows-and-uis-that-you-can-share-and-run-at-scale)
   - [Main Concepts](#main-concepts)
@@ -73,8 +60,8 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
 ## Main Concepts
 
 1. Define a minimal and generic script in Python, Typescript, Go or Bash that
-   solves a specific task. Here sending a POST request. The code can be defined
-   in the provided Web IDE or synchronized with your own github repo:
+   solves a specific task. Here sending an email with SMTP. The code can be
+   defined in the provided Web IDE or synchronized with your own github repo:
    ![Step 1](./imgs/windmill-editor.png)
 
 2. Your scripts parameters are automatically parsed and generate a frontend.
@@ -82,12 +69,12 @@ https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-82
 
 3. Make it flow! You can chain your scripts or scripts made by the community
    shared on [WindmillHub](https://hub.windmill.dev).
-   ![Step 3](./imgs/windmill-flow.png)
+   ![Step 4](./imgs/windmill-flow.png)
 
 4. Build complex UI on top of your scripts and flows.
-   ![Step 4](./imgs/windmill-builder.png)
+   ![Step 5](./imgs/windmill-builder.png)
 
-Scripts and flows can also be triggered by a cron schedule '_/5 _ \* \* \*' or
+Scripts and flows can also be triggered by a cron schedule '*/5 * * * *' or
 through webhooks.
 
 You can build your entire infra on top of Windmill!
@@ -95,49 +82,46 @@ You can build your entire infra on top of Windmill!
 ## Show me some actual script code
 
 ```typescript
-import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts";
+import * as wmill from "https://deno.land/x/windmill@v1.62.0/mod.ts"
 //import any dependency  from npm
 
-import cowsay from "npm:cowsay@1.5.0";
+import cowsay from 'npm:cowsay@1.5.0'
 
 export async function main(
-  a: number,
-  // unions generate enums
-  b: "my" | "enum",
-  // default parameters prefill the field
-  d = "default arg",
-  // nested objects work c = { nested: "object" },
-  // permissioned and typed json
-  db: wmill.Resource<"postgresql">
-) {
-  const email = Deno.env.get("WM_EMAIL");
-  // variables are permissioned and by path
-  let variable = await wmill.getVariable("f/company-folder/my_secret");
-  const lastTimeRun = await wmill.getState();
-  // logs are printed and always inspectable
-  console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }));
-  await wmill.setState(Date.now());
+    a: number,
+    // unions generate enums
+    b: "my" | "enum",
+    // default parameters prefill the field 
+    d = "default arg",
+    // nested objects work c = { nested: "object" }, 
+    // permissioned and typed json
+    db: wmill.Resource<"postgresql">) {
 
-  // return is serialized as JSON
-  return { foo: d, variable };
+    const email = Deno.env.get('WM_EMAIL')
+    // variables are permissioned and by path
+    let variable = await wmill.getVariable('f/company-folder/my_secret')
+    const lastTimeRun = await wmill.getState()
+    // logs are printed and always inspectable
+    console.log(cowsay.say({ text: "hello " + email + " " + lastTimeRun }))
+    await wmill.setState(Date.now())
+
+    // return is serialized as JSON
+    return { foo: d, variable };
 }
 ```
 
 ## CLI
 
 We have a powerful CLI to interact with the windmill platform and sync your
-scripts from local files, github repos and to run scripts and flows on the
-instance from local commands. See
-[more details](https://github.com/windmill-labs/windmill/tree/main/cli).
+scripts from local files, github repos and to run scripts and flows on the instance from local commands. See
+[more details](https://github.com/windmill-labs/windmill/tree/main/cli)
 
 ![CLI Screencast](./cli/vhs/output/setup.gif)
 
+
 ### Running scripts locally
 
-You can run your script locally easily, you simply need to pass the right
-environment variables for the `wmill` client library to fetch resources and
-variables from your instance if necessary. See more:
-<https://docs.windmill.dev/docs/advanced/local_development/>.
+You can run your script locally easily, you simply need to pass the right environment variables for the `wmill` client library to fetch resource and variables from your instance if necessary. See more: <https://docs.windmill.dev/docs/advanced/local_development/>
 
 ## Stack
 
@@ -164,7 +148,7 @@ variables from your instance if necessary. See more:
 
 Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
 sandboxing. It is production multi-tenant grade secure. Do not take our word for
-it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/).
+it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/)
 
 ### Secrets, credentials and sensitive values
 
@@ -194,34 +178,30 @@ back to the database is ~50ms. A typical lightweight deno job will take around
 
 We only provide docker-compose setup here. For more advanced setups, like
 compiling from source or using without a postgres super user, see
-[documentation](https://docs.windmill.dev/docs/advanced/self_host).
+[documentation](https://docs.windmill.dev/docs/advanced/self_host)
 
 ### Docker compose
 
-```
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
-curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
-
-docker compose up -d
-```
+`docker compose up` with the following docker-compose is sufficient:
+<https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml>
 
 Go to http://localhost et voilà :)
 
-The default super-admin user is: admin@windmill.dev / changeme.
 
-From there, you can follow the setup app and create other users.
+The default super-admin user is: admin@windmill.dev / changeme
+
+From there, you can create other users (do not forget to change the password!)
 
 ### Kubernetes (k8s) and Helm charts
 
 We publish helm charts at:
-<https://github.com/windmill-labs/windmill-helm-charts>.
+<https://github.com/windmill-labs/windmill-helm-charts>
 
 ### Postgres without superuser
 
-If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
-postgres superuser, you can run `./init-db-as-superuser.sql` to init the
-required users for Windmill.
+If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a postgres superuser,
+you can run `./init-db-as-superuser.sql` to init the required users for windmill.
+
 
 ### Commercial license
 
@@ -236,9 +216,9 @@ be AGPLv3 or you must get a commercial license. Contact us at
 <ruben@windmill.dev> if you have any doubts.
 
 In addition, a commercial license grants you a dedicated engineer to transition
-your current infrastructure to Windmill, support with tight SLA, and our global
-cache sync for high-performance/no dependency cache miss of cluster from 10+
-nodes to 200+ nodes.
+your current infrastructure to Windmill, support with tight SLA, audit logs
+export features, SSO, unlimited users creation, advanced permission managing
+features such as groups and the ability to create more than one workspace.
 
 ### OAuth for self-hosting
 
@@ -260,10 +240,6 @@ and mount it at `/usr/src/app/oauth.json`.
 The redirect url for the oauth clients is:
 `<instance_url>/user/login_callback/<client>`
 
-Even if you setup oauth, you will still want to **login as admin@windmill.dev /
-changeme** to setup your instance as a super-admin and give yourself admin
-rights.
-
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
@@ -299,72 +275,52 @@ You may also add your own custom OAuth2 IdP and OAuth2 Resource provider:
 ### Resource types
 
 You will also want to import all the approved resource types from
-[WindmillHub](https://hub.windmill.dev). A setup script will prompt you to have
-it being synced automatically everyday.
+[WindmillHub](https://hub.windmill.dev). A setup script will prompt
+you to have it being synced automatically everyday.
 
 ## Environment Variables
 
-| Environment Variable name           | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
-| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
-| DATABASE_URL                        |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL                      | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
-| SERVER_BIND_ADDR                    | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
-| PORT                                | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS                         | 3                                          | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
-| DISABLE_SERVER                      | false                                      | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
-| METRICS_ADDR                        | None                                       | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
-| JSON_FMT                            | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
-| BASE_URL                            | http://localhost:8000                      | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
-| TIMEOUT                             | 300                                        | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
-| ZOMBIE_JOB_TIMEOUT                  | 30                                         | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
-| RESTART_ZOMBIE_JOBS                 | true                                       | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
-| SLEEP_QUEUE                         | 50                                         | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
-| MAX_LOG_SIZE                        | 500000                                     | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
-| DISABLE_NUSER                       | false                                      | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
-| KEEP_JOB_DIR                        | false                                      | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
-| LICENSE_KEY (EE only)               | None                                       | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
-| S3_CACHE_BUCKET (EE only)           | None                                       | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
-| TAR_CACHE_RATE (EE only)            | 100                                        | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
-| SLACK_SIGNING_SECRET                | None                                       | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
-| COOKIE_DOMAIN                       | None                                       | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
-| DENO_PATH                           | /usr/bin/deno                              | The path to the deno binary.                                                                                                                                                                       | Worker                |
-| PYTHON_PATH                         | /usr/local/bin/python3                     | The path to the python binary.                                                                                                                                                                     | Worker                |
-| GO_PATH                             | /usr/bin/go                                | The path to the go binary.                                                                                                                                                                         | Worker                |
-| GOPRIVATE                           |                                            | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
-| GOPROXY                             |                                            | The GOPROXY env variable to use                                                                                                                                                                    | Worker                |
-| NETRC                               |                                            | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
-| PIP_INDEX_URL                       | None                                       | The index url to pass for pip.                                                                                                                                                                     | Worker                |
-| PIP_EXTRA_INDEX_URL                 | None                                       | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
-| PIP_TRUSTED_HOST                    | None                                       | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
-| PATH                                | None                                       | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
-| HOME                                | None                                       | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
-| DATABASE_CONNECTIONS                | 50 (Server)/3 (Worker)                     | The max number of connections in the database connection pool                                                                                                                                      | All                   |
-| SUPERADMIN_SECRET                   | None                                       | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
-| TIMEOUT_WAIT_RESULT                 | 20                                         | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
-| QUEUE_LIMIT_WAIT_RESULT             | None                                       | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
-| DENO_AUTH_TOKENS                    | None                                       | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
-| DENO_FLAGS                          | None                                       | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
-| NPM_CONFIG_REGISTRY                 | None                                       | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
-| PIP_LOCAL_DEPENDENCIES              | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
-| ADDITIONAL_PYTHON_PATHS             | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
-| INCLUDE_HEADERS                     | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
-| WHITELIST_WORKSPACES                | None                                       | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| BLACKLIST_WORKSPACES                | None                                       | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| INSTANCE_EVENTS_WEBHOOK             | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
-| GLOBAL_CACHE_INTERVAL               | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
-| WORKER_TAGS                         | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
-| CUSTOM_TAGS                         | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
-| JOB_RETENTION_SECS                  | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
-| WAIT_RESULT_FAST_POLL_INTERVAL_MS   | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
-| WAIT_RESULT_SLOW_POLL_INTERVAL_MS   | 200                                        | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
-| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2                                          | The duration of fast poll mode before switching to slow poll                                                                                                                                       | Server                |
-| EXIT_AFTER_NO_JOB_FOR_SECS          | None                                       | Exit worker if no job is received after duration in secs if defined                                                                                                                                | Worker                |
-| OAUTH_JSON_AS_BASE64                | None                                       | Base64 encoded JSON of the OAuth configuration. e.g `OAUTH_JSON_AS_BASE64=$(base64 oauth.json \| tr -d '\n')` to encode it                                                                         | Server                |
-| REQUEST_SIZE_LIMIT                  | 2097152 (2MB)                              | Max request size which impact the maximum size of resources and payload size of job args                                                                                                           | Server                |
-| ACCEPT_INVALID_CERTS                | false                                      | Accept invalid certificates, including self-signed and expired certificates                                                                                                                        | Server                |
-| HTTP_PROXY                          | None                                       | http_proxy                                                                                                                                                                                         | Server + Worker       |
-| HTTPS_PROXY                         | None                                       | https_proxy                                                                                                                                                                                        | Server + Worker       |
-| NO_PROXY                            | None                                       | no_proxy                                                                                                                                                                                           | Server + Worker       |
+| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |  |
+| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
+| DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
+| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                   | 300                    | The timeout in seconds for the execution of a script                                                                                                                                               | Worker                |
+| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
+| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |  |
+| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
+| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+| TIMEOUT_WAIT_RESULT       | 20                     | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
+| QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
+| DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
+| DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
+| ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
+| INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| NEW_USER_WEBHOOK          | None                   | Webhook to notify of a new user added, signup/invite. Can hook back to windmill to send emails                                                                                                     | Server                |
 
 ## Run a local dev setup
 
@@ -373,11 +329,11 @@ it being synced automatically everyday.
 This will use the backend of <https://app.windmill.dev> but your own frontend
 with hot-code reloading.
 
-1. Go to `frontend/`:
-   1. `npm install`
-   2. `npm run generate-backend-client`
-   3. `npm run dev`
-2. Et voilà, windmill should be available at `http://localhost:3000/`
+1. Install [caddy](https://caddyserver.com)
+2. Go to `frontend/`:
+   1. `npm install`, `npm run generate-backend-client` then `npm run dev`
+   2. In another shell `sudo caddy run --config CaddyfileRemote`
+3. Et voilà, windmill should be available at `http://localhost/`
 
 ### Backend + Frontend
 
@@ -393,14 +349,15 @@ running options.
 3. Install deno and python3, have the bins at `/usr/bin/deno` and
    `/usr/local/bin/python3`
 4. Install [caddy](https://caddyserver.com)
-5. Go to `frontend/`:
+5. Install the [lld linker](https://lld.llvm.org/)
+6. Go to `frontend/`:
    1. `npm install`, `npm run generate-backend-client` then `npm run dev`
    2. In another shell `npm run build` otherwise the backend will not find the
       `frontend/build` folder and will crash
    3. In another shell `sudo caddy run --config Caddyfile`
-6. Go to `backend/`:
+7. Go to `backend/`:
    `DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
-7. Et voilà, windmill should be available at `http://localhost/`
+8. Et voilà, windmill should be available at `http://localhost/`
 
 ## Contributors
 

backend/.cargo/config.toml

@@ -1,18 +1,10 @@
 [build]
 rustflags = [
     "--cfg",
-    "tokio_unstable"
+    "tokio_unstable",
+    "-C",
+    "link-arg=-fuse-ld=lld",
+    "-Clink-arg=-Wl,--no-rosegment",
 ]
 incremental = true
 
-[target.x86_64-apple-darwin]
-rustflags = [
-    "-C", "link-arg=-undefined",
-    "-C", "link-arg=dynamic_lookup",
-]
-
-[target.aarch64-apple-darwin]
-rustflags = [
-    "-C", "link-arg=-undefined",
-    "-C", "link-arg=dynamic_lookup",
-]

backend/.vscode/settings.json

@@ -1,5 +1,3 @@
 {
-  "python.analysis.typeCheckingMode": "basic",
-  "rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"],
-  "rust-analyzer.showUnlinkedFileNotification": false
+  "python.analysis.typeCheckingMode": "basic"
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.117.0"
+version = "1.70.1"
 authors.workspace = true
 edition.workspace = true
 
@@ -14,15 +14,12 @@ members = [
     "./windmill-api-client",
     "./parsers/windmill-parser",
     "./parsers/windmill-parser-ts",
-    "./parsers/windmill-parser-wasm",
     "./parsers/windmill-parser-go",
-    "./parsers/windmill-parser-bash",
     "./parsers/windmill-parser-py",
-    "./parsers/windmill-parser-py-imports",
 ]
 
 [workspace.package]
-version = "1.117.0"
+version = "1.70.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -30,12 +27,12 @@ edition = "2021"
 name = "windmill"
 path = "./src/main.rs"
 
-[profile.dev]
-opt-level = 1
-incremental = true
-
 [features]
-enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise"]
+enterprise = [
+    "windmill-worker/enterprise",
+    "windmill-queue/enterprise",
+    "windmill-api/enterprise",
+]
 
 [dependencies]
 anyhow.workspace = true
@@ -55,20 +52,12 @@ git-version.workspace = true
 rsa.workspace = true
 base64.workspace = true
 sha2.workspace = true
-rsmq_async.workspace = true
-url.workspace = true
-lazy_static.workspace = true
-once_cell.workspace = true
-prometheus.workspace = true
-uuid.workspace = true
-gethostname.workspace = true
 
 [dev-dependencies]
 serde_json.workspace = true
 reqwest.workspace = true
 windmill-queue.workspace = true
 axum.workspace = true
-serde.workspace = true
 
 [workspace.dependencies]
 windmill-api = { path = "./windmill-api" }
@@ -80,7 +69,6 @@ windmill-audit = { path = "./windmill-audit" }
 windmill-parser = { path = "./parsers/windmill-parser" }
 windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
 windmill-parser-py = { path = "./parsers/windmill-parser-py" }
-windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" }
 windmill-parser-go = { path = "./parsers/windmill-parser-go" }
 windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
 axum = { version = "^0", features = ["headers"] }
@@ -88,7 +76,7 @@ headers = "^0"
 hyper = { version = "^0", features = ["full"] }
 tokio = { version = "^1", features = ["full", "tracing"] }
 tower = "^0"
-tower-http = { version = "^0", features = ["trace", "cors"] }
+tower-http = { version = "^0", features = ["trace"] }
 tower-cookies = "^0"
 serde = "^1"
 serde_json = { version = "^1", features = ["preserve_order"] }
@@ -96,7 +84,6 @@ uuid = { version = "^1", features = ["serde", "v4"] }
 thiserror = "^1"
 anyhow = "^1"
 chrono = { version = "^0", features = ["serde"] }
-chrono-tz = "^0"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
 prometheus = { version = "^0", default-features = false }
@@ -112,7 +99,7 @@ rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
-rustpython-parser = "0.2.0"
+rustpython-parser = { git = "https://github.com/RustPython/RustPython" }
 cron = "^0"
 lettre = { version = "^0", features = [
     "rustls-tls",
@@ -133,12 +120,13 @@ tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
-deno_core = "0.188.0"
+deno_core = "^0"
 async-recursion = "^1"
-swc_common = "0.29.39"
-swc_ecma_parser = "0.128.2"
-swc_ecma_ast = "0.98.1"
+swc_common = "^0"
+swc_ecma_parser = "^0"
+swc_ecma_ast = "^0"
 base64 = "0.21.0"
+unicode-general-category = "^0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
 sqlx = { version = "^0", features = [
@@ -154,7 +142,6 @@ sqlx = { version = "^0", features = [
 dotenv = "^0"
 ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"
-futures-core = "^0"
 tokio-metrics = "0.1.0"
 lazy_static = "1.4.0"
 serde_derive = "1.0.147"
@@ -166,12 +153,3 @@ async-stripe = { version = "0.14", features = [
     "checkout",
 ] }
 async_zip = { version = "0.0.11", features = ["full"] }
-once_cell = "1.17.1"
-rsmq_async = { version = "5.1.5" }
-gosyn = "0.2.2"
-bytes = "1.4.0"
-gethostname = "0.4.3"
-wasm-bindgen = "0.2"
-serde-wasm-bindgen = "0.4"
-wasm-bindgen-test = "0.3.0"
-convert_case = "0.6.0"

backend/README.md

@@ -1,7 +1,6 @@
 # Windmill Backend
 
-This folder holds all backend components, the [src/](./src/) folder only
-contains files used to build the "root" binary.
+This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
 
 ## Components
 
@@ -14,9 +13,3 @@ contains files used to build the "root" binary.
 | [windmill-queue](./windmill-queue/)           | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
 | [windmill-worker](./windmill-worker/)         | The worker. Used to process and execute flows & jobs.                                                     |
 | [parsers](./parsers/)                         | Contains code to parse signatures in different langauges.                                                 |
-
-### Compile sqlx for offline ci
-
-```
-cargo sqlx prepare --merged -- --bin windmill --features enterprise
-```

backend/migrations/20220123221902_migrate_root.up.sql

@@ -1,72 +0,0 @@
-DO
-$do$
-BEGIN
-    IF NOT EXISTS (
-        SELECT
-        FROM   pg_catalog.pg_roles
-        WHERE  rolname = 'windmill_user') THEN
-
-        LOCK TABLE pg_catalog.pg_roles;
-
-        CREATE ROLE windmill_user;
-
-    END IF;
-EXCEPTION WHEN OTHERS THEN
-    RAISE NOTICE 'error creating the windmill_user role: %', SQLERRM;
-END
-$do$;
-
-DO
-$do$
-BEGIN
-    LOCK TABLE pg_catalog.pg_roles;
-
-    GRANT ALL
-    ON ALL TABLES IN SCHEMA public 
-    TO windmill_user;
-
-    GRANT ALL PRIVILEGES 
-    ON ALL SEQUENCES IN SCHEMA public 
-    TO windmill_user;
-
-    ALTER DEFAULT PRIVILEGES 
-        IN SCHEMA public
-        GRANT ALL ON TABLES TO windmill_user;
-
-    ALTER DEFAULT PRIVILEGES 
-        IN SCHEMA public
-        GRANT ALL ON SEQUENCES TO windmill_user;
-
-EXCEPTION WHEN OTHERS THEN
-    RAISE NOTICE 'error granting proper permission to windmill_user: %', SQLERRM;
-END
-$do$;
-
-DO
-$do$
-BEGIN
-    IF NOT EXISTS (
-        SELECT
-        FROM   pg_catalog.pg_roles
-        WHERE  rolname = 'windmill_admin') THEN
-
-        LOCK TABLE pg_catalog.pg_roles;
-
-        CREATE ROLE windmill_admin WITH BYPASSRLS;
-
-
-    END IF;
-EXCEPTION WHEN OTHERS THEN
-    RAISE NOTICE 'error creating the windmill_admin role: %', SQLERRM;
-END
-$do$;
-
-
-DO
-$do$
-BEGIN
-    GRANT windmill_user TO windmill_admin;
-EXCEPTION WHEN OTHERS THEN
-    RAISE NOTICE 'error granting windmill_user to windmill_admin: %', SQLERRM;
-END
-$do$;

backend/migrations/20230303064952_add_root_leaf_jobs.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230303064952_add_root_leaf_jobs.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-ALTER TABLE queue ADD COLUMN root_job uuid;
-ALTER TABLE queue ADD COLUMN leaf_jobs jsonb;

backend/migrations/20230327115727_update_app_setup.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230327201258_add_granular_share_apps.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230327201258_add_granular_share_apps.up.sql

@@ -1,11 +0,0 @@
--- Add up migration script here
-CREATE POLICY see_extra_perms_user ON app FOR ALL
-USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
-WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
-
-CREATE POLICY see_extra_perms_groups ON app FOR ALL
-USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-WITH CHECK (exists(
-    SELECT key, value FROM jsonb_each_text(extra_perms) 
-    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-    AND value::boolean));

backend/migrations/20230405071524_schedule_timezone.down.sql

@@ -1,2 +0,0 @@
-ALTER TABLE schedule DROP COLUMN timezone;
-ALTER TABLE schedule ADD COLUMN offset_ INTEGER NOT NULL DEFAULT 0;

backend/migrations/20230405071524_schedule_timezone.up.sql

@@ -1,26 +0,0 @@
-ALTER TABLE schedule ADD COLUMN timezone VARCHAR(255) NOT NULL DEFAULT 'UTC';
-
--- INSERT the correct IANA timezone string for each offset value
-
-UPDATE schedule SET timezone = 'Pacific/Honolulu' WHERE offset_ = 600;
-UPDATE schedule SET timezone = 'America/Anchorage' WHERE offset_ = 540;
-UPDATE schedule SET timezone = 'America/Los_Angeles' WHERE offset_ = 480;
-UPDATE schedule SET timezone = 'America/Chicago' WHERE offset_ = 360;
-UPDATE schedule SET timezone = 'America/New_York' WHERE offset_ = 300;
-UPDATE schedule SET timezone = 'America/Halifax' WHERE offset_ = 240;
-UPDATE schedule SET timezone = 'America/Sao_Paulo' WHERE offset_ = 180;
-UPDATE schedule SET timezone = 'Atlantic/South_Georgia' WHERE offset_ = 120;
-UPDATE schedule SET timezone = 'Atlantic/Cape_Verde' WHERE offset_ = 60;
-UPDATE schedule SET timezone = 'Europe/London' WHERE offset_ = 0;
-UPDATE schedule SET timezone = 'Europe/Berlin' WHERE offset_ = -60;
-UPDATE schedule SET timezone = 'Europe/Athens' WHERE offset_ = -120;
-UPDATE schedule SET timezone = 'Europe/Moscow' WHERE offset_ = -180;
-UPDATE schedule SET timezone = 'Asia/Dubai' WHERE offset_ = -240;
-UPDATE schedule SET timezone = 'Asia/Aqtau' WHERE offset_ = -300;
-UPDATE schedule SET timezone = 'Asia/Almaty' WHERE offset_ = -360;
-UPDATE schedule SET timezone = 'Asia/Bangkok' WHERE offset_ = -420;
-UPDATE schedule SET timezone = 'Asia/Hong_Kong' WHERE offset_ = -480;
-UPDATE schedule SET timezone = 'Asia/Tokyo' WHERE offset_ = -540;
-UPDATE schedule SET timezone = 'Australia/Sydney' WHERE offset_ = -600;
-
-ALTER TABLE schedule DROP COLUMN offset_;

backend/migrations/20230405195753_relax_varchar_restrictions.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230405195753_relax_varchar_restrictions.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-ALTER TABLE password ALTER COLUMN company TYPE VARCHAR(255);
-ALTER TABLE password ALTER COLUMN name TYPE VARCHAR(255);

backend/migrations/20230408203759_cache_pip_resolution.down.sql

@@ -1,2 +0,0 @@
--- Add down migration script here
-DROP TABLE pip_resolution_cache;

backend/migrations/20230408203759_cache_pip_resolution.up.sql

@@ -1,6 +0,0 @@
--- Add up migration script here
-CREATE TABLE pip_resolution_cache(
-    hash VARCHAR(255) PRIMARY KEY,
-    expiration TIMESTAMP NOT NULL,
-    lockfile TEXT NOT NULL
-);

backend/migrations/20230410122342_saved_inputs.down.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS input;
-DROP TYPE RUNNABLE_TYPE;

backend/migrations/20230410122342_saved_inputs.up.sql

@@ -1,13 +0,0 @@
-CREATE TYPE RUNNABLE_TYPE AS ENUM ('ScriptHash', 'ScriptPath', 'FlowPath');
-
-CREATE TABLE IF NOT EXISTS input (
-  id UUID PRIMARY KEY,
-  workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-  runnable_id VARCHAR(255) NOT NULL,
-  runnable_type RUNNABLE_TYPE NOT NULL,
-  name TEXT NOT NULL,
-  args JSONB NOT NULL,
-  created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-  created_by VARCHAR(50) NOT NULL,
-  is_public BOOLEAN NOT NULL DEFAULT FALSE
-);

backend/migrations/20230414163230_input_grant.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230414163230_input_grant.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-GRANT ALL ON input TO windmill_user;
-GRANT ALL ON input TO windmill_admin;

backend/migrations/20230421111850_add_tagged_scripts.down.sql

@@ -1,4 +0,0 @@
--- Add down migration script here
-ALTER TABLE script DROP COLUMN tag;
-ALTER TABLE completed_job DROP COLUMN tag;
-ALTER TABLE queue DROP COLUMN tag;

backend/migrations/20230421111850_add_tagged_scripts.up.sql

@@ -1,4 +0,0 @@
--- Add up migration script here
-ALTER TABLE script ADD COLUMN tag VARCHAR(50);
-ALTER TABLE completed_job  ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
-ALTER TABLE queue ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';

backend/migrations/20230429082953_add_drafts.down.sql

@@ -1,3 +0,0 @@
--- Add down migration script here
-DROP TABLE draft;
-DROP TYPE DRAFT_TYPE;

backend/migrations/20230429082953_add_drafts.up.sql

@@ -1,18 +0,0 @@
--- Add up migration script here
-CREATE TYPE DRAFT_TYPE AS ENUM ('script', 'flow', 'app');
-
-CREATE TABLE draft (
-    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-    path VARCHAR(255) NOT NULL,
-    typ DRAFT_TYPE NOT NULL,
-    value JSONB NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT now(),
-    PRIMARY KEY (workspace_id, path, typ)
-);
-
-GRANT ALL ON draft TO windmill_admin;
-GRANT ALL ON draft TO windmill_user;
-
-ALTER TABLE script ADD COLUMN draft_only BOOLEAN;
-ALTER TABLE flow ADD COLUMN draft_only BOOLEAN;
-ALTER TABLE app ADD COLUMN draft_only BOOLEAN;

backend/migrations/20230507170410_raw_app.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230507170410_raw_app.up.sql

@@ -1,33 +0,0 @@
--- Add up migration script here
-CREATE TABLE raw_app (
-    path varchar(255) PRIMARY KEY,
-    version INTEGER NOT NULL DEFAULT 0,
-    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-    summary VARCHAR(1000) NOT NULL DEFAULT '',
-    edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
-    data TEXT NOT NULL,
-    extra_perms JSONB NOT NULL DEFAULT '{}'
-);
-
-CREATE POLICY see_own ON raw_app FOR ALL
-USING (SPLIT_PART(raw_app.path, '/', 1) = 'u' AND SPLIT_PART(raw_app.path, '/', 2) = current_setting('session.user'));
-
-CREATE POLICY see_member ON raw_app FOR ALL
-USING (SPLIT_PART(raw_app.path, '/', 1) = 'g' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
-
-CREATE POLICY see_extra_perms_user ON raw_app FOR ALL
-USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
-WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
-
-CREATE POLICY see_extra_perms_groups ON raw_app FOR ALL
-USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-WITH CHECK (exists(
-    SELECT key, value FROM jsonb_each_text(extra_perms) 
-    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
-    AND value::boolean));
-
-CREATE POLICY see_folder_extra_perms_user ON raw_app FOR ALL
-USING (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
-WITH CHECK (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
-
-ALTER TYPE FAVORITE_KIND ADD VALUE 'raw_app';

backend/migrations/20230509130016_raw_app_grant.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230509130016_raw_app_grant.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-GRANT ALL ON raw_app TO windmill_admin;
-GRANT ALL ON raw_app TO windmill_user;

backend/migrations/20230510101952_jsonb_to_json.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230510101952_jsonb_to_json.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-ALTER TABLE app_version
-ALTER COLUMN value TYPE json;

backend/migrations/20230510104144_jsonb_to_json_draft.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230510104144_jsonb_to_json_draft.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-ALTER TABLE draft
-ALTER COLUMN value TYPE json;

backend/migrations/20230512191209_update_app_n.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230515131104_rls_raw_app.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230515131104_rls_raw_app.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TABLE raw_app ENABLE ROW LEVEL SECURITY;

backend/migrations/20230516140626_capture_check_increase.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230516140626_capture_check_increase.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-alter table capture DROP constraint capture_payload_check;
-alter table capture add constraint caputre_payload_too_big check  (length(payload::text) < 512 * 1024);

backend/migrations/20230516141630_capture_check_increase_2.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230516141630_capture_check_increase_2.up.sql

@@ -1,3 +0,0 @@
--- Add up migration script here
-alter table capture DROP constraint caputre_payload_too_big;
-alter table capture add constraint capture_payload_too_big check (length(payload::text) < 512 * 1024);

backend/migrations/20230522170237_schedule_on_failure.down.sql

@@ -1,2 +0,0 @@
--- Add down migration script here
-ALTER TABLE schedule DROP COLUMN on_failure;

backend/migrations/20230522170237_schedule_on_failure.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TABLE schedule ADD COLUMN on_failure VARCHAR(1000) DEFAULT NULL;

backend/migrations/20230524065919_fix_migration_break.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230524065919_fix_migration_break.up.sql

@@ -1,11 +0,0 @@
--- Add up migration script here
-do
- $$
-BEGIN
-
-IF (NOT EXISTS (SELECT workspace_id FROM script  WHERE workspace_id = 'demo' UNION ALL SELECT workspace_id FROM flow  WHERE workspace_id = 'demo' UNION ALL SELECT workspace_id FROM app WHERE workspace_id = 'demo'))
-THEN
-    DELETE FROM workspace_invite WHERE workspace_id = 'demo';
-END IF;
-END
-$$

backend/migrations/20230524065920_remove_demo_starter.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230524065920_remove_demo_starter.up.sql

@@ -1,43 +0,0 @@
--- Add up migration script here
-do
- $$
-BEGIN
-
-IF (NOT EXISTS (SELECT workspace_id FROM script  WHERE workspace_id = 'demo' UNION ALL SELECT workspace_id FROM flow  WHERE workspace_id = 'demo' UNION ALL SELECT workspace_id FROM app WHERE workspace_id = 'demo'))
-THEN
-    DELETE FROM usr WHERE workspace_id = 'demo';
-    DELETE FROM usr_to_group WHERE workspace_id = 'demo';
-    DELETE FROM queue WHERE workspace_id = 'demo';
-    DELETE FROM completed_job WHERE workspace_id = 'demo';
-    DELETE FROM raw_app WHERE workspace_id = 'demo';
-    DELETE FROM variable WHERE workspace_id = 'demo';
-    DELETE FROM schedule WHERE workspace_id = 'demo';
-    DELETE FROM resource WHERE workspace_id = 'demo';
-    DELETE FROM resource_type WHERE workspace_id = 'demo';
-    DELETE FROM workspace_key WHERE workspace_id = 'demo';
-    DELETE FROM group_ WHERE workspace_id = 'demo';
-    DELETE FROM workspace_settings WHERE workspace_id = 'demo';
-    DELETE FROM workspace WHERE id = 'demo';
-END IF;
-
-IF (NOT EXISTS (SELECT workspace_id FROM script  WHERE workspace_id = 'starter' UNION ALL SELECT workspace_id FROM flow  WHERE workspace_id = 'starter' UNION ALL SELECT workspace_id FROM app WHERE workspace_id = 'starter'))
-THEN
-    DELETE FROM usr WHERE workspace_id = 'starter';
-    DELETE FROM usr_to_group WHERE workspace_id = 'starter';
-    DELETE FROM queue WHERE workspace_id = 'starter';
-    DELETE FROM completed_job WHERE workspace_id = 'starter';
-    DELETE FROM raw_app WHERE workspace_id = 'starter';
-    DELETE FROM variable WHERE workspace_id = 'starter';
-    DELETE FROM schedule WHERE workspace_id = 'starter';
-    DELETE FROM resource WHERE workspace_id = 'starter';
-    DELETE FROM resource_type WHERE workspace_id = 'starter';
-    DELETE FROM workspace_key WHERE workspace_id = 'starter';
-    DELETE FROM group_ WHERE workspace_id = 'starter';
-    DELETE FROM workspace_settings WHERE workspace_id = 'starter';
-    DELETE FROM workspace WHERE id = 'starter';
-END IF;
-
-
-END
-$$
-

backend/migrations/20230524133926_update_cli.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230524133926_update_cli.up.sql

@@ -1,15 +0,0 @@
--- Add up migration script here
-UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.104.0/main.ts";
-export async function main() {
-  await run(
-    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
-
-  await run("hub", "pull");
-}
-
-async function run(...cmd: string[]) {
-  console.log("Running \"" + cmd.join('' '') + "\"");
-  await wmill.parse(cmd);
-}', summary = 'Synchronize Hub Resource types with instance',
-description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
-WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230524163022_update_cli_2.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230524163022_update_cli_2.up.sql

@@ -1,15 +0,0 @@
--- Add up migration script here
-UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.104.1/main.ts";
-export async function main() {
-  await run(
-    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
-
-  await run("hub", "pull");
-}
-
-async function run(...cmd: string[]) {
-  console.log("Running \"" + cmd.join('' '') + "\"");
-  await wmill.parse(cmd);
-}', summary = 'Synchronize Hub Resource types with instance',
-description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
-WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230527130558_update_hub_sync.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230527130558_update_hub_sync.up.sql

@@ -1,15 +0,0 @@
--- Add up migration script here
-UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.105.0/main.ts";
-export async function main() {
-  await run(
-    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
-
-  await run("hub", "pull");
-}
-
-async function run(...cmd: string[]) {
-  console.log("Running \"" + cmd.join('' '') + "\"");
-  await wmill.parse(cmd);
-}', summary = 'Synchronize Hub Resource types with instance',
-description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
-WHERE hash = -28028598712388162 AND workspace_id = 'admins';

backend/migrations/20230527141935_remove_starter_policies.down.sql

@@ -1 +0,0 @@
--- Add down migration script here

backend/migrations/20230527141935_remove_starter_policies.up.sql

@@ -1,18 +0,0 @@
--- Add up migration script here
-DO
-$do$
-  DECLARE
-    i text;
-    arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow', 'completed_job'];
-  BEGIN
-  FOREACH i IN ARRAY arr
-  LOOP
-    EXECUTE FORMAT(
-      $$
-      DROP POLICY see_starter ON %1$I;
-      $$,
-      i
-    );
-  END LOOP;
-  END
-$do$;

backend/migrations/20230529144123_add_token_scopes.down.sql

@@ -1,2 +0,0 @@
--- Add down migration script here
-ALTER TABLE token DROP COLUMN scopes;

backend/migrations/20230529144123_add_token_scopes.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TABLE token ADD COLUMN scopes text[];

backend/migrations/20230605043929_add_envs.down.sql

@@ -1,2 +0,0 @@
--- Add down migration script here
-ALTER TABLE script DROP COLUMN envs;

backend/migrations/20230605043929_add_envs.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TABLE script ADD COLUMN envs VARCHAR(1000)[];

backend/migrations/20230617055441_add_deployment_workspace.down.sql

@@ -1,2 +0,0 @@
--- Add down migration script here
-ALTER TABLE workspace_settings DROP COLUMN deploy_to;

backend/migrations/20230617055441_add_deployment_workspace.up.sql

@@ -1,2 +0,0 @@
--- Add up migration script here
-ALTER TABLE workspace_settings ADD COLUMN deploy_to VARCHAR(255);

backend/parsers/windmill-parser-bash/Cargo.toml

@@ -10,7 +10,10 @@ path = "./src/lib.rs"
 
 [dependencies]
 windmill-parser.workspace = true
+windmill-common.workspace = true
+phf.workspace = true
+unicode-general-category.workspace = true
+itertools.workspace = true
 anyhow.workspace = true
 regex.workspace = true
-lazy_static.workspace = true
-serde_json.workspace = true
+lazy_static.workspace = true

backend/parsers/windmill-parser-bash/src/lib.rs

@@ -1,48 +1,35 @@
 #![allow(non_snake_case)] // TODO: switch to parse_* function naming
 
-use anyhow::anyhow;
 use regex::Regex;
-use serde_json::json;
 
 use std::collections::HashMap;
 use windmill_parser::{Arg, MainArgSignature, Typ};
 
-pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
+pub fn parse_bash_sig(code: &str) -> windmill_common::error::Result<MainArgSignature> {
     let parsed = parse_file(&code)?;
     if let Some(x) = parsed {
         let args = x;
         Ok(MainArgSignature { star_args: false, star_kwargs: false, args })
     } else {
-        Err(anyhow!("Error parsing bash script".to_string()))
+        Err(windmill_common::error::Error::BadRequest(
+            "Error parsing bash script".to_string(),
+        ))
     }
 }
 
-lazy_static::lazy_static! {
-    static ref RE: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+):-(.*)\})"$"#).unwrap();
-}
-
 fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
-    let mut hm: HashMap<i32, (String, Option<String>)> = HashMap::new();
-    for cap in RE.captures_iter(code) {
-        hm.insert(
-            cap.get(2)
-                .or(cap.get(3))
-                .and_then(|x| x.as_str().parse::<i32>().ok())
-                .ok_or_else(|| anyhow!("Impossible to parse arg digit"))?,
-            (
-                cap[1].to_string(),
-                cap.get(4).map(|x| x.as_str().to_string()),
-            ),
-        );
+    let mut hm = HashMap::new();
+    let re = Regex::new(r#"(?m)^(\w+)="\$(\d+)"$"#).unwrap();
+    for cap in re.captures_iter(code) {
+        hm.insert(cap[2].parse::<i32>()?, cap[1].to_string());
     }
     let mut args = vec![];
     for i in 1..20 {
         if hm.contains_key(&i) {
-            let (name, default) = hm.get(&i).unwrap();
             args.push(Arg {
-                name: name.clone(),
+                name: hm[&i].clone(),
                 typ: Typ::Str(None),
-                default: default.clone().map(|x| json!(x)),
+                default: None,
                 otyp: None,
                 has_default: false,
             });
@@ -56,8 +43,6 @@ fn parse_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
 #[cfg(test)]
 mod tests {
 
-    use serde_json::json;
-
     use super::*;
 
     #[test]
@@ -65,7 +50,8 @@ mod tests {
         let code = r#"
 token="$1"
 image="$2"
-digest="${3:-latest with spaces}"
+digest="${3:-latest}"
+foo="$4"
 
 "#;
         //println!("{}", serde_json::to_string()?);
@@ -88,13 +74,6 @@ digest="${3:-latest with spaces}"
                         typ: Typ::Str(None),
                         default: None,
                         has_default: false
-                    },
-                    Arg {
-                        otyp: None,
-                        name: "digest".to_string(),
-                        typ: Typ::Str(None),
-                        default: Some(json!("latest with spaces")),
-                        has_default: false
                     }
                 ]
             }

backend/parsers/windmill-parser-go/Cargo.toml

@@ -10,6 +10,8 @@ path = "./src/lib.rs"
 
 [dependencies]
 windmill-parser.workspace = true
+windmill-common.workspace = true
+phf.workspace = true
+unicode-general-category.workspace = true
 itertools.workspace = true
 anyhow.workspace = true
-gosyn.workspace = true

backend/parsers/windmill-parser-go/src/parser_go_ast.rs

@@ -0,0 +1,347 @@
+#![allow(clippy::large_enum_variant)] // TODO: we allow large enum variant for now, let's profile properly to see if we want to box.
+
+use crate::parser_go_token::{Position, Token};
+use std::collections::BTreeMap;
+
+// https://pkg.go.dev/go/ast#CommentGroup
+#[derive(Debug)]
+pub struct CommentGroup {
+    // List []*Comment // len(List) > 0
+}
+
+// https://pkg.go.dev/go/ast#FieldList
+#[derive(Debug)]
+pub struct FieldList<'a> {
+    pub opening: Option<Position<'a>>, // position of opening parenthesis/brace, if any
+    pub list: Vec<Field<'a>>,          // field list; or nil
+    pub closing: Option<Position<'a>>, // position of closing parenthesis/brace, if any
+}
+
+// https://pkg.go.dev/go/ast#Field
+#[derive(Debug)]
+pub struct Field<'a> {
+    pub doc: Option<CommentGroup>,     // associated documentation; or nil
+    pub names: Option<Vec<Ident<'a>>>, // field/method/(type) parameter names, or type "type"; or nil
+    pub type_: Option<Expr<'a>>,       // field/method/parameter type, type list type; or nil
+    pub tag: Option<BasicLit<'a>>,     // field tag; or nil
+    pub comment: Option<CommentGroup>, // line comments; or nil
+}
+
+// https://pkg.go.dev/go/ast#File
+#[derive(Debug)]
+pub struct File<'a> {
+    // package name
+    pub decls: Vec<Decl<'a>>, // top-level declarations; or nil // list of all comments in the source file
+}
+
+// https://pkg.go.dev/go/ast#FuncDecl
+#[derive(Debug)]
+pub struct FuncDecl<'a> {
+    pub doc: Option<CommentGroup>,   // associated documentation; or nil
+    pub recv: Option<FieldList<'a>>, // receiver (methods); or nil (functions)
+    pub name: Ident<'a>,             // function/method name
+    pub type_: FuncType<'a>, // function signature: type and value parameters, results, and position of "func" keyword
+    pub body: Option<BlockStmt<'a>>, // function body; or nil for external (non-Go) function
+}
+
+// https://pkg.go.dev/go/ast#BlockStmt
+#[derive(Debug)]
+pub struct BlockStmt<'a> {
+    pub lbrace: Position<'a>, // position of "{"
+    pub list: Vec<Stmt>,
+    pub rbrace: Position<'a>, // position of "}", if any (may be absent due to syntax error)
+}
+
+// https://pkg.go.dev/go/ast#FuncType
+#[derive(Debug)]
+pub struct FuncType<'a> {
+    pub func: Option<Position<'a>>, // position of "func" keyword (token.NoPos if there is no "func")
+    pub params: FieldList<'a>,      // (incoming) parameters; non-nil
+    pub results: Option<FieldList<'a>>, // (outgoing) results; or nil
+}
+
+// https://pkg.go.dev/go/ast#Ident
+#[derive(Debug)]
+pub struct Ident<'a> {
+    pub name_pos: Position<'a>,       // identifier position
+    pub name: &'a str,                // identifier name
+    pub obj: Option<Box<Object<'a>>>, // denoted object; or nil
+}
+
+// https://pkg.go.dev/go/ast#ValueSpec
+#[derive(Debug)]
+pub struct ValueSpec<'a> {
+    pub doc: Option<CommentGroup>,     // associated documentation; or nil
+    pub names: Vec<Ident<'a>>,         // value names (len(Names) > 0)
+    pub type_: Option<Expr<'a>>,       // value type; or nil
+    pub values: Option<Vec<Expr<'a>>>, // initial values; or nil
+    pub comment: Option<CommentGroup>, // line comments; or nil
+}
+
+// https://pkg.go.dev/go/ast#BasicLit
+#[derive(Debug)]
+pub struct BasicLit<'a> {
+    pub value_pos: Position<'a>, // literal position
+    pub kind: Token,             // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
+    pub value: &'a str, // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
+}
+
+// https://pkg.go.dev/go/ast#Object
+#[derive(Debug)]
+pub struct Object<'a> {
+    pub kind: ObjKind,
+    pub name: &'a str,         // declared name
+    pub decl: Option<ObjDecl>, // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil
+    pub data: Option<usize>,   // object-specific data; or nil
+    pub type_: Option<()>,     // placeholder for type information; may be nil
+}
+
+// https://pkg.go.dev/go/ast#Ellipsis
+#[derive(Debug)]
+pub struct Ellipsis<'a> {
+    pub ellipsis: Position<'a>,     // position of "..."
+    pub elt: Option<Box<Expr<'a>>>, // ellipsis element type (parameter lists only); or nil
+}
+
+// https://pkg.go.dev/go/ast#Ellipsis
+#[derive(Debug)]
+pub struct TypeAssertExpr<'a> {
+    pub x: Box<Expr<'a>>,     // expression
+    pub lparen: Position<'a>, // position of "("
+    pub type_: Box<Expr<'a>>, // asserted type; nil means type switch X.(type)
+    pub rparen: Position<'a>, // position of ")"
+}
+
+// https://pkg.go.dev/go/ast#SliceExpr
+#[derive(Debug)]
+pub struct SliceExpr<'a> {
+    pub x: Box<Expr<'a>>,            // expression
+    pub lbrack: Position<'a>,        // position of "["
+    pub low: Option<Box<Expr<'a>>>,  // begin of slice range; or nil
+    pub high: Option<Box<Expr<'a>>>, // end of slice range; or nil
+    pub max: Option<Box<Expr<'a>>>,  // maximum capacity of slice; or nil
+    pub slice3: bool,                // true if 3-index slice (2 colons present)
+    pub rbrack: Position<'a>,        // position of "]"
+}
+
+// https://pkg.go.dev/go/ast#ObjKind
+#[derive(Debug)]
+pub enum ObjKind {}
+
+#[derive(Debug)]
+pub enum ObjDecl {}
+
+// https://pkg.go.dev/go/ast#Decl
+#[derive(Debug)]
+pub enum Decl<'a> {
+    FuncDecl(FuncDecl<'a>),
+}
+
+// https://pkg.go.dev/go/ast#Scope
+#[derive(Debug)]
+pub struct Scope<'a> {
+    pub outer: Option<Box<Scope<'a>>>,
+    pub objects: BTreeMap<&'a str, Object<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#GenDecl
+#[derive(Debug)]
+pub struct GenDecl<'a> {
+    pub doc: Option<CommentGroup>,    // associated documentation; or nil
+    pub tok_pos: Position<'a>,        // position of Tok
+    pub tok: Token,                   // IMPORT, CONST, TYPE, or VAR
+    pub lparen: Option<Position<'a>>, // position of '(', if any
+    pub specs: Vec<Spec>,
+    pub rparen: Option<Position<'a>>, // position of ')', if any
+}
+
+// https://pkg.go.dev/go/ast#AssignStmt
+#[derive(Debug)]
+pub struct AssignStmt<'a> {
+    pub lhs: Vec<Expr<'a>>,
+    pub tok_pos: Position<'a>, // position of Tok
+    pub tok: Token,            // assignment token, DEFINE
+    pub rhs: Vec<Expr<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#BinaryExpr
+#[derive(Debug)]
+pub struct BinaryExpr<'a> {
+    pub x: Box<Expr<'a>>,     // left operand
+    pub op_pos: Position<'a>, // position of Op
+    pub op: Token,            // operator
+    pub y: Box<Expr<'a>>,     // right operand
+}
+
+// https://pkg.go.dev/go/ast#ReturnStmt
+#[derive(Debug)]
+pub struct ReturnStmt<'a> {
+    pub return_: Position<'a>,  // position of "return" keyword
+    pub results: Vec<Expr<'a>>, // result expressions; or nil
+}
+
+// https://pkg.go.dev/go/ast#TypeSpec
+#[derive(Debug)]
+pub struct TypeSpec<'a> {
+    pub doc: Option<CommentGroup>,     // associated documentation; or nil
+    pub name: Option<Ident<'a>>,       // type name
+    pub assign: Option<Position<'a>>,  // position of '=', if any
+    pub type_: Expr<'a>, // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes
+    pub comment: Option<CommentGroup>, // line comments; or nil
+}
+
+// https://pkg.go.dev/go/ast#StructType
+#[derive(Debug)]
+pub struct StructType<'a> {
+    pub struct_: Position<'a>,         // position of "struct" keyword
+    pub fields: Option<FieldList<'a>>, // list of field declarations
+    pub incomplete: bool,              // true if (source) fields are missing in the Fields list
+}
+
+// https://pkg.go.dev/go/ast#StarExpr
+#[derive(Debug)]
+pub struct StarExpr<'a> {
+    pub star: Position<'a>, // position of "*"
+    pub x: Box<Expr<'a>>,   // operand
+}
+
+// https://pkg.go.dev/go/ast#InterfaceType
+#[derive(Debug)]
+pub struct InterfaceType<'a> {
+    pub interface: Position<'a>,        // position of "interface" keyword
+    pub methods: Option<FieldList<'a>>, // list of embedded interfaces, methods, or types
+    pub incomplete: bool, // true if (source) methods or types are missing in the Methods list
+}
+
+// https://pkg.go.dev/go/ast#UnaryExpr
+#[derive(Debug)]
+pub struct UnaryExpr<'a> {
+    pub op_pos: Position<'a>, // position of Op
+    pub op: Token,            // operator
+    pub x: Box<Expr<'a>>,     // operand
+}
+
+// https://pkg.go.dev/go/ast#CallExpr
+#[derive(Debug)]
+pub struct CallExpr<'a> {
+    pub fun: Box<Expr<'a>>,             // function expression
+    pub lparen: Position<'a>,           // position of "("
+    pub args: Option<Vec<Expr<'a>>>,    // function arguments; or nil
+    pub ellipsis: Option<Position<'a>>, // position of "..." (token.NoPos if there is no "...")
+    pub rparen: Position<'a>,           // position of ")"
+}
+
+// https://pkg.go.dev/go/ast#SelectorExpr
+#[derive(Debug)]
+pub struct SelectorExpr<'a> {
+    pub x: Box<Expr<'a>>, // expression
+    pub sel: Ident<'a>,   // field selector
+}
+
+// https://pkg.go.dev/go/ast#ParenExpr
+#[derive(Debug)]
+pub struct ParenExpr<'a> {
+    pub lparen: Position<'a>, // position of "("
+    pub x: Box<Expr<'a>>,     // parenthesized expression
+    pub rparen: Position<'a>, // position of ")"
+}
+
+// https://pkg.go.dev/go/ast#FuncLit
+#[derive(Debug)]
+pub struct FuncLit<'a> {
+    pub type_: FuncType<'a>, // function type
+    pub body: BlockStmt<'a>, // function body
+}
+
+// https://pkg.go.dev/go/ast#ChanType
+#[derive(Debug)]
+pub struct ChanType<'a> {
+    pub begin: Position<'a>, // position of "chan" keyword or "<-" (whichever comes first)
+    pub arrow: Option<Position<'a>>, // position of "<-" (token.NoPos if there is no "<-")
+    pub dir: u8,             // channel direction
+    pub value: Box<Expr<'a>>, // value type
+}
+
+// htt/opt/visual-studio-code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.htmlps://pkg.go.dev/go/ast#IndexExpr
+#[derive(Debug)]
+pub struct IndexExpr<'a> {
+    pub x: Box<Expr<'a>>,     // expression
+    pub lbrack: Position<'a>, // position of "["
+    pub index: Box<Expr<'a>>, // index expression
+    pub rbrack: Position<'a>, // position of "]"
+}
+
+// https://pkg.go.dev/go/ast#MapType
+#[derive(Debug)]
+pub struct MapType<'a> {
+    pub map: Position<'a>,
+    pub key: Box<Expr<'a>>,
+    pub value: Box<Expr<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#CompositeLit
+#[derive(Debug)]
+pub struct CompositeLit<'a> {
+    pub type_: Box<Expr<'a>>,        // literal type; or nil
+    pub lbrace: Position<'a>,        // position of "{"
+    pub elts: Option<Vec<Expr<'a>>>, // list of composite elements; or nil
+    pub rbrace: Position<'a>,        // position of "}"
+    pub incomplete: bool,            // true if (source) expressions are missing in the Elts list
+}
+
+// https://pkg.go.dev/go/ast#KeyValueExpr
+#[derive(Debug)]
+pub struct KeyValueExpr<'a> {
+    pub key: Box<Expr<'a>>,
+    pub colon: Position<'a>, // position of ":"
+    pub value: Box<Expr<'a>>,
+}
+
+// https://pkg.go.dev/go/ast#ArrayType
+#[derive(Debug)]
+pub struct ArrayType<'a> {
+    pub lbrack: Position<'a>,       // position of "["
+    pub len: Option<Box<Expr<'a>>>, // Ellipsis node for [...]T array types, nil for slice types
+    pub elt: Box<Expr<'a>>,         // element type
+}
+
+// https://pkg.go.dev/go/ast#ChanDir
+#[derive(Debug)]
+pub enum ChanDir {
+    SEND = 1 << 0,
+    RECV = 1 << 1,
+}
+
+// https://pkg.go.dev/go/ast#Spec
+#[derive(Debug)]
+pub enum Spec {}
+
+// https://pkg.go.dev/go/ast#Expr
+#[derive(Debug)]
+pub enum Expr<'a> {
+    ArrayType(ArrayType<'a>),
+    BasicLit(BasicLit<'a>),
+    BinaryExpr(BinaryExpr<'a>),
+    CallExpr(CallExpr<'a>),
+    ChanType(ChanType<'a>),
+    CompositeLit(CompositeLit<'a>),
+    Ellipsis(Ellipsis<'a>),
+    FuncLit(FuncLit<'a>),
+    FuncType(FuncType<'a>),
+    Ident(Ident<'a>),
+    IndexExpr(IndexExpr<'a>),
+    InterfaceType(InterfaceType<'a>),
+    KeyValueExpr(KeyValueExpr<'a>),
+    MapType(MapType<'a>),
+    ParenExpr(ParenExpr<'a>),
+    SelectorExpr(SelectorExpr<'a>),
+    SliceExpr(SliceExpr<'a>),
+    StarExpr(StarExpr<'a>),
+    StructType(StructType<'a>),
+    TypeAssertExpr(TypeAssertExpr<'a>),
+    UnaryExpr(UnaryExpr<'a>),
+}
+
+// https://pkg.go.dev/go/ast#Stmt
+#[derive(Debug)]
+pub enum Stmt {}

backend/parsers/windmill-parser-go/src/parser_go_scanner.rs

@@ -0,0 +1,948 @@
+// https://golang.org/ref/spec#Lexical_elements
+
+use crate::parser_go_token::{Position, Token};
+use phf::{phf_map, Map};
+use std::fmt;
+use unicode_general_category::{get_general_category, GeneralCategory};
+
+pub type Step<'a> = (Position<'a>, Token, &'a str);
+
+#[derive(Debug)]
+pub enum ScannerError {
+    HexadecimalNotFound,
+    OctalNotFound,
+    UnterminatedComment,
+    UnterminatedEscapedChar,
+    UnterminatedRune,
+    UnterminatedString,
+    InvalidDirective,
+}
+
+impl std::error::Error for ScannerError {}
+
+impl fmt::Display for ScannerError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "scanner error: {:?}", self)
+    }
+}
+
+pub type Result<T> = std::result::Result<T, ScannerError>;
+
+#[derive(Debug)]
+pub struct Scanner<'a> {
+    directory: &'a str,
+    file: &'a str,
+    buffer: &'a str,
+    //
+    chars: std::iter::Peekable<std::str::Chars<'a>>,
+    current_char: Option<char>,
+    current_char_len: usize,
+    //
+    offset: usize,
+    line: usize,
+    column: usize,
+    start_offset: usize,
+    start_line: usize,
+    start_column: usize,
+    //
+    hide_column: bool,
+    insert_semi: bool,
+    pending_line_info: Option<LineInfo<'a>>,
+}
+
+type LineInfo<'a> = (Option<&'a str>, usize, Option<usize>, bool);
+
+impl<'a> Scanner<'a> {
+    pub fn new(filename: &'a str, buffer: &'a str) -> Self {
+        let (directory, file) = filename.rsplit_once('/').unwrap_or(("", filename));
+        let mut s = Scanner {
+            directory,
+            file,
+            buffer,
+            //
+            chars: buffer.chars().peekable(),
+            current_char: None,
+            current_char_len: 0,
+            //
+            offset: 0,
+            line: 1,
+            column: 1,
+            start_offset: 0,
+            start_line: 1,
+            start_column: 1,
+            //
+            hide_column: false,
+            insert_semi: false,
+            pending_line_info: None,
+        };
+        s.next(); // read the first character
+        s
+    }
+
+    #[allow(clippy::cognitive_complexity)] // Allow complex scan function
+    pub fn scan(&mut self) -> Result<Step<'a>> {
+        let insert_semi = self.insert_semi;
+        self.insert_semi = false;
+
+        while let Some(c) = self.current_char {
+            self.reset_start();
+
+            match c {
+                ' ' | '\t' | '\r' => {
+                    self.next();
+                }
+
+                '\n' => {
+                    self.next();
+                    if insert_semi {
+                        return Ok((self.position(), Token::SEMICOLON, "\n"));
+                    }
+                }
+
+                _ => break,
+            }
+        }
+
+        if let Some(c) = self.current_char {
+            match c {
+                '+' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::ADD_ASSIGN, ""));
+                        }
+                        Some('+') => {
+                            self.insert_semi = true;
+                            self.next();
+                            return Ok((self.position(), Token::INC, ""));
+                        }
+                        _ => return Ok((self.position(), Token::ADD, "")),
+                    }
+                }
+
+                '-' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::SUB_ASSIGN, ""));
+                        }
+                        Some('-') => {
+                            self.insert_semi = true;
+                            self.next();
+                            return Ok((self.position(), Token::DEC, ""));
+                        }
+                        _ => return Ok((self.position(), Token::SUB, "")),
+                    }
+                }
+
+                '*' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::MUL_ASSIGN, ""));
+                        }
+                        _ => return Ok((self.position(), Token::MUL, "")),
+                    }
+                }
+
+                '/' => match self.peek() {
+                    Some('=') => {
+                        self.next();
+                        self.next();
+                        return Ok((self.position(), Token::QUO_ASSIGN, ""));
+                    }
+                    Some('/') => {
+                        if insert_semi {
+                            return Ok((self.position(), Token::SEMICOLON, "\n"));
+                        }
+                        return self.scan_line_comment();
+                    }
+                    Some('*') => {
+                        if insert_semi && self.find_line_end() {
+                            return Ok((self.position(), Token::SEMICOLON, "\n"));
+                        }
+                        return self.scan_general_comment();
+                    }
+                    _ => {
+                        self.next();
+                        return Ok((self.position(), Token::QUO, ""));
+                    }
+                },
+
+                '%' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::REM_ASSIGN, ""));
+                        }
+                        _ => return Ok((self.position(), Token::REM, "")),
+                    }
+                }
+
+                '&' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::AND_ASSIGN, ""));
+                        }
+                        Some('&') => {
+                            self.next();
+                            return Ok((self.position(), Token::LAND, ""));
+                        }
+                        Some('^') => {
+                            self.next();
+                            match self.current_char {
+                                Some('=') => {
+                                    self.next();
+                                    return Ok((self.position(), Token::AND_NOT_ASSIGN, ""));
+                                }
+                                _ => return Ok((self.position(), Token::AND_NOT, "")),
+                            }
+                        }
+                        _ => return Ok((self.position(), Token::AND, "")),
+                    }
+                }
+
+                '|' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::OR_ASSIGN, ""));
+                        }
+                        Some('|') => {
+                            self.next();
+                            return Ok((self.position(), Token::LOR, ""));
+                        }
+                        _ => return Ok((self.position(), Token::OR, "")),
+                    }
+                }
+
+                '^' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::XOR_ASSIGN, ""));
+                        }
+                        _ => return Ok((self.position(), Token::XOR, "")),
+                    }
+                }
+
+                '<' => {
+                    self.next();
+                    match self.current_char {
+                        Some('<') => {
+                            self.next();
+                            match self.current_char {
+                                Some('=') => {
+                                    self.next();
+                                    return Ok((self.position(), Token::SHL_ASSIGN, ""));
+                                }
+                                _ => return Ok((self.position(), Token::SHL, "")),
+                            }
+                        }
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::LEQ, ""));
+                        }
+                        Some('-') => {
+                            self.next();
+                            return Ok((self.position(), Token::ARROW, ""));
+                        }
+                        _ => return Ok((self.position(), Token::LSS, "")),
+                    }
+                }
+
+                '>' => {
+                    self.next();
+                    match self.current_char {
+                        Some('>') => {
+                            self.next();
+                            match self.current_char {
+                                Some('=') => {
+                                    self.next();
+                                    return Ok((self.position(), Token::SHR_ASSIGN, ""));
+                                }
+                                _ => {
+                                    return Ok((self.position(), Token::SHR, ""));
+                                }
+                            }
+                        }
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::GEQ, ""));
+                        }
+                        _ => return Ok((self.position(), Token::GTR, "")),
+                    }
+                }
+
+                ':' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::DEFINE, ""));
+                        }
+                        _ => return Ok((self.position(), Token::COLON, "")),
+                    }
+                }
+
+                '!' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::NEQ, ""));
+                        }
+                        _ => return Ok((self.position(), Token::NOT, "")),
+                    }
+                }
+
+                ',' => {
+                    self.next();
+                    return Ok((self.position(), Token::COMMA, ""));
+                }
+
+                '(' => {
+                    self.next();
+                    return Ok((self.position(), Token::LPAREN, ""));
+                }
+
+                ')' => {
+                    self.insert_semi = true;
+                    self.next();
+                    return Ok((self.position(), Token::RPAREN, ""));
+                }
+
+                '[' => {
+                    self.next();
+                    return Ok((self.position(), Token::LBRACK, ""));
+                }
+
+                ']' => {
+                    self.insert_semi = true;
+                    self.next();
+                    return Ok((self.position(), Token::RBRACK, ""));
+                }
+
+                '{' => {
+                    self.next();
+                    return Ok((self.position(), Token::LBRACE, ""));
+                }
+
+                '}' => {
+                    self.insert_semi = true;
+                    self.next();
+                    return Ok((self.position(), Token::RBRACE, ""));
+                }
+
+                ';' => {
+                    self.next();
+                    return Ok((self.position(), Token::SEMICOLON, ";"));
+                }
+
+                '.' => {
+                    self.next();
+                    match self.current_char {
+                        Some('0'..='9') => return self.scan_int_or_float_or_imag(true),
+                        Some('.') => match self.peek() {
+                            Some('.') => {
+                                self.next();
+                                self.next();
+                                return Ok((self.position(), Token::ELLIPSIS, ""));
+                            }
+                            _ => return Ok((self.position(), Token::PERIOD, "")),
+                        },
+                        _ => return Ok((self.position(), Token::PERIOD, "")),
+                    }
+                }
+
+                '=' => {
+                    self.next();
+                    match self.current_char {
+                        Some('=') => {
+                            self.next();
+                            return Ok((self.position(), Token::EQL, ""));
+                        }
+                        _ => return Ok((self.position(), Token::ASSIGN, "")),
+                    }
+                }
+
+                '0'..='9' => return self.scan_int_or_float_or_imag(false),
+                '\'' => return self.scan_rune(),
+                '"' => return self.scan_interpreted_string(),
+                '`' => return self.scan_raw_string(),
+                _ => return self.scan_pkg_or_keyword_or_ident(),
+            };
+        }
+
+        self.reset_start();
+        if insert_semi {
+            Ok((self.position(), Token::SEMICOLON, "\n"))
+        } else {
+            Ok((self.position(), Token::EOF, ""))
+        }
+    }
+
+    // https://golang.org/ref/spec#Keywords
+    // https://golang.org/ref/spec#Identifiers
+    fn scan_pkg_or_keyword_or_ident(&mut self) -> Result<Step<'a>> {
+        self.next();
+
+        while let Some(c) = self.current_char {
+            if !(is_letter(c) || is_unicode_digit(c)) {
+                break;
+            }
+            self.next()
+        }
+
+        let pos = self.position();
+        let literal = self.literal();
+
+        if literal.len() > 1 {
+            if let Some(&token) = KEYWORDS.get(literal) {
+                self.insert_semi = matches!(
+                    token,
+                    Token::BREAK | Token::CONTINUE | Token::FALLTHROUGH | Token::RETURN
+                );
+                return Ok((pos, token, literal));
+            }
+        }
+
+        self.insert_semi = true;
+        Ok((pos, Token::IDENT, literal))
+    }
+
+    // https://golang.org/ref/spec#Integer_literals
+    // https://golang.org/ref/spec#Floating-point_literals
+    // https://golang.org/ref/spec#Imaginary_literals
+    fn scan_int_or_float_or_imag(&mut self, preceding_dot: bool) -> Result<Step<'a>> {
+        self.insert_semi = true;
+
+        let mut token = Token::INT;
+        let mut digits = "_0123456789";
+        let mut exp = "eE";
+
+        if !preceding_dot {
+            if matches!(self.current_char, Some('0')) {
+                self.next();
+                match self.current_char {
+                    Some('b' | 'B') => {
+                        digits = "_01";
+                        exp = "";
+                        self.next();
+                    }
+                    Some('o' | 'O') => {
+                        digits = "_01234567";
+                        exp = "";
+                        self.next();
+                    }
+                    Some('x' | 'X') => {
+                        digits = "_0123456789abcdefABCDEF";
+                        exp = "pP";
+                        self.next();
+                    }
+                    _ => {}
+                };
+            }
+
+            while let Some(c) = self.current_char {
+                if !digits.contains(c) {
+                    break;
+                }
+                self.next();
+            }
+        }
+
+        if preceding_dot || matches!(self.current_char, Some('.')) {
+            token = Token::FLOAT;
+            self.next();
+            while let Some(c) = self.current_char {
+                if !digits.contains(c) {
+                    break;
+                }
+                self.next();
+            }
+        }
+
+        if !exp.is_empty() {
+            if let Some(c) = self.current_char {
+                if exp.contains(c) {
+                    token = Token::FLOAT;
+                    self.next();
+                    if matches!(self.current_char, Some('-' | '+')) {
+                        self.next();
+                    }
+                    while let Some(c) = self.current_char {
+                        if !matches!(c, '_' | '0'..='9') {
+                            break;
+                        }
+                        self.next();
+                    }
+                }
+            }
+        }
+
+        if matches!(self.current_char, Some('i')) {
+            token = Token::IMAG;
+            self.next();
+        }
+
+        Ok((self.position(), token, self.literal()))
+    }
+
+    // https://golang.org/ref/spec#Rune_literals
+    fn scan_rune(&mut self) -> Result<Step<'a>> {
+        self.insert_semi = true;
+        self.next();
+
+        match self.current_char {
+            Some('\\') => self.require_escaped_char::<'\''>()?,
+            Some(_) => self.next(),
+            _ => return Err(ScannerError::UnterminatedRune),
+        }
+
+        if matches!(self.current_char, Some('\'')) {
+            self.next();
+            return Ok((self.position(), Token::CHAR, self.literal()));
+        }
+
+        Err(ScannerError::UnterminatedRune)
+    }
+
+    // https://golang.org/ref/spec#String_literals
+    fn scan_interpreted_string(&mut self) -> Result<Step<'a>> {
+        self.insert_semi = true;
+        self.next();
+
+        while let Some(c) = self.current_char {
+            match c {
+                '"' => {
+                    self.next();
+                    return Ok((self.position(), Token::STRING, self.literal()));
+                }
+                '\\' => self.require_escaped_char::<'"'>()?,
+                _ => self.next(),
+            }
+        }
+
+        Err(ScannerError::UnterminatedString)
+    }
+
+    // https://golang.org/ref/spec#String_literals
+    fn scan_raw_string(&mut self) -> Result<Step<'a>> {
+        self.insert_semi = true;
+        self.next();
+
+        while let Some(c) = self.current_char {
+            match c {
+                '`' => {
+                    self.next();
+                    return Ok((self.position(), Token::STRING, self.literal()));
+                }
+                _ => self.next(),
+            }
+        }
+
+        Err(ScannerError::UnterminatedString)
+    }
+
+    // https://golang.org/ref/spec#Comments
+    fn scan_general_comment(&mut self) -> Result<Step<'a>> {
+        self.next();
+        self.next();
+
+        while let Some(c) = self.current_char {
+            match c {
+                '*' => {
+                    self.next();
+                    if matches!(self.current_char, Some('/')) {
+                        self.next();
+
+                        let pos = self.position();
+                        let lit = self.literal();
+
+                        // look for compiler directives
+                        self.directive(&lit["/*".len()..lit.len() - "*/".len()], true)?;
+
+                        return Ok((pos, Token::COMMENT, lit));
+                    }
+                }
+                _ => self.next(),
+            }
+        }
+
+        Err(ScannerError::UnterminatedComment)
+    }
+
+    // https://golang.org/ref/spec#Comments
+    fn scan_line_comment(&mut self) -> Result<Step<'a>> {
+        self.next();
+        self.next();
+
+        while let Some(c) = self.current_char {
+            if is_newline(c) {
+                break;
+            }
+            self.next();
+        }
+
+        let pos = self.position();
+        let lit = self.literal();
+
+        // look for compiler directives (at the beginning of line)
+        if self.start_column == 1 {
+            self.directive(lit["//".len()..].trim_end(), false)?;
+        }
+
+        Ok((pos, Token::COMMENT, self.literal()))
+    }
+
+    // https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives
+    fn directive(&mut self, input: &'a str, immediate: bool) -> Result<()> {
+        if let Some(line_directive) = input.strip_prefix("line ") {
+            self.pending_line_info = self.parse_line_directive(line_directive)?;
+            if immediate {
+                self.consume_pending_line_info();
+            }
+        }
+        Ok(())
+    }
+
+    fn parse_line_directive(&mut self, line_directive: &'a str) -> Result<Option<LineInfo<'a>>> {
+        if let Some((file, line)) = line_directive.rsplit_once(':') {
+            let line = line.parse().map_err(|_| ScannerError::InvalidDirective)?;
+
+            if let Some((file, l)) = file.rsplit_once(':') {
+                if let Ok(l) = l.parse() {
+                    //line :line:col
+                    //line filename:line:col
+                    /*line :line:col*/
+                    /*line filename:line:col*/
+                    let file = if !file.is_empty() { Some(file) } else { None };
+                    let col = Some(line);
+                    let line = l;
+                    let hide_column = false;
+                    return Ok(Some((file, line, col, hide_column)));
+                }
+            }
+
+            //line :line
+            //line filename:line
+            /*line :line*/
+            /*line filename:line*/
+            Ok(Some((Some(file), line, None, true)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    const fn find_line_end(&self) -> bool {
+        let buffer = self.buffer.as_bytes();
+        let mut in_comment = true;
+
+        let mut i = self.offset;
+        let max = self.buffer.len();
+        while i < max {
+            let c = buffer[i] as char;
+
+            if i < max - 1 {
+                let n = buffer[i + 1] as char;
+
+                if !in_comment && c == '/' && n == '/' {
+                    return true;
+                }
+
+                if c == '/' && n == '*' {
+                    i += 2;
+                    in_comment = true;
+                    continue;
+                }
+
+                if c == '*' && n == '/' {
+                    i += 2;
+                    in_comment = false;
+                    continue;
+                }
+            }
+
+            if is_newline(c) {
+                return true;
+            }
+
+            if !in_comment && !matches!(c, ' ' | '\t' | '\r') {
+                return false;
+            }
+
+            i += 1;
+        }
+
+        !in_comment
+    }
+
+    fn consume_pending_line_info(&mut self) {
+        if let Some(line_info) = self.pending_line_info.take() {
+            if let Some(file) = line_info.0 {
+                self.file = file;
+            }
+
+            self.line = line_info.1;
+
+            if let Some(column) = line_info.2 {
+                self.column = column;
+            }
+
+            self.hide_column = line_info.3;
+        }
+    }
+
+    fn peek(&mut self) -> Option<char> {
+        self.chars.peek().copied()
+    }
+
+    fn next(&mut self) {
+        self.offset += self.current_char_len;
+        self.column += self.current_char_len;
+        let last_char = self.current_char;
+
+        self.current_char = self.chars.next();
+        if let Some(c) = self.current_char {
+            self.current_char_len = c.len_utf8();
+            if matches!(last_char, Some('\n')) {
+                self.line += 1;
+                self.column = 1;
+                self.consume_pending_line_info();
+            }
+        } else {
+            self.current_char_len = 0
+        }
+    }
+
+    const fn position(&self) -> Position<'a> {
+        Position {
+            directory: self.directory,
+            file: self.file,
+            offset: self.start_offset,
+            line: self.start_line,
+            column: if self.hide_column {
+                0
+            } else {
+                self.start_column
+            },
+        }
+    }
+
+    fn reset_start(&mut self) {
+        self.start_offset = self.offset;
+        self.start_line = self.line;
+        self.start_column = self.column;
+    }
+
+    fn literal(&self) -> &'a str {
+        &self.buffer[self.start_offset..self.offset]
+    }
+
+    fn require_escaped_char<const DELIM: char>(&mut self) -> Result<()> {
+        self.next();
+
+        let c = self
+            .current_char
+            .ok_or(ScannerError::UnterminatedEscapedChar)?;
+
+        // TODO: move this to the match when const generics can be referenced in patterns
+        if c == DELIM {
+            self.next();
+            return Ok(());
+        }
+
+        match c {
+            'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | '\\' => self.next(),
+            'x' => {
+                self.next();
+                self.require_hex_digits::<2>()?
+            }
+            'u' => {
+                self.next();
+                self.require_hex_digits::<4>()?;
+            }
+            'U' => {
+                self.next();
+                self.require_hex_digits::<8>()?;
+            }
+            '0'..='7' => self.require_octal_digits::<3>()?,
+            _ => return Err(ScannerError::UnterminatedEscapedChar),
+        }
+
+        Ok(())
+    }
+
+    fn require_octal_digits<const COUNT: usize>(&mut self) -> Result<()> {
+        for _ in 0..COUNT {
+            let c = self.current_char.ok_or(ScannerError::OctalNotFound)?;
+
+            if !is_octal_digit(c) {
+                return Err(ScannerError::OctalNotFound);
+            }
+
+            self.next();
+        }
+
+        Ok(())
+    }
+
+    fn require_hex_digits<const COUNT: usize>(&mut self) -> Result<()> {
+        for _ in 0..COUNT {
+            let c = self.current_char.ok_or(ScannerError::HexadecimalNotFound)?;
+
+            if !is_hex_digit(c) {
+                return Err(ScannerError::HexadecimalNotFound);
+            }
+
+            self.next();
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a> IntoIterator for Scanner<'a> {
+    type Item = Result<Step<'a>>;
+    type IntoIter = IntoIter<'a>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        Self::IntoIter::new(self)
+    }
+}
+
+pub struct IntoIter<'a> {
+    scanner: Scanner<'a>,
+    done: bool,
+}
+
+impl<'a> IntoIter<'a> {
+    const fn new(scanner: Scanner<'a>) -> Self {
+        Self { scanner, done: false }
+    }
+}
+
+impl<'a> Iterator for IntoIter<'a> {
+    type Item = Result<Step<'a>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done {
+            return None;
+        }
+
+        match self.scanner.scan() {
+            Ok((pos, tok, lit)) => {
+                if tok == Token::EOF {
+                    self.done = true;
+                }
+                Some(Ok((pos, tok, lit)))
+            }
+            Err(err) => {
+                self.done = true;
+                Some(Err(err))
+            }
+        }
+    }
+}
+
+// https://golang.org/ref/spec#Letters_and_digits
+
+fn is_letter(c: char) -> bool {
+    c == '_' || is_unicode_letter(c)
+}
+
+//const fn is_decimal_digit(c: char) -> bool {
+//matches!(c, '0'..='9')
+//}
+
+//const fn is_binary_digit(c: char) -> bool {
+//matches!(c, '0'..='1')
+//}
+
+const fn is_octal_digit(c: char) -> bool {
+    matches!(c, '0'..='7')
+}
+
+const fn is_hex_digit(c: char) -> bool {
+    matches!(c, '0'..='9' | 'A'..='F' | 'a'..='f')
+}
+
+// https://golang.org/ref/spec#Characters
+
+const fn is_newline(c: char) -> bool {
+    c == '\n'
+}
+
+//const fn is_unicode_char(c: char) -> bool {
+//c != '\n'
+//}
+
+fn is_unicode_letter(c: char) -> bool {
+    matches!(
+        get_general_category(c),
+        GeneralCategory::UppercaseLetter
+            | GeneralCategory::LowercaseLetter
+            | GeneralCategory::TitlecaseLetter
+            | GeneralCategory::ModifierLetter
+            | GeneralCategory::OtherLetter
+    )
+}
+
+fn is_unicode_digit(c: char) -> bool {
+    get_general_category(c) == GeneralCategory::DecimalNumber
+}
+
+// https://golang.org/ref/spec#Keywords
+
+static KEYWORDS: Map<&'static str, Token> = phf_map! {
+  "break" => Token::BREAK,
+  "case" => Token::CASE,
+  "chan" => Token::CHAN,
+  "const" => Token::CONST,
+  "continue" => Token::CONTINUE,
+
+  "default" => Token::DEFAULT,
+  "defer" => Token::DEFER,
+  "else" => Token::ELSE,
+  "fallthrough" => Token::FALLTHROUGH,
+  "for" => Token::FOR,
+
+  "func" => Token::FUNC,
+  "go" => Token::GO,
+  "goto" => Token::GOTO,
+  "if" => Token::IF,
+  "import" => Token::IMPORT,
+
+  "interface" => Token::INTERFACE,
+  "map" => Token::MAP,
+  "package" => Token::PACKAGE,
+  "range" => Token::RANGE,
+  "return" => Token::RETURN,
+
+  "select" => Token::SELECT,
+  "struct" => Token::STRUCT,
+  "switch" => Token::SWITCH,
+  "type" => Token::TYPE,
+  "var" => Token::VAR,
+};
+
+#[cfg(test)]
+mod tests {
+    use super::Scanner;
+
+    #[test] // fuzz
+    fn it_should_return_an_error_on_missing_line_number() {
+        let input = "/*line :*/";
+        let mut out: Vec<_> = Scanner::new(file!(), input).into_iter().collect();
+        assert!(out.pop().unwrap().is_err());
+    }
+}

backend/parsers/windmill-parser-go/src/parser_go_token.rs

@@ -0,0 +1,273 @@
+// https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/go/token/token.go
+
+#![allow(non_camel_case_types)] // For consistency with the Go tokens
+
+use std::fmt;
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Position<'a> {
+    pub directory: &'a str,
+    pub file: &'a str,
+    pub offset: usize,
+    pub line: usize,
+    pub column: usize,
+}
+
+impl<'a> fmt::Display for Position<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.file.is_empty() {
+            write!(f, ":{}:{}", self.line, self.column)
+        } else if self.file.starts_with('/') {
+            write!(f, "{}:{}:{}", self.file, self.line, self.column)
+        } else {
+            write!(
+                f,
+                "{}/{}:{}:{}",
+                self.directory, self.file, self.line, self.column
+            )
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq)]
+pub enum Token {
+    EOF,
+    COMMENT,
+
+    IDENT,  // main
+    INT,    // 12345
+    FLOAT,  // 123.45
+    IMAG,   // 123.45i
+    CHAR,   // 'a'
+    STRING, // "abc"
+
+    ADD, // +
+    SUB, // -
+    MUL, // *
+    QUO, // /
+    REM, // %
+
+    AND,     // &
+    OR,      // |
+    XOR,     // ^
+    SHL,     // <<
+    SHR,     // >>
+    AND_NOT, // &^
+
+    ADD_ASSIGN, // +=
+    SUB_ASSIGN, // -=
+    MUL_ASSIGN, // *=
+    QUO_ASSIGN, // /=
+    REM_ASSIGN, // %=
+
+    AND_ASSIGN,     // &=
+    OR_ASSIGN,      // |=
+    XOR_ASSIGN,     // ^=
+    SHL_ASSIGN,     // <<=
+    SHR_ASSIGN,     // >>=
+    AND_NOT_ASSIGN, // &^=
+
+    LAND,  // &&
+    LOR,   // ||
+    ARROW, // <-
+    INC,   // ++
+    DEC,   // --
+
+    EQL,    // ==
+    LSS,    // <
+    GTR,    // >
+    ASSIGN, // =
+    NOT,    // !
+
+    NEQ,      // !=
+    LEQ,      // <=
+    GEQ,      // >=
+    DEFINE,   // :=
+    ELLIPSIS, // ...
+
+    LPAREN, // (
+    LBRACK, // [
+    LBRACE, // {
+    COMMA,  // ,
+    PERIOD, // .
+
+    RPAREN,    // )
+    RBRACK,    // ]
+    RBRACE,    // }
+    SEMICOLON, // ;
+    COLON,     // :
+
+    BREAK,
+    CASE,
+    CHAN,
+    CONST,
+    CONTINUE,
+
+    DEFAULT,
+    DEFER,
+    ELSE,
+    FALLTHROUGH,
+    FOR,
+
+    FUNC,
+    GO,
+    GOTO,
+    IF,
+    IMPORT,
+
+    INTERFACE,
+    MAP,
+    PACKAGE,
+    RANGE,
+    RETURN,
+
+    SELECT,
+    STRUCT,
+    SWITCH,
+    TYPE,
+    VAR,
+}
+
+impl Token {
+    pub const fn is_assign_op(&self) -> bool {
+        use Token::*;
+        matches!(
+            self,
+            ADD_ASSIGN
+                | SUB_ASSIGN
+                | MUL_ASSIGN
+                | QUO_ASSIGN
+                | REM_ASSIGN
+                | AND_ASSIGN
+                | OR_ASSIGN
+                | XOR_ASSIGN
+                | SHL_ASSIGN
+                | SHR_ASSIGN
+                | AND_NOT_ASSIGN
+        )
+    }
+
+    // https://go.dev/ref/spec#Operator_precedence
+    pub fn precedence(&self) -> u8 {
+        use Token::*;
+        match self {
+            MUL | QUO | REM | SHL | SHR | AND | AND_NOT => 5,
+            ADD | SUB | OR | XOR => 4,
+            EQL | NEQ | LSS | LEQ | GTR | GEQ => 3,
+            LAND => 2,
+            LOR => 1,
+            _ => unreachable!(
+                "precedence() is only supported for binary operators, called with: {:?}",
+                self
+            ),
+        }
+    }
+
+    pub const fn lowest_precedence() -> u8 {
+        0
+    }
+}
+
+impl From<&Token> for &'static str {
+    fn from(token: &Token) -> Self {
+        use Token::*;
+
+        match token {
+            EOF => "EOF",
+            COMMENT => "COMMENT",
+
+            IDENT => "IDENT",
+            INT => "INT",
+            FLOAT => "FLOAT",
+            IMAG => "IMAG",
+            CHAR => "CHAR",
+            STRING => "STRING",
+
+            ADD => "+",
+            SUB => "-",
+            MUL => "*",
+            QUO => "/",
+            REM => "%",
+
+            AND => "&",
+            OR => "|",
+            XOR => "^",
+            SHL => "<<",
+            SHR => ">>",
+            AND_NOT => "&^",
+
+            ADD_ASSIGN => "+=",
+            SUB_ASSIGN => "-=",
+            MUL_ASSIGN => "*=",
+            QUO_ASSIGN => "/=",
+            REM_ASSIGN => "%=",
+
+            AND_ASSIGN => "&=",
+            OR_ASSIGN => "|=",
+            XOR_ASSIGN => "^=",
+            SHL_ASSIGN => "<<=",
+            SHR_ASSIGN => ">>=",
+            AND_NOT_ASSIGN => "&^=",
+
+            LAND => "&&",
+            LOR => "||",
+            ARROW => "<-",
+            INC => "++",
+            DEC => "--",
+
+            EQL => "==",
+            LSS => "<",
+            GTR => ">",
+            ASSIGN => "=",
+            NOT => "!",
+
+            NEQ => "!=",
+            LEQ => "<=",
+            GEQ => ">=",
+            DEFINE => ":=",
+            ELLIPSIS => "...",
+
+            LPAREN => "(",
+            LBRACK => "[",
+            LBRACE => "{",
+            COMMA => ",",
+            PERIOD => ".",
+
+            RPAREN => ")",
+            RBRACK => "]",
+            RBRACE => "}",
+            SEMICOLON => ";",
+            COLON => ":",
+
+            BREAK => "break",
+            CASE => "case",
+            CHAN => "chan",
+            CONST => "const",
+            CONTINUE => "continue",
+
+            DEFAULT => "default",
+            DEFER => "defer",
+            ELSE => "else",
+            FALLTHROUGH => "fallthrough",
+            FOR => "for",
+
+            FUNC => "func",
+            GO => "go",
+            GOTO => "goto",
+            IF => "if",
+            IMPORT => "import",
+
+            INTERFACE => "interface",
+            MAP => "map",
+            PACKAGE => "package",
+            RANGE => "range",
+            RETURN => "return",
+
+            SELECT => "select",
+            STRUCT => "struct",
+            SWITCH => "switch",
+            TYPE => "type",
+            VAR => "var",
+        }
+    }
+}

backend/parsers/windmill-parser-py-imports/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "windmill-parser-py-imports"
-version.workspace = true
-edition.workspace = true
-authors.workspace = true
-
-[lib]
-name = "windmill_parser_py_imports"
-path = "./src/lib.rs"
-
-[dependencies]
-windmill-parser.workspace = true
-windmill-common.workspace = true
-rustpython-parser.workspace = true
-phf.workspace = true
-itertools.workspace = true
-regex.workspace = true
-serde_json.workspace = true
-anyhow.workspace = true
-lazy_static.workspace = true

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -1,459 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-use itertools::Itertools;
-use lazy_static::lazy_static;
-use phf::phf_map;
-use regex::Regex;
-
-use windmill_common::error;
-
-use rustpython_parser::ast::{Located, StmtKind};
-use rustpython_parser::parser::parse_program;
-
-const DEF_MAIN: &str = "def main(";
-
-static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
-    "psycopg2" => "psycopg2-binary",
-    "psycopg" => "psycopg[binary, pool]",
-    "yaml" => "pyyaml",
-    "git" => "GitPython",
-    "u" => "requests",
-    "f" => "requests",
-    "." => "requests",
-    "shopify" => "ShopifyAPI",
-    "seleniumwire" => "selenium-wire",
-    "openbb-terminal" => "openbb[all]",
-    "riskfolio" => "riskfolio-lib",
-    "smb" => "pysmb",
-    "PIL" => "Pillow",
-};
-
-fn replace_import(x: String) -> String {
-    PYTHON_IMPORTS_REPLACEMENT
-        .get(&x)
-        .map(|x| x.to_owned())
-        .unwrap_or(&x)
-        .to_string()
-}
-
-lazy_static! {
-    static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap();
-}
-
-pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
-    let find_requirements = code
-        .lines()
-        .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
-    if let Some((pos, _)) = find_requirements {
-        let lines = code
-            .lines()
-            .skip(pos + 1)
-            .map_while(|x| {
-                RE.captures(x)
-                    .map(|x| x.get(1).unwrap().as_str().to_string())
-            })
-            .collect();
-        Ok(lines)
-    } else {
-        let code = code.split(DEF_MAIN).next().unwrap_or("");
-        let ast = parse_program(code, "main.py").map_err(|e| {
-            error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
-        })?;
-        let mut imports: Vec<String> = ast
-            .into_iter()
-            .filter_map(|x| match x {
-                Located { node, .. } => match node {
-                    StmtKind::Import { names } => Some(
-                        names
-                            .into_iter()
-                            .map(|x| {
-                                let name = x.node.name;
-                                if name.starts_with('.') {
-                                    ".".to_string()
-                                } else {
-                                    name.split('.').next().unwrap_or("").to_string()
-                                }
-                            })
-                            .map(replace_import)
-                            .collect::<Vec<String>>(),
-                    ),
-                    StmtKind::ImportFrom { level: Some(i), .. } if i > 0 => {
-                        Some(vec!["requests".to_string()])
-                    }
-                    StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
-                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
-                        Some(vec![replace_import(imprt)])
-                    }
-                    _ => None,
-                },
-            })
-            .flatten()
-            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
-            .unique()
-            .collect();
-        imports.sort();
-        Ok(imports)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-
-    #[test]
-    fn test_parse_python_imports() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-import wmill
-from zanzibar.estonie import talin
-import matplotlib.pyplot as plt
-from . import tests
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_imports2() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-#requirements:
-#burkina=0.4
-#nigeria
-#
-#congo
-
-import os
-import wmill
-from zanzibar.estonie import talin
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
-
-        Ok(())
-    }
-}
-
-const STDIMPORTS: [&str; 301] = [
-    "__future__",
-    "_abc",
-    "_aix_support",
-    "_ast",
-    "_asyncio",
-    "_bisect",
-    "_blake2",
-    "_bootsubprocess",
-    "_bz2",
-    "_codecs",
-    "_codecs_cn",
-    "_codecs_hk",
-    "_codecs_iso2022",
-    "_codecs_jp",
-    "_codecs_kr",
-    "_codecs_tw",
-    "_collections",
-    "_collections_abc",
-    "_compat_pickle",
-    "_compression",
-    "_contextvars",
-    "_crypt",
-    "_csv",
-    "_ctypes",
-    "_curses",
-    "_curses_panel",
-    "_datetime",
-    "_dbm",
-    "_decimal",
-    "_elementtree",
-    "_frozen_importlib",
-    "_frozen_importlib_external",
-    "_functools",
-    "_gdbm",
-    "_hashlib",
-    "_heapq",
-    "_imp",
-    "_io",
-    "_json",
-    "_locale",
-    "_lsprof",
-    "_lzma",
-    "_markupbase",
-    "_md5",
-    "_msi",
-    "_multibytecodec",
-    "_multiprocessing",
-    "_opcode",
-    "_operator",
-    "_osx_support",
-    "_overlapped",
-    "_pickle",
-    "_posixshmem",
-    "_posixsubprocess",
-    "_py_abc",
-    "_pydecimal",
-    "_pyio",
-    "_queue",
-    "_random",
-    "_sha1",
-    "_sha256",
-    "_sha3",
-    "_sha512",
-    "_signal",
-    "_sitebuiltins",
-    "_socket",
-    "_sqlite3",
-    "_sre",
-    "_ssl",
-    "_stat",
-    "_statistics",
-    "_string",
-    "_strptime",
-    "_struct",
-    "_symtable",
-    "_thread",
-    "_threading_local",
-    "_tkinter",
-    "_tracemalloc",
-    "_uuid",
-    "_warnings",
-    "_weakref",
-    "_weakrefset",
-    "_winapi",
-    "_zoneinfo",
-    "abc",
-    "aifc",
-    "antigravity",
-    "argparse",
-    "array",
-    "ast",
-    "asynchat",
-    "asyncio",
-    "asyncore",
-    "atexit",
-    "audioop",
-    "base64",
-    "bdb",
-    "binascii",
-    "binhex",
-    "bisect",
-    "builtins",
-    "bz2",
-    "cProfile",
-    "calendar",
-    "cgi",
-    "cgitb",
-    "chunk",
-    "cmath",
-    "cmd",
-    "code",
-    "codecs",
-    "codeop",
-    "collections",
-    "colorsys",
-    "compileall",
-    "concurrent",
-    "configparser",
-    "contextlib",
-    "contextvars",
-    "copy",
-    "copyreg",
-    "crypt",
-    "csv",
-    "ctypes",
-    "curses",
-    "dataclasses",
-    "datetime",
-    "dbm",
-    "decimal",
-    "difflib",
-    "dis",
-    "distutils",
-    "doctest",
-    "email",
-    "encodings",
-    "ensurepip",
-    "enum",
-    "errno",
-    "faulthandler",
-    "fcntl",
-    "filecmp",
-    "fileinput",
-    "fnmatch",
-    "fractions",
-    "ftplib",
-    "functools",
-    "gc",
-    "genericpath",
-    "getopt",
-    "getpass",
-    "gettext",
-    "glob",
-    "graphlib",
-    "grp",
-    "gzip",
-    "hashlib",
-    "heapq",
-    "hmac",
-    "html",
-    "http",
-    "idlelib",
-    "imaplib",
-    "imghdr",
-    "imp",
-    "importlib",
-    "inspect",
-    "io",
-    "ipaddress",
-    "itertools",
-    "json",
-    "keyword",
-    "lib2to3",
-    "linecache",
-    "locale",
-    "logging",
-    "lzma",
-    "mailbox",
-    "mailcap",
-    "marshal",
-    "math",
-    "mimetypes",
-    "mmap",
-    "modulefinder",
-    "msilib",
-    "msvcrt",
-    "multiprocessing",
-    "netrc",
-    "nis",
-    "nntplib",
-    "nt",
-    "ntpath",
-    "nturl2path",
-    "numbers",
-    "opcode",
-    "operator",
-    "optparse",
-    "os",
-    "ossaudiodev",
-    "pathlib",
-    "pdb",
-    "pickle",
-    "pickletools",
-    "pipes",
-    "pkgutil",
-    "platform",
-    "plistlib",
-    "poplib",
-    "posix",
-    "posixpath",
-    "pprint",
-    "profile",
-    "pstats",
-    "pty",
-    "pwd",
-    "py_compile",
-    "pyclbr",
-    "pydoc",
-    "pydoc_data",
-    "pyexpat",
-    "queue",
-    "quopri",
-    "random",
-    "re",
-    "readline",
-    "reprlib",
-    "resource",
-    "rlcompleter",
-    "runpy",
-    "sched",
-    "secrets",
-    "select",
-    "selectors",
-    "shelve",
-    "shlex",
-    "shutil",
-    "signal",
-    "site",
-    "smtpd",
-    "smtplib",
-    "sndhdr",
-    "socket",
-    "socketserver",
-    "spwd",
-    "sqlite3",
-    "sre_compile",
-    "sre_constants",
-    "sre_parse",
-    "ssl",
-    "stat",
-    "statistics",
-    "string",
-    "stringprep",
-    "struct",
-    "subprocess",
-    "sunau",
-    "symtable",
-    "sys",
-    "sysconfig",
-    "syslog",
-    "tabnanny",
-    "tarfile",
-    "telnetlib",
-    "tempfile",
-    "termios",
-    "textwrap",
-    "this",
-    "threading",
-    "time",
-    "timeit",
-    "tkinter",
-    "token",
-    "tokenize",
-    "trace",
-    "traceback",
-    "tracemalloc",
-    "tty",
-    "turtle",
-    "turtledemo",
-    "types",
-    "typing",
-    "unicodedata",
-    "unittest",
-    "urllib",
-    "uu",
-    "uuid",
-    "venv",
-    "warnings",
-    "wave",
-    "weakref",
-    "webbrowser",
-    "winreg",
-    "winsound",
-    "wsgiref",
-    "xdrlib",
-    "xml",
-    "xmlrpc",
-    "zipapp",
-    "zipfile",
-    "zipimport",
-    "",
-];

backend/parsers/windmill-parser-py/Cargo.toml

@@ -10,7 +10,11 @@ path = "./src/lib.rs"
 
 [dependencies]
 windmill-parser.workspace = true
+windmill-common.workspace = true
 rustpython-parser.workspace = true
+phf.workspace = true
 itertools.workspace = true
+regex.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
+lazy_static.workspace = true

backend/parsers/windmill-parser-py/src/lib.rs

@@ -9,12 +9,16 @@
 use std::collections::HashMap;
 
 use itertools::Itertools;
+use lazy_static::lazy_static;
+use phf::phf_map;
+use regex::Regex;
 
 use serde_json::json;
+use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ};
 
+use rustpython_parser as parser;
 use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind};
-use rustpython_parser::parser::parse_program;
 
 const DEF_MAIN: &str = "def main(";
 const FUNCTION_CALL: &str = "<function call>";
@@ -53,13 +57,16 @@ fn filter_non_main(code: &str) -> String {
     return filtered_code;
 }
 
-pub fn parse_python_signature(code: &str) -> anyhow::Result<MainArgSignature> {
+pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
     let filtered_code = filter_non_main(code);
     if filtered_code.is_empty() {
-        return Err(anyhow::anyhow!("No main function found".to_string(),));
+        return Err(error::Error::BadRequest(
+            "No main function found".to_string(),
+        ));
     }
-    let ast = parse_program(&filtered_code, "main.py")
-        .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
+    let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| {
+        error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+    })?;
     let param = ast.into_iter().find_map(|x| match x {
         Located { node: StmtKind::FunctionDef { name, args, .. }, .. } if &name == "main" => {
             Some(*args)
@@ -113,7 +120,7 @@ pub fn parse_python_signature(code: &str) -> anyhow::Result<MainArgSignature> {
                 .collect(),
         })
     } else {
-        Err(anyhow::anyhow!(
+        Err(error::Error::ExecutionErr(
             "main function was not findable".to_string(),
         ))
     }
@@ -127,7 +134,10 @@ fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
                 .into_iter()
                 .zip(values)
                 .map(|(k, v)| {
-                    let key = to_value(&k.node)
+                    let key = k
+                        .as_ref()
+                        .map(|x| x.node.clone())
+                        .and_then(|n| to_value(&n))
                         .and_then(|x| match x {
                             serde_json::Value::String(s) => Some(s),
                             _ => None,
@@ -164,6 +174,77 @@ fn constant_to_value(c: &Constant) -> serde_json::Value {
     }
 }
 
+static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
+    "psycopg2" => "psycopg2-binary",
+    "psycopg" => "psycopg[binary, pool]",
+    "yaml" => "pyyaml",
+    "git" => "GitPython",
+    "u" => "requests",
+    "f" => "requests",
+    "shopify" => "ShopifyAPI",
+    "seleniumwire" => "selenium-wire",
+    "openbb-terminal" => "openbb[all]",
+};
+
+fn replace_import(x: String) -> String {
+    PYTHON_IMPORTS_REPLACEMENT
+        .get(&x)
+        .map(|x| x.to_owned())
+        .unwrap_or(&x)
+        .to_string()
+}
+
+lazy_static! {
+    static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap();
+}
+
+pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
+    let find_requirements = code
+        .lines()
+        .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
+    if let Some((pos, _)) = find_requirements {
+        let lines = code
+            .lines()
+            .skip(pos + 1)
+            .map_while(|x| {
+                RE.captures(x)
+                    .map(|x| x.get(1).unwrap().as_str().to_string())
+            })
+            .collect();
+        Ok(lines)
+    } else {
+        let code = code.split(DEF_MAIN).next().unwrap_or("");
+        let ast = parser::parse_program(code, "main.py").map_err(|e| {
+            error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+        })?;
+        let imports = ast
+            .into_iter()
+            .filter_map(|x| match x {
+                Located { node, .. } => match node {
+                    StmtKind::Import { names } => Some(
+                        names
+                            .into_iter()
+                            .map(|x| x.node.name.split('.').next().unwrap_or("").to_string())
+                            .map(replace_import)
+                            .collect::<Vec<String>>(),
+                    ),
+                    StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
+                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
+
+                        Some(vec![replace_import(imprt)])
+                    }
+                    _ => None,
+                },
+            })
+            .flatten()
+            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+            .unique()
+            .collect();
+
+        Ok(imports)
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -351,4 +432,353 @@ def main(test1: str,
 
         Ok(())
     }
+
+    #[test]
+    fn test_parse_python_imports() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+import wmill
+from zanzibar.estonie import talin
+import matplotlib.pyplot as plt
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code)?;
+        // println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_imports2() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+#requirements:
+#burkina=0.4
+#nigeria
+#
+#congo
+
+import os
+import wmill
+from zanzibar.estonie import talin
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
+
+        Ok(())
+    }
 }
+
+const STDIMPORTS: [&str; 301] = [
+    "__future__",
+    "_abc",
+    "_aix_support",
+    "_ast",
+    "_asyncio",
+    "_bisect",
+    "_blake2",
+    "_bootsubprocess",
+    "_bz2",
+    "_codecs",
+    "_codecs_cn",
+    "_codecs_hk",
+    "_codecs_iso2022",
+    "_codecs_jp",
+    "_codecs_kr",
+    "_codecs_tw",
+    "_collections",
+    "_collections_abc",
+    "_compat_pickle",
+    "_compression",
+    "_contextvars",
+    "_crypt",
+    "_csv",
+    "_ctypes",
+    "_curses",
+    "_curses_panel",
+    "_datetime",
+    "_dbm",
+    "_decimal",
+    "_elementtree",
+    "_frozen_importlib",
+    "_frozen_importlib_external",
+    "_functools",
+    "_gdbm",
+    "_hashlib",
+    "_heapq",
+    "_imp",
+    "_io",
+    "_json",
+    "_locale",
+    "_lsprof",
+    "_lzma",
+    "_markupbase",
+    "_md5",
+    "_msi",
+    "_multibytecodec",
+    "_multiprocessing",
+    "_opcode",
+    "_operator",
+    "_osx_support",
+    "_overlapped",
+    "_pickle",
+    "_posixshmem",
+    "_posixsubprocess",
+    "_py_abc",
+    "_pydecimal",
+    "_pyio",
+    "_queue",
+    "_random",
+    "_sha1",
+    "_sha256",
+    "_sha3",
+    "_sha512",
+    "_signal",
+    "_sitebuiltins",
+    "_socket",
+    "_sqlite3",
+    "_sre",
+    "_ssl",
+    "_stat",
+    "_statistics",
+    "_string",
+    "_strptime",
+    "_struct",
+    "_symtable",
+    "_thread",
+    "_threading_local",
+    "_tkinter",
+    "_tracemalloc",
+    "_uuid",
+    "_warnings",
+    "_weakref",
+    "_weakrefset",
+    "_winapi",
+    "_zoneinfo",
+    "abc",
+    "aifc",
+    "antigravity",
+    "argparse",
+    "array",
+    "ast",
+    "asynchat",
+    "asyncio",
+    "asyncore",
+    "atexit",
+    "audioop",
+    "base64",
+    "bdb",
+    "binascii",
+    "binhex",
+    "bisect",
+    "builtins",
+    "bz2",
+    "cProfile",
+    "calendar",
+    "cgi",
+    "cgitb",
+    "chunk",
+    "cmath",
+    "cmd",
+    "code",
+    "codecs",
+    "codeop",
+    "collections",
+    "colorsys",
+    "compileall",
+    "concurrent",
+    "configparser",
+    "contextlib",
+    "contextvars",
+    "copy",
+    "copyreg",
+    "crypt",
+    "csv",
+    "ctypes",
+    "curses",
+    "dataclasses",
+    "datetime",
+    "dbm",
+    "decimal",
+    "difflib",
+    "dis",
+    "distutils",
+    "doctest",
+    "email",
+    "encodings",
+    "ensurepip",
+    "enum",
+    "errno",
+    "faulthandler",
+    "fcntl",
+    "filecmp",
+    "fileinput",
+    "fnmatch",
+    "fractions",
+    "ftplib",
+    "functools",
+    "gc",
+    "genericpath",
+    "getopt",
+    "getpass",
+    "gettext",
+    "glob",
+    "graphlib",
+    "grp",
+    "gzip",
+    "hashlib",
+    "heapq",
+    "hmac",
+    "html",
+    "http",
+    "idlelib",
+    "imaplib",
+    "imghdr",
+    "imp",
+    "importlib",
+    "inspect",
+    "io",
+    "ipaddress",
+    "itertools",
+    "json",
+    "keyword",
+    "lib2to3",
+    "linecache",
+    "locale",
+    "logging",
+    "lzma",
+    "mailbox",
+    "mailcap",
+    "marshal",
+    "math",
+    "mimetypes",
+    "mmap",
+    "modulefinder",
+    "msilib",
+    "msvcrt",
+    "multiprocessing",
+    "netrc",
+    "nis",
+    "nntplib",
+    "nt",
+    "ntpath",
+    "nturl2path",
+    "numbers",
+    "opcode",
+    "operator",
+    "optparse",
+    "os",
+    "ossaudiodev",
+    "pathlib",
+    "pdb",
+    "pickle",
+    "pickletools",
+    "pipes",
+    "pkgutil",
+    "platform",
+    "plistlib",
+    "poplib",
+    "posix",
+    "posixpath",
+    "pprint",
+    "profile",
+    "pstats",
+    "pty",
+    "pwd",
+    "py_compile",
+    "pyclbr",
+    "pydoc",
+    "pydoc_data",
+    "pyexpat",
+    "queue",
+    "quopri",
+    "random",
+    "re",
+    "readline",
+    "reprlib",
+    "resource",
+    "rlcompleter",
+    "runpy",
+    "sched",
+    "secrets",
+    "select",
+    "selectors",
+    "shelve",
+    "shlex",
+    "shutil",
+    "signal",
+    "site",
+    "smtpd",
+    "smtplib",
+    "sndhdr",
+    "socket",
+    "socketserver",
+    "spwd",
+    "sqlite3",
+    "sre_compile",
+    "sre_constants",
+    "sre_parse",
+    "ssl",
+    "stat",
+    "statistics",
+    "string",
+    "stringprep",
+    "struct",
+    "subprocess",
+    "sunau",
+    "symtable",
+    "sys",
+    "sysconfig",
+    "syslog",
+    "tabnanny",
+    "tarfile",
+    "telnetlib",
+    "tempfile",
+    "termios",
+    "textwrap",
+    "this",
+    "threading",
+    "time",
+    "timeit",
+    "tkinter",
+    "token",
+    "tokenize",
+    "trace",
+    "traceback",
+    "tracemalloc",
+    "tty",
+    "turtle",
+    "turtledemo",
+    "types",
+    "typing",
+    "unicodedata",
+    "unittest",
+    "urllib",
+    "uu",
+    "uuid",
+    "venv",
+    "warnings",
+    "wave",
+    "weakref",
+    "webbrowser",
+    "winreg",
+    "winsound",
+    "wsgiref",
+    "xdrlib",
+    "xml",
+    "xmlrpc",
+    "zipapp",
+    "zipfile",
+    "zipimport",
+    "",
+];

backend/parsers/windmill-parser-ts/Cargo.toml

@@ -4,20 +4,16 @@ version.workspace = true
 edition.workspace = true
 authors.workspace = true
 
-
 [lib]
 name = "windmill_parser_ts"
 path = "./src/lib.rs"
 
-[target.'cfg(target_arch = "wasm32")'.dependencies]
-wasm-bindgen.workspace = true
-serde-wasm-bindgen.workspace = true
-
 [dependencies]
 windmill-parser.workspace = true
+windmill-common.workspace = true
+deno_core.workspace = true
 swc_common.workspace = true
 swc_ecma_parser.workspace = true
 swc_ecma_ast.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
-convert_case.workspace = true

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -1,4 +1,3 @@
-use convert_case::{Case, Casing};
 /*
  * Author: Ruben Fiszel
  * Copyright: Windmill Labs, Inc 2022
@@ -6,26 +5,22 @@ use convert_case::{Case, Casing};
  * Please see the included NOTICE for copyright information and
  * LICENSE-AGPL for a copy of the license.
  */
-// use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
-use serde_json::Value;
+use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
+use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};
 
-use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
+use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned};
 use swc_ecma_ast::{
     ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
-    ModuleDecl, ModuleItem, Number, ObjectLit, Param, Pat, Str, TsArrayType, TsEntityName,
-    TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsParenthesizedType,
-    TsPropertySignature, TsType, TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType,
-    TsUnionType,
+    ModuleDecl, ModuleItem, Number, ObjectLit, Pat, Str, TsArrayType, TsEntityName, TsKeywordType,
+    TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsPropertySignature, TsType,
+    TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
 };
 use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
 
-#[cfg(target_arch = "wasm32")]
-use wasm_bindgen::prelude::*;
-
-pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainArgSignature> {
+pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
     let cm: Lrc<SourceMap> = Default::default();
-    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
+    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
     let lexer = Lexer::new(
         // We want to parse ecmascript
         Syntax::Typescript(TsConfig::default()),
@@ -44,9 +39,14 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainA
 
     let ast = parser
         .parse_module()
-        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid typescript"))?
+        .map_err(|_| {
+            error::Error::ExecutionErr(format!(
+                "Error while parsing code, it is invalid typescript"
+            ))
+        })?
         .body;
 
+    // println!("{ast:?}");
     let params = ast.into_iter().find_map(|x| match x {
         ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
             decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, function, .. }),
@@ -54,95 +54,73 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainA
         })) if &sym.to_string() == "main" => Some(function.params),
         _ => None,
     });
-
     if let Some(params) = params {
-        let r = MainArgSignature {
+        Ok(MainArgSignature {
             star_args: false,
             star_kwargs: false,
             args: params
                 .into_iter()
-                .map(|x| parse_param(x, &cm, skip_dflt))
-                .collect::<anyhow::Result<Vec<Arg>>>()?,
-        };
-        Ok(r)
+                .map(|x| match x.pat {
+                    Pat::Ident(ident) => {
+                        let (name, typ, nullable) = binding_ident_to_arg(&ident);
+                        Ok(Arg {
+                            otyp: None,
+                            name,
+                            typ,
+                            default: None,
+                            has_default: ident.id.optional || nullable,
+                        })
+                    }
+                    Pat::Assign(AssignPat { left, right, .. }) => {
+                        let (name, mut typ, _nullable) =
+                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
+                                error::Error::ExecutionErr(format!(
+                                    "parameter syntax unsupported: `{}`",
+                                    cm.span_to_snippet(left.span())
+                                        .unwrap_or_else(|_| cm.span_to_string(left.span()))
+                                ))
+                            })?;
+
+                        let span = match *right {
+                            Expr::Lit(Lit::Str(Str { span, .. })) => Some(span),
+                            Expr::Lit(Lit::Num(Number { span, .. })) => Some(span),
+                            Expr::Lit(Lit::BigInt(BigInt { span, .. })) => Some(span),
+                            Expr::Lit(Lit::Bool(Bool { span, .. })) => Some(span),
+                            Expr::Object(ObjectLit { span, .. }) => Some(span),
+                            Expr::Array(ArrayLit { span, .. }) => Some(span),
+                            _ => None,
+                        };
+                        let expr = span
+                            .and_then(|x| cm.span_to_snippet(x).ok())
+                            .map(|x| serde_json::from_str(&x).map_err(|_| x));
+
+                        let default = match expr.clone() {
+                            Some(Ok(x)) => Some(x),
+                            Some(Err(x)) => eval_sync(&x).ok(),
+                            None => None,
+                        };
+
+                        if typ == Typ::Unknown && default.is_some() {
+                            typ = json_to_typ(default.as_ref().unwrap());
+                        }
+                        Ok(Arg { otyp: None, name, typ, default, has_default: true })
+                    }
+                    _ => Err(error::Error::ExecutionErr(format!(
+                        "parameter syntax unsupported: `{}`",
+                        cm.span_to_snippet(x.span())
+                            .unwrap_or_else(|_| cm.span_to_string(x.span()))
+                    ))),
+                })
+                .collect::<Result<Vec<Arg>, error::Error>>()?,
+        })
     } else {
-        Err(anyhow::anyhow!(
+        Err(error::Error::ExecutionErr(
             "main function was not findable (expected to find 'export function main(...)'"
                 .to_string(),
         ))
     }
 }
 
-fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> anyhow::Result<Arg> {
-    let r = match x.pat {
-        Pat::Ident(ident) => {
-            let (name, typ, nullable) = binding_ident_to_arg(&ident);
-            Ok(Arg {
-                otyp: None,
-                name,
-                typ,
-                default: None,
-                has_default: ident.id.optional || nullable,
-            })
-        }
-        Pat::Assign(AssignPat { left, right, .. }) => {
-            let (name, mut typ, _nullable) =
-                left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
-                    anyhow::anyhow!(
-                        "parameter syntax unsupported: `{}`",
-                        cm.span_to_snippet(left.span())
-                            .unwrap_or_else(|_| cm.span_to_string(left.span()))
-                    )
-                })?;
-
-            let dflt = if skip_dflt {
-                None
-            } else {
-                match *right {
-                    Expr::Lit(Lit::Str(Str { value, .. })) => {
-                        Some(Value::String(value.to_string()))
-                    }
-                    Expr::Lit(Lit::Num(Number { value, .. }))
-                        if (value == (value as u64) as f64) =>
-                    {
-                        Some(serde_json::json!(value as u64))
-                    }
-                    Expr::Lit(Lit::Num(Number { value, .. })) => Some(serde_json::json!(value)),
-                    Expr::Lit(Lit::BigInt(BigInt { value, .. })) => Some(serde_json::json!(value)),
-                    Expr::Lit(Lit::Bool(Bool { value, .. })) => Some(Value::Bool(value)),
-                    Expr::Object(ObjectLit { span, .. }) => eval_span(span, cm),
-                    Expr::Array(ArrayLit { span, .. }) => eval_span(span, cm),
-                    _ => None,
-                }
-            };
-
-            if typ == Typ::Unknown && dflt.is_some() {
-                typ = json_to_typ(dflt.as_ref().unwrap());
-            }
-            Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true })
-        }
-        _ => Err(anyhow::anyhow!(
-            "parameter syntax unsupported: `{}`",
-            cm.span_to_snippet(x.span())
-                .unwrap_or_else(|_| cm.span_to_string(x.span()))
-        )),
-    };
-    r
-}
-
-fn eval_span(span: Span, cm: &Lrc<SourceMap>) -> Option<Value> {
-    let expr = cm
-        .span_to_snippet(span)
-        .ok()
-        .map(|x| serde_json::from_str(&x).map_err(|_| x));
-
-    match expr {
-        Some(Ok(x)) => Some(x),
-        Some(Err(x)) => eval_sync(&x).ok(),
-        None => None,
-    }
-}
-
 fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
     let (typ, nullable) = type_ann
         .as_ref()
@@ -152,7 +130,7 @@ fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String
 }
 
 fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
-    // log(&format!("{:?}", ts_type));
+    //println!("{:?}", ts_type);
     match ts_type {
         TsType::TsKeywordType(t) => (
             match t.kind {
@@ -188,9 +166,6 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
                 .collect();
             (Typ::Object(properties), false)
         }
-        TsType::TsParenthesizedType(TsParenthesizedType { type_ann, .. }) => {
-            tstype_to_typ(type_ann)
-        }
         // TODO: we can do better here and extract the inner type of array
         TsType::TsArrayType(TsArrayType { elem_type, .. }) => {
             (Typ::List(Box::new(tstype_to_typ(&**elem_type).0)), false)
@@ -258,34 +233,221 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
                     ),
                     false,
                 ),
-                "Date" => (Typ::Datetime, false),
                 "Base64" => (Typ::Bytes, false),
                 "Email" => (Typ::Email, false),
                 "Sql" => (Typ::Sql, false),
-                x @ _ => (Typ::Resource(x.to_case(Case::Snake)), false),
+                _ => (Typ::Unknown, false),
             }
         }
         _ => (Typ::Unknown, false),
     }
 }
 
-#[cfg(target_arch = "wasm32")]
-#[wasm_bindgen]
-extern "C" {
-    pub fn eval(s: &str) -> JsValue;
-    pub fn alert(s: &str);
-    // #[wasm_bindgen(js_namespace = console)]
-    // fn log(s: &str);
-
-}
-
-#[cfg(target_arch = "wasm32")]
 pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
-    serde_wasm_bindgen::from_value(eval(format!("let x = {}; x", code).as_str()))
-        .map_err(|err| format!("Cannot deserialize value: {:?}", err))
+    let mut context = JsRuntime::new(RuntimeOptions::default());
+    let code = format!("let x = {}; x", code);
+    let res = context.execute_script("<anon>", &code);
+    match res {
+        Ok(global) => {
+            let scope = &mut context.handle_scope();
+            let local = v8::Local::new(scope, global);
+            let deserialized_value = serde_v8::from_v8::<serde_json::Value>(scope, local);
+
+            match deserialized_value {
+                Ok(value) => Ok(value),
+                Err(err) => Err(format!("Cannot deserialize value: {:?}", err)),
+            }
+        }
+        Err(err) => Err(format!("Evaling error: {:?}", err)),
+    }
 }
 
-#[cfg(not(target_arch = "wasm32"))]
-pub fn eval_sync(_code: &str) -> Result<serde_json::Value, String> {
-    panic!("eval_sync is only available in wasm32")
+#[cfg(test)]
+mod tests {
+
+    use serde_json::json;
+
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+
+    #[test]
+    fn test_parse_deno_sig() -> anyhow::Result<()> {
+        let code = "
+export function main(test1?: string, test2: string = \"burkina\",
+    test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], 
+    email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
+    opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
+    min_object: {a: string, b: number}) {
+    console.log(42)
+}
+";
+        assert_eq!(
+            parse_deno_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: None,
+                        name: "test1".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "test2".to_string(),
+                        typ: Typ::Str(None),
+                        default: Some(json!("burkina")),
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "test3".to_string(),
+                        typ: Typ::Resource("postgres".to_string()),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "b64".to_string(),
+                        typ: Typ::Bytes,
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "ls".to_string(),
+                        typ: Typ::List(Box::new(Typ::Bytes)),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "email".to_string(),
+                        typ: Typ::Email,
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "literal".to_string(),
+                        typ: Typ::Str(Some(vec!["test".to_string()])),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "literal_union".to_string(),
+                        typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])),
+                        default: None,
+                        has_default: false
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "opt_type".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "opt_type_union".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "opt_type_union_union2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "min_object".to_string(),
+                        typ: Typ::Object(vec![
+                            ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
+                            ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Float) }
+                        ]),
+                        default: None,
+                        has_default: false
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_deno_sig_implicit_types() -> anyhow::Result<()> {
+        let code = "
+export function main(test2 = \"burkina\",
+    bool = true,
+    float = 4.2,
+    int = 42,
+    ls = [\"test\"],
+    min_object = {a: \"test\", b: 42}) {
+    console.log(42)
+}
+";
+        assert_eq!(
+            parse_deno_signature(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: None,
+                        name: "test2".to_string(),
+                        typ: Typ::Str(None),
+                        default: Some(json!("burkina")),
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "bool".to_string(),
+                        typ: Typ::Bool,
+                        default: Some(json!(true)),
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "float".to_string(),
+                        typ: Typ::Float,
+                        default: Some(json!(4.2)),
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "int".to_string(),
+                        typ: Typ::Int,
+                        default: Some(json!(42)),
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "ls".to_string(),
+                        typ: Typ::List(Box::new(Typ::Str(None))),
+                        default: Some(json!(["test"])),
+                        has_default: true
+                    },
+                    Arg {
+                        otyp: None,
+                        name: "min_object".to_string(),
+                        typ: Typ::Object(vec![
+                            ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
+                            ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Int) }
+                        ]),
+                        default: Some(json!({"a": "test", "b": 42})),
+                        has_default: true
+                    }
+                ]
+            }
+        );
+
+        Ok(())
+    }
 }

backend/parsers/windmill-parser-wasm/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "windmill-parser-wasm"
-version.workspace = true
-edition.workspace = true
-authors.workspace = true
-
-
-[lib]
-crate-type = ["cdylib"]
-name = "windmill_parser_wasm"
-path = "./src/lib.rs"
-
-[dev-dependencies]
-wasm-bindgen-test.workspace = true
-
-[dependencies]
-anyhow.workspace = true
-windmill-parser.workspace = true
-windmill-parser-go.workspace = true
-windmill-parser-bash.workspace = true
-windmill-parser-py.workspace = true
-windmill-parser-ts.workspace = true
-wasm-bindgen.workspace = true
-serde_json.workspace = true
-

backend/parsers/windmill-parser-wasm/package-lock.json

@@ -1,6 +0,0 @@
-{
-  "name": "windmill-parser-wasm",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {}
-}